From dcae5fb5f0dcc1c97bdb3df0ba870c8cbc95e073 Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Mon, 9 Nov 2015 11:04:42 -0500 Subject: [PATCH 01/95] ensure no operation is blocked before initial license notification closes elastic/elasticsearch#906 update comment Original commit: elastic/x-pack-elasticsearch@0bd788720f2d020ba904cfe03a262c4377da1d13 --- .../shield/license/ShieldLicenseState.java | 5 ++-- .../shield/license/ShieldLicensee.java | 24 +++++++++---------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicenseState.java b/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicenseState.java index 0fbb7602e14..5815014bf45 100644 --- a/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicenseState.java +++ b/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicenseState.java @@ -16,8 +16,9 @@ import org.elasticsearch.license.plugin.core.Licensee.Status; */ public class ShieldLicenseState { - // if we start disabled then we can emit false disabled messages and block legitimate requests... - protected volatile Status status = new Status(OperationMode.TRIAL, LicenseState.ENABLED); + // we initialize the licensee status to enabled with trial operation mode to ensure no + // legitimate requests are blocked before initial license plugin notification + protected volatile Status status = Status.ENABLED; /** * @return true if the license allows for security features to be enabled (authc, authz, ip filter, audit, etc) diff --git a/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicensee.java b/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicensee.java index 797c4159092..32ea9a70b09 100644 --- a/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicensee.java +++ b/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicensee.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.core.License; -import org.elasticsearch.license.core.License.OperationMode; import org.elasticsearch.license.plugin.core.*; import org.elasticsearch.shield.ShieldPlugin; @@ -25,16 +24,16 @@ public class ShieldLicensee extends AbstractLicenseeComponent im @Inject public ShieldLicensee(Settings settings, LicenseeRegistry clientService, ShieldLicenseState shieldLicenseState) { super(settings, ShieldPlugin.NAME, clientService); - add(new Listener() { - @Override - public void onChange(Status status) { - shieldLicenseState.updateStatus(status); - } - }); this.shieldLicenseState = shieldLicenseState; this.isTribeNode = settings.getGroups("tribe", true).isEmpty() == false; } + @Override + public void onChange(Status status) { + super.onChange(status); + shieldLicenseState.updateStatus(status); + } + @Override public String[] expirationMessages() { return new String[] { @@ -74,12 +73,11 @@ public class ShieldLicensee extends AbstractLicenseeComponent im } @Override - protected void doStart() throws ElasticsearchException {; - if (isTribeNode) { - //TODO currently we disable licensing on tribe node. remove this once es core supports merging cluster - this.status = new Status(OperationMode.TRIAL, LicenseState.ENABLED); - shieldLicenseState.updateStatus(status); - } else { + protected void doStart() throws ElasticsearchException { + // we rely on the initial licensee state to be enabled with trial operation mode + // to ensure no operation is blocked due to not registering the licensee on a + // tribe node + if (!isTribeNode) { super.doStart(); } } From c916c5e35b2d1f73e4d5b026defb7ee0cafcd394 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 4 Nov 2015 19:07:14 +0700 Subject: [PATCH 02/95] watcher: If during startup a watch record couldn't be saved because of a version conflict then overwrite it. In certain cases this could cause Watcher to prevent from being started (either automatically or manually): * The watch of the triggered watch doesn't exist * The execution service rejected the execution the watch. (due to not enough threads in the thread pool) If a watch record gets overwritten this means it has been executed before when watcher was running, but we were unable to delete the triggered watch. So we executed the a watch twice with the same trigger event. This can happen if watcher was stopped before because of the node running Watcher unexpectedly stopped (master left, power loss etc.) In stead of failing to write the watch record, we overwrite it set the state 'executed_multiple_times'. Closes elastic/elasticsearch#695 Original commit: elastic/x-pack-elasticsearch@20f805cb9971765a18b0cdc055db3200d5d80b00 --- watcher/docs/release-notes.asciidoc | 10 ++++ .../watcher/execution/ExecutionService.java | 29 +++++++-- .../watcher/execution/ExecutionState.java | 3 +- .../execution/WatchExecutionContext.java | 4 ++ .../watcher/history/HistoryStore.java | 34 +++++++++++ .../test/integration/BootStrapTests.java | 60 +++++++++++++++++-- 6 files changed, 129 insertions(+), 11 deletions(-) diff --git a/watcher/docs/release-notes.asciidoc b/watcher/docs/release-notes.asciidoc index 7fb3fd7cce7..9e9f4ed0fa6 100644 --- a/watcher/docs/release-notes.asciidoc +++ b/watcher/docs/release-notes.asciidoc @@ -47,6 +47,16 @@ bin/plugin remove watcher .Bug fixes * Fixed an issue where the scheduler may get stuck during Watcher startup. This caused no watches to ever fire. +* Fixed an issue where under specific conditions Watcher would not start if there are not finished watch executions from the + previous time that watcher was running and those watch execution are unable the execute during the current start process. + + +[float] +==== 2.0.1 + +.Bug fixes +* Fixed an issue where under specific conditions Watcher would not start if there are not finished watch executions from the + previous time that watcher was running and those watch execution are unable the execute during the current start process. [float] diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionService.java b/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionService.java index dbb08f6a0e9..27c4101e88f 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionService.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionService.java @@ -18,7 +18,6 @@ import org.elasticsearch.watcher.condition.Condition; import org.elasticsearch.watcher.history.HistoryStore; import org.elasticsearch.watcher.history.WatchRecord; import org.elasticsearch.watcher.input.Input; -import org.elasticsearch.watcher.license.WatcherLicensee; import org.elasticsearch.watcher.support.clock.Clock; import org.elasticsearch.watcher.support.validation.WatcherSettingsValidation; import org.elasticsearch.watcher.transform.Transform; @@ -285,7 +284,11 @@ public class ExecutionService extends AbstractComponent { } finally { if (ctx.knownWatch() && record != null && ctx.recordExecution()) { try { - historyStore.put(record); + if (ctx.overrideRecordOnConflict()) { + historyStore.forcePut(record); + } else { + historyStore.put(record); + } } catch (Exception e) { logger.error("failed to update watch record [{}]", e, ctx.id()); } @@ -322,7 +325,11 @@ public class ExecutionService extends AbstractComponent { String message = "failed to run triggered watch [" + triggeredWatch.id() + "] due to thread pool capacity"; logger.debug(message); WatchRecord record = ctx.abortBeforeExecution(ExecutionState.FAILED, message); - historyStore.put(record); + if (ctx.overrideRecordOnConflict()) { + historyStore.forcePut(record); + } else { + historyStore.put(record); + } triggeredWatchStore.delete(triggeredWatch.id()); } } @@ -382,10 +389,10 @@ public class ExecutionService extends AbstractComponent { if (watch == null) { String message = "unable to find watch for record [" + triggeredWatch.id().watchId() + "]/[" + triggeredWatch.id() + "], perhaps it has been deleted, ignoring..."; WatchRecord record = new WatchRecord(triggeredWatch.id(), triggeredWatch.triggerEvent(), ExecutionState.NOT_EXECUTED_WATCH_MISSING, message); - historyStore.put(record); + historyStore.forcePut(record); triggeredWatchStore.delete(triggeredWatch.id()); } else { - TriggeredExecutionContext ctx = new TriggeredExecutionContext(watch, clock.now(DateTimeZone.UTC), triggeredWatch.triggerEvent(), defaultThrottlePeriod); + TriggeredExecutionContext ctx = new StartupExecutionContext(watch, clock.now(DateTimeZone.UTC), triggeredWatch.triggerEvent(), defaultThrottlePeriod); executeAsync(ctx, triggeredWatch); counter++; } @@ -393,6 +400,18 @@ public class ExecutionService extends AbstractComponent { logger.debug("executed [{}] watches from the watch history", counter); } + private final static class StartupExecutionContext extends TriggeredExecutionContext { + + public StartupExecutionContext(Watch watch, DateTime executionTime, TriggerEvent triggerEvent, TimeValue defaultThrottlePeriod) { + super(watch, executionTime, triggerEvent, defaultThrottlePeriod); + } + + @Override + public boolean overrideRecordOnConflict() { + return true; + } + } + private final class WatchExecutionTask implements Runnable { private final WatchExecutionContext ctx; diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionState.java b/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionState.java index 35ec90671b5..2e17151a257 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionState.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionState.java @@ -13,7 +13,8 @@ public enum ExecutionState { THROTTLED, EXECUTED, FAILED, - NOT_EXECUTED_WATCH_MISSING; + NOT_EXECUTED_WATCH_MISSING, + EXECUTED_MULTIPLE_TIMES; public String id() { return name().toLowerCase(Locale.ROOT); diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionContext.java b/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionContext.java index 279c4aeb0c6..cb47bb486eb 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionContext.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionContext.java @@ -93,6 +93,10 @@ public abstract class WatchExecutionContext { return defaultThrottlePeriod; } + public boolean overrideRecordOnConflict() { + return false; + } + public TriggerEvent triggerEvent() { return triggerEvent; } diff --git a/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java b/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java index 16e57e96d39..62432a5e15b 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java @@ -11,6 +11,8 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.engine.VersionConflictEngineException; +import org.elasticsearch.watcher.execution.ExecutionState; import org.elasticsearch.watcher.support.init.proxy.ClientProxy; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; @@ -60,6 +62,10 @@ public class HistoryStore extends AbstractComponent { } + /** + * Stores the specified watchRecord. + * If the specified watchRecord already was stored this call will fail with a version conflict. + */ public void put(WatchRecord watchRecord) throws Exception { if (!started.get()) { throw new IllegalStateException("unable to persist watch record history store is not ready"); @@ -78,6 +84,34 @@ public class HistoryStore extends AbstractComponent { } } + /** + * Stores the specified watchRecord. + * Any existing watchRecord will be overwritten. + */ + public void forcePut(WatchRecord watchRecord) throws Exception { + if (!started.get()) { + throw new IllegalStateException("unable to persist watch record history store is not ready"); + } + String index = getHistoryIndexNameForTime(watchRecord.triggerEvent().triggeredTime()); + putUpdateLock.lock(); + try { + IndexRequest request = new IndexRequest(index, DOC_TYPE, watchRecord.id().value()) + .source(XContentFactory.jsonBuilder().value(watchRecord)) + .opType(IndexRequest.OpType.CREATE); + client.index(request, (TimeValue) null); + } catch (VersionConflictEngineException vcee) { + logger.warn("watch record [{}] has executed multiple times, this can happen during watcher restarts", watchRecord); + watchRecord = new WatchRecord(watchRecord, ExecutionState.EXECUTED_MULTIPLE_TIMES, "watch record has been stored before, previous state [" + watchRecord.state() + "]"); + IndexRequest request = new IndexRequest(index, DOC_TYPE, watchRecord.id().value()) + .source(XContentFactory.jsonBuilder().value(watchRecord)); + client.index(request, (TimeValue) null); + } catch (IOException ioe) { + throw ioException("failed to persist watch record [{}]", ioe, watchRecord); + } finally { + putUpdateLock.unlock(); + } + } + /** * Calculates the correct history index name for a given time */ diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BootStrapTests.java b/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BootStrapTests.java index 346b9335d26..45b3eb666da 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BootStrapTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BootStrapTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.watcher.test.integration; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -13,13 +12,12 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.watcher.WatcherState; +import org.elasticsearch.watcher.client.WatchSourceBuilder; +import org.elasticsearch.watcher.client.WatchSourceBuilders; import org.elasticsearch.watcher.condition.Condition; import org.elasticsearch.watcher.condition.always.AlwaysCondition; import org.elasticsearch.watcher.condition.compare.CompareCondition; -import org.elasticsearch.watcher.execution.ExecutionState; -import org.elasticsearch.watcher.execution.TriggeredWatch; -import org.elasticsearch.watcher.execution.TriggeredWatchStore; -import org.elasticsearch.watcher.execution.Wid; +import org.elasticsearch.watcher.execution.*; import org.elasticsearch.watcher.history.HistoryStore; import org.elasticsearch.watcher.history.WatchRecord; import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; @@ -29,6 +27,7 @@ import org.elasticsearch.watcher.watch.Watch; import org.elasticsearch.watcher.watch.WatchStore; import org.hamcrest.Matchers; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import java.util.concurrent.TimeUnit; @@ -37,10 +36,12 @@ import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.watcher.actions.ActionBuilders.indexAction; +import static org.elasticsearch.watcher.actions.ActionBuilders.loggingAction; import static org.elasticsearch.watcher.client.WatchSourceBuilders.watchBuilder; import static org.elasticsearch.watcher.condition.ConditionBuilders.alwaysCondition; import static org.elasticsearch.watcher.condition.ConditionBuilders.compareCondition; import static org.elasticsearch.watcher.input.InputBuilders.searchInput; +import static org.elasticsearch.watcher.input.InputBuilders.simpleInput; import static org.elasticsearch.watcher.test.WatcherTestUtils.newInputSearchRequest; import static org.elasticsearch.watcher.trigger.TriggerBuilders.schedule; import static org.elasticsearch.watcher.trigger.schedule.Schedules.cron; @@ -354,4 +355,53 @@ public class BootStrapTests extends AbstractWatcherIntegrationTestCase { response = watcherClient().prepareWatcherStats().get(); assertThat(response.getWatcherMetaData().manuallyStopped(), is(false)); } + + public void testWatchRecordSavedTwice() throws Exception { + // Watcher could prevent to start if a watch record tried to executed twice or more and the watch didn't exist + // for that watch record or the execution threadpool rejected the watch record. + // A watch record without a watch is the easiest to simulate, so that is what this test does. + + DateTime triggeredTime = new DateTime(2015, 11, 5, 0, 0, 0, 0, DateTimeZone.UTC); + final String watchRecordIndex = HistoryStore.getHistoryIndexNameForTime(triggeredTime); + + int numRecords = scaledRandomIntBetween(8, 32); + for (int i = 0; i < numRecords; i++) { + String watchId = Integer.toString(i); + ScheduleTriggerEvent event = new ScheduleTriggerEvent(watchId, triggeredTime, triggeredTime); + Wid wid = new Wid(watchId, 0, triggeredTime); + TriggeredWatch triggeredWatch = new TriggeredWatch(wid, event); + client().prepareIndex(TriggeredWatchStore.INDEX_NAME, TriggeredWatchStore.DOC_TYPE, triggeredWatch.id().value()) + .setSource(jsonBuilder().value(triggeredWatch)) + .get(); + + WatchRecord watchRecord = new WatchRecord(wid, event, ExecutionState.EXECUTED, "executed"); + client().prepareIndex(watchRecordIndex, HistoryStore.DOC_TYPE, watchRecord.id().value()) + .setSource(jsonBuilder().value(watchRecord)) + .get(); + + } + + stopWatcher(); + startWatcher(); + assertBusy(new Runnable() { + + @Override + public void run() { + // We need to wait until all the records are processed from the internal execution queue, only then we can assert + // that numRecords watch records have been processed as part of starting up. + WatcherStatsResponse response = watcherClient().prepareWatcherStats().get(); + assertThat(response.getWatcherState(), equalTo(WatcherState.STARTED)); + assertThat(response.getThreadPoolQueueSize(), equalTo(0l)); + + // but even then since the execution of the watch record is async it may take a little bit before + // the actual documents are in the output index + refresh(); + SearchResponse searchResponse = client().prepareSearch(watchRecordIndex).setSize(numRecords).get(); + assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo((long) numRecords)); + for (int i = 0; i < numRecords; i++) { + assertThat(searchResponse.getHits().getAt(i).getSource().get("state"), Matchers.equalTo("executed_multiple_times")); + } + } + }); + } } From 3a66aeb6a46cf308123e01d7bce61b5ddaa0b7c0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 10 Nov 2015 07:54:19 -0800 Subject: [PATCH 03/95] Build: Remove repositories section now that it is included by build-tools With elastic/elasticsearchelastic/elasticsearch#14645, repositories, including the lucene snapshot repo, are now handled by the build plugin itself. This change removes this from xplugins. Original commit: elastic/x-pack-elasticsearch@8e0c7ef8947b859329d2826ffa28d1d434813ff3 --- build.gradle | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/build.gradle b/build.gradle index 5805ea21882..04c5da67800 100644 --- a/build.gradle +++ b/build.gradle @@ -1,29 +1,11 @@ -import org.elasticsearch.gradle.ElasticsearchProperties - allprojects { apply plugin: 'idea' apply plugin: 'eclipse' } subprojects { - project.group = 'org.elasticsearch' - project.version = ElasticsearchProperties.version - project.ext.luceneVersion = ElasticsearchProperties.luceneVersion - repositories { - mavenCentral() - maven { - name 'sonatype-snapshots' - url 'http://oss.sonatype.org/content/repositories/snapshots/' - } - if (luceneVersion.contains('-snapshot')) { - String revision = (luceneVersion =~ /\d\.\d\.\d-snapshot-(\d+)/)[0][1] - maven { - name 'lucene-snapshots' - url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" - } - } - } - + group = 'org.elasticsearch' + version = org.elasticsearch.gradle.VersionProperties.elasticsearch } if (hasProperty('projectsPrefix') == false) { From 39e58b03d5c2357e21e908fad856040ca36c6be5 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 10 Nov 2015 23:10:04 +0700 Subject: [PATCH 04/95] Fix upstream changes to IndexSearcherWrapper: https://github.com/elastic/elasticsearch/pull/14654 Original commit: elastic/x-pack-elasticsearch@b32ba2ad7825051c9b019c789fb7814dd7c0c8b0 --- .../accesscontrol/ShieldIndexSearcherWrapper.java | 8 ++++---- .../ShieldIndexSearcherWrapperIntegrationTests.java | 4 +--- .../ShieldIndexSearcherWrapperUnitTests.java | 10 +++------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java b/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java index eb925c0f885..c195c366c29 100644 --- a/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java +++ b/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java @@ -131,7 +131,7 @@ public class ShieldIndexSearcherWrapper extends IndexSearcherWrapper { } @Override - protected IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + protected IndexSearcher wrap(IndexSearcher searcher) throws EngineException { if (shieldLicenseState.documentAndFieldLevelSecurityEnabled() == false) { return searcher; } @@ -185,9 +185,9 @@ public class ShieldIndexSearcherWrapper extends IndexSearcherWrapper { } } }; - indexSearcher.setQueryCache(engineConfig.getQueryCache()); - indexSearcher.setQueryCachingPolicy(engineConfig.getQueryCachingPolicy()); - indexSearcher.setSimilarity(engineConfig.getSimilarity()); + indexSearcher.setQueryCache(indexSearcher.getQueryCache()); + indexSearcher.setQueryCachingPolicy(indexSearcher.getQueryCachingPolicy()); + indexSearcher.setSimilarity(indexSearcher.getSimilarity(true)); return indexSearcher; } return searcher; diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java index 1d83f80f767..f1ecc7483b5 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java @@ -67,8 +67,6 @@ public class ShieldIndexSearcherWrapperIntegrationTests extends ESTestCase { public void testDLS() throws Exception { ShardId shardId = new ShardId("_index", 0); - EngineConfig engineConfig = new EngineConfig(shardId, null, null, Settings.EMPTY, null, null, null, null, null, null, null, null, null, null, null, QueryCachingPolicy.ALWAYS_CACHE, null, TimeValue.timeValueMinutes(5)); // can't mock... - MapperService mapperService = mock(MapperService.class); when(mapperService.docMappers(anyBoolean())).thenReturn(Collections.emptyList()); when(mapperService.simpleMatchToIndexNames(anyString())).then(new Answer>() { @@ -153,7 +151,7 @@ public class ShieldIndexSearcherWrapperIntegrationTests extends ESTestCase { ParsedQuery parsedQuery = new ParsedQuery(new TermQuery(new Term("field", values[i]))); when(queryShardContext.parse(any(BytesReference.class))).thenReturn(parsedQuery); DirectoryReader wrappedDirectoryReader = wrapper.wrap(directoryReader); - IndexSearcher indexSearcher = wrapper.wrap(engineConfig, new IndexSearcher(wrappedDirectoryReader)); + IndexSearcher indexSearcher = wrapper.wrap(new IndexSearcher(wrappedDirectoryReader)); int expectedHitCount = valuesHitCount[i]; logger.info("Going to verify hit count with query [{}] with expected total hits [{}]", parsedQuery.query(), expectedHitCount); diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java index 8ae05f39a6e..24143f29380 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java @@ -126,10 +126,8 @@ public class ShieldIndexSearcherWrapperUnitTests extends ESTestCase { public void testWrapSearcherWhenFeatureDisabled() throws Exception { ShardId shardId = new ShardId("_index", 0); - EngineConfig engineConfig = new EngineConfig(shardId, null, null, Settings.EMPTY, null, null, null, null, null, null, new BM25Similarity(), null, null, null, new NoneQueryCache(IndexSettingsModule.newIndexSettings(shardId.index(), Settings.EMPTY, Collections.EMPTY_LIST)), QueryCachingPolicy.ALWAYS_CACHE, null, TimeValue.timeValueMinutes(5)); // can't mock... - IndexSearcher indexSearcher = new IndexSearcher(esIn); - IndexSearcher result = shieldIndexSearcherWrapper.wrap(engineConfig, indexSearcher); + IndexSearcher result = shieldIndexSearcherWrapper.wrap(indexSearcher); assertThat(result, sameInstance(indexSearcher)); } @@ -226,8 +224,6 @@ public class ShieldIndexSearcherWrapperUnitTests extends ESTestCase { public void testDelegateSimilarity() throws Exception { ShardId shardId = new ShardId("_index", 0); - EngineConfig engineConfig = new EngineConfig(shardId, null, null, Settings.EMPTY, null, null, null, null, null, null, new BM25Similarity(), null, null, null, new NoneQueryCache(IndexSettingsModule.newIndexSettings(shardId.index(), Settings.EMPTY, Collections.EMPTY_LIST)), QueryCachingPolicy.ALWAYS_CACHE, null, TimeValue.timeValueMinutes(5)); // can't mock... - IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.emptyList()); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null), new BitsetFilterCache.Listener() { @Override @@ -242,9 +238,9 @@ public class ShieldIndexSearcherWrapperUnitTests extends ESTestCase { }); DirectoryReader directoryReader = DocumentSubsetReader.wrap(esIn, bitsetFilterCache, new MatchAllDocsQuery()); IndexSearcher indexSearcher = new IndexSearcher(directoryReader); - IndexSearcher result = shieldIndexSearcherWrapper.wrap(engineConfig, indexSearcher); + IndexSearcher result = shieldIndexSearcherWrapper.wrap(indexSearcher); assertThat(result, not(sameInstance(indexSearcher))); - assertThat(result.getSimilarity(true), sameInstance(engineConfig.getSimilarity())); + assertThat(result.getSimilarity(true), sameInstance(indexSearcher.getSimilarity(true))); bitsetFilterCache.close(); } From c9dd618a0bf5d319b126b499a15f96a3338a479e Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 11 Nov 2015 10:00:24 -0700 Subject: [PATCH 05/95] Fix @Nullable import in watcher Original commit: elastic/x-pack-elasticsearch@017ead8ff71c0d42ce8025b90c6c7993aeddabee --- .../elasticsearch/watcher/actions/hipchat/HipChatAction.java | 2 +- .../watcher/actions/hipchat/service/HipChatMessage.java | 2 +- .../watcher/actions/hipchat/service/IntegrationAccount.java | 2 +- .../watcher/actions/hipchat/service/SentMessages.java | 2 +- .../watcher/actions/hipchat/service/UserAccount.java | 2 +- .../watcher/actions/hipchat/service/V1Account.java | 2 +- .../org/elasticsearch/watcher/actions/slack/SlackAction.java | 2 +- .../watcher/actions/slack/service/SentMessages.java | 2 +- .../watcher/support/SearchRequestEquivalence.java | 2 +- .../org/elasticsearch/watcher/support/http/HttpResponse.java | 5 ++--- .../watcher/support/xcontent/WatcherXContentParser.java | 2 +- .../watcher/trigger/manual/ManualTriggerEngine.java | 2 +- .../watcher/trigger/schedule/ScheduleTriggerEngine.java | 2 +- 13 files changed, 14 insertions(+), 15 deletions(-) diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatAction.java b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatAction.java index 24df3d09f45..d855bd13581 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatAction.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatAction.java @@ -7,6 +7,7 @@ package org.elasticsearch.watcher.actions.hipchat; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -16,7 +17,6 @@ import org.elasticsearch.watcher.actions.hipchat.service.HipChatMessage; import org.elasticsearch.watcher.actions.hipchat.service.SentMessages; import org.elasticsearch.watcher.support.text.TextTemplate; -import javax.annotation.Nullable; import java.io.IOException; /** diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessage.java b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessage.java index b1301a46922..ab7eb0cc9c4 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessage.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessage.java @@ -6,6 +6,7 @@ package org.elasticsearch.watcher.actions.hipchat.service; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.settings.Settings; @@ -15,7 +16,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.watcher.support.text.TextTemplateEngine; import org.elasticsearch.watcher.support.text.TextTemplate; -import javax.annotation.Nullable; import java.io.IOException; import java.util.*; diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccount.java b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccount.java index 4f65cfcab57..e8cf4a8947b 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccount.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccount.java @@ -7,6 +7,7 @@ package org.elasticsearch.watcher.actions.hipchat.service; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; @@ -19,7 +20,6 @@ import org.elasticsearch.watcher.actions.hipchat.service.HipChatMessage.Format; import org.elasticsearch.watcher.support.http.*; import org.elasticsearch.watcher.support.text.TextTemplateEngine; -import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; import java.util.List; diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/SentMessages.java b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/SentMessages.java index 45b64b02447..c7b71e879f9 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/SentMessages.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/SentMessages.java @@ -5,13 +5,13 @@ */ package org.elasticsearch.watcher.actions.hipchat.service; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.watcher.support.http.HttpRequest; import org.elasticsearch.watcher.support.http.HttpResponse; -import javax.annotation.Nullable; import java.io.IOException; import java.util.Collections; import java.util.Iterator; diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccount.java b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccount.java index 4eed1805b06..d5a24b2f693 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccount.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccount.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.watcher.actions.hipchat.service; +import org.elasticsearch.common.Nullable; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.logging.ESLogger; @@ -19,7 +20,6 @@ import org.elasticsearch.watcher.actions.hipchat.service.HipChatMessage.Format; import org.elasticsearch.watcher.support.http.*; import org.elasticsearch.watcher.support.text.TextTemplateEngine; -import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; import java.util.List; diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/V1Account.java b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/V1Account.java index d5d0dc97c03..167a82974a9 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/V1Account.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/V1Account.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.watcher.actions.hipchat.service; +import org.elasticsearch.common.Nullable; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.logging.ESLogger; @@ -15,7 +16,6 @@ import org.elasticsearch.watcher.actions.hipchat.service.HipChatMessage.Format; import org.elasticsearch.watcher.support.http.*; import org.elasticsearch.watcher.support.text.TextTemplateEngine; -import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; import java.util.List; diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackAction.java b/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackAction.java index ff75efc2df7..bceb5ab7c07 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackAction.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackAction.java @@ -6,6 +6,7 @@ package org.elasticsearch.watcher.actions.slack; +import org.elasticsearch.common.Nullable; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; @@ -15,7 +16,6 @@ import org.elasticsearch.watcher.actions.Action; import org.elasticsearch.watcher.actions.slack.service.SentMessages; import org.elasticsearch.watcher.actions.slack.service.message.SlackMessage; -import javax.annotation.Nullable; import java.io.IOException; /** diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SentMessages.java b/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SentMessages.java index 2c26b81c743..eb702767565 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SentMessages.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SentMessages.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.watcher.actions.slack.service; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; @@ -12,7 +13,6 @@ import org.elasticsearch.watcher.actions.slack.service.message.SlackMessage; import org.elasticsearch.watcher.support.http.HttpRequest; import org.elasticsearch.watcher.support.http.HttpResponse; -import javax.annotation.Nullable; import java.io.IOException; import java.util.Collections; import java.util.Iterator; diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/SearchRequestEquivalence.java b/watcher/src/main/java/org/elasticsearch/watcher/support/SearchRequestEquivalence.java index 04d294bca63..e5d6dc58850 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/support/SearchRequestEquivalence.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/support/SearchRequestEquivalence.java @@ -6,9 +6,9 @@ package org.elasticsearch.watcher.support; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import javax.annotation.Nullable; import java.util.Arrays; import static org.elasticsearch.watcher.support.Exceptions.illegalState; diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpResponse.java b/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpResponse.java index a0d8c62fe78..e59f4659a20 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpResponse.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpResponse.java @@ -6,6 +6,7 @@ package org.elasticsearch.watcher.support.http; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.bytes.BytesArray; @@ -23,8 +24,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.annotation.Nullable; - import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; @@ -218,4 +217,4 @@ public class HttpResponse implements ToXContent { ParseField HEADERS = new ParseField("headers"); ParseField BODY = new ParseField("body"); } -} \ No newline at end of file +} diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentParser.java b/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentParser.java index e5beb8f00ee..e9524f769fd 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentParser.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentParser.java @@ -7,6 +7,7 @@ package org.elasticsearch.watcher.support.xcontent; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; @@ -16,7 +17,6 @@ import org.elasticsearch.watcher.support.clock.SystemClock; import org.elasticsearch.watcher.support.secret.Secret; import org.elasticsearch.watcher.support.secret.SecretService; -import javax.annotation.Nullable; import java.io.IOException; import java.util.List; import java.util.Map; diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEngine.java b/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEngine.java index 800ae4074ea..e57c8ea3ae3 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEngine.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEngine.java @@ -5,13 +5,13 @@ */ package org.elasticsearch.watcher.trigger.manual; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.watcher.support.Exceptions; import org.elasticsearch.watcher.trigger.TriggerEngine; import org.elasticsearch.watcher.trigger.TriggerService; -import javax.annotation.Nullable; import java.io.IOException; import java.util.Collection; import java.util.Map; diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEngine.java b/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEngine.java index 0d91bc05c11..1203c29ad1e 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEngine.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEngine.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.watcher.trigger.schedule; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.watcher.support.WatcherDateTimeUtils; @@ -13,7 +14,6 @@ import org.elasticsearch.watcher.trigger.AbstractTriggerEngine; import org.elasticsearch.watcher.trigger.TriggerService; import org.joda.time.DateTime; -import javax.annotation.Nullable; import java.io.IOException; import java.util.Map; From 26c8e949a481a96008edfab3ee814fb6f3d21153 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 11 Nov 2015 09:18:24 -0800 Subject: [PATCH 06/95] Build: Remove transitive dependencies In elastic/elasticsearchelastic/elasticsearch#14668, transitive dependencies were removed from elasticsearch, and plugins now no longer get transitive deps either. This commit fixes xplugins to remove transitive deps. Original commit: elastic/x-pack-elasticsearch@5c41657b6e2f1558ec2eab4c1e69f5880565be45 --- build.gradle | 2 ++ marvel/build.gradle | 14 ++++++++++++-- shield/build.gradle | 2 +- watcher/build.gradle | 24 +++++++++++++++++------- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/build.gradle b/build.gradle index 04c5da67800..8b8532b21f9 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,5 @@ +apply plugin: 'elasticsearch.project-attachment' + allprojects { apply plugin: 'idea' apply plugin: 'eclipse' diff --git a/marvel/build.gradle b/marvel/build.gradle index c4b7a92b04b..245f21fc837 100644 --- a/marvel/build.gradle +++ b/marvel/build.gradle @@ -12,12 +12,22 @@ configurations { licensePluginZip } +ext.versions = [ + okhttp: '2.3.0' +] + dependencies { licensePluginZip project(path: "${projectsPrefix}:license:plugin") // zip provided project(path: "${projectsPrefix}:license:plugin", configuration: 'runtime') provided project(path: "${projectsPrefix}:shield", configuration: 'runtime') testCompile 'org.elasticsearch:securemock:1.1' - testCompile 'com.squareup.okhttp:mockwebserver:2.3.0' + + // mock web server + testCompile "com.squareup.okhttp:mockwebserver:${versions.okhttp}" + testCompile "com.squareup.okhttp:okhttp:${versions.okhttp}" + testCompile "com.squareup.okhttp:okhttp-ws:${versions.okhttp}" + testCompile 'com.squareup.okio:okio:1.3.0' + testCompile 'org.bouncycastle:bcprov-jdk15on:1.50' } compileJava.options.compilerArgs << '-Xlint:-rawtypes,-unchecked' @@ -41,7 +51,7 @@ processTestResources { integTest { dependsOn configurations.licensePluginZip cluster { - plugin 'installLicensePlugin', configurations.licensePluginZip + plugin 'license', configurations.licensePluginZip } } diff --git a/shield/build.gradle b/shield/build.gradle index 41a04874f77..46fe5fb252b 100644 --- a/shield/build.gradle +++ b/shield/build.gradle @@ -1,5 +1,6 @@ apply plugin: 'elasticsearch.esplugin' esplugin { + name 'shield' description 'Elasticsearch Shield (security)' classname 'org.elasticsearch.shield.ShieldPlugin' isolated false @@ -15,7 +16,6 @@ dependencies { compile project("${projectsPrefix}:license:plugin-api") compile 'dk.brics.automaton:automaton:1.11-8' compile 'com.unboundid:unboundid-ldapsdk:2.3.8' - testCompile "org.elasticsearch:test-framework:${version}" testCompile 'org.slf4j:slf4j-log4j12:1.6.2' testCompile 'org.elasticsearch:securemock:1.1' testCompile 'com.google.jimfs:jimfs:1.0' diff --git a/watcher/build.gradle b/watcher/build.gradle index f956f57c3f4..01eca7c392c 100644 --- a/watcher/build.gradle +++ b/watcher/build.gradle @@ -2,6 +2,7 @@ import org.elasticsearch.gradle.MavenFilteringHack apply plugin: 'elasticsearch.esplugin' esplugin { + name 'watcher' description 'Elasticsearch Watcher' classname 'org.elasticsearch.watcher.WatcherPlugin' isolated false @@ -11,23 +12,32 @@ configurations { licensePluginZip } +ext.versions = [ + okhttp: '2.3.0' +] + dependencies { licensePluginZip project(path: "${projectsPrefix}:license:plugin") // zip provided project(path: "${projectsPrefix}:license:plugin", configuration: 'runtime') provided project(path: "${projectsPrefix}:shield", configuration: 'runtime') compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239' + compile 'com.google.guava:guava:19.0-rc2' + compile 'com.google.code.findbugs:jsr305:3.0.1' compile 'com.sun.mail:javax.mail:1.5.3' compile 'javax.activation:activation:1.1.1' - // this should be "provided"... - provided 'com.github.spullara.mustache.java:compiler:0.9.1' - testCompile "org.elasticsearch:test-framework:${version}" - testCompile('org.subethamail:subethasmtp:3.1.7') { - exclude group: 'javax.mail', module: 'mail' - } - testCompile 'com.squareup.okhttp:mockwebserver:2.3.0' + testCompile 'org.subethamail:subethasmtp:3.1.7' + + // mock web server + testCompile "com.squareup.okhttp:mockwebserver:${versions.okhttp}" + testCompile "com.squareup.okhttp:okhttp:${versions.okhttp}" + testCompile "com.squareup.okhttp:okhttp-ws:${versions.okhttp}" + testCompile 'com.squareup.okio:okio:1.3.0' + testCompile 'org.bouncycastle:bcprov-jdk15on:1.50' + testCompile 'org.slf4j:slf4j-log4j12:1.6.2' + testCompile 'org.slf4j:slf4j-api:1.6.2' testCompile 'org.elasticsearch:securemock:1.1' } From d74de5acc8bb9fad7717d704ace2a3d200b75020 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 11 Nov 2015 08:03:06 -0500 Subject: [PATCH 07/95] shield: only wrap readers if the RequestContext can be located Previously, when the RequestContext could not be located a FieldSubsetReader was returned that only allowed meta fields to be read. This was done for safety in case there was an API missed so we did not leak data. However, this causes issues because some requests in elasticsearch execute on a different thread than the one with the RequestContext so we effectively lose this context and prevent access to the fields in the document. This is especially problematic with update requests, because that means that fields that aren't included in the updated document will be lost. This commit removes the wrapping of the readers in this case and adds tests for bulk updates. Closes elastic/elasticsearch#938 Original commit: elastic/x-pack-elasticsearch@74c8059da027f899952cc50a8e22818c5cea9d4b --- .../ShieldIndexSearcherWrapper.java | 5 +- .../integration/BulkUpdateTests.java | 94 +++++++++++++++++++ 2 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java b/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java index c195c366c29..10f0a6d800f 100644 --- a/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java +++ b/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.logging.support.LoggerMessageFormat; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; @@ -84,8 +83,8 @@ public class ShieldIndexSearcherWrapper extends IndexSearcherWrapper { try { RequestContext context = RequestContext.current(); if (context == null) { - logger.debug("couldn't locate the current request, field level security will only allow meta fields"); - return FieldSubsetReader.wrap(reader, allowedMetaFields); + logger.debug("couldn't locate the current request, field level security will not be applied"); + return reader; } IndicesAccessControl indicesAccessControl = context.getRequest().getFromContext(InternalAuthorizationService.INDICES_PERMISSIONS_KEY); diff --git a/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java b/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java new file mode 100644 index 00000000000..3888dee2355 --- /dev/null +++ b/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.integration; + +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; +import org.elasticsearch.shield.authc.support.SecuredString; +import org.elasticsearch.shield.authc.support.UsernamePasswordToken; +import org.elasticsearch.test.ShieldIntegTestCase; +import org.elasticsearch.test.ShieldSettingsSource; +import org.elasticsearch.test.rest.client.http.HttpResponse; + +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class BulkUpdateTests extends ShieldIntegTestCase { + + @Override + public Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(Node.HTTP_ENABLED, true) + .build(); + } + + public void testThatBulkUpdateDoesNotLoseFields() { + assertThat(client().prepareIndex("index1", "type").setSource("{\"test\": \"test\"}").setId("1").get().isCreated(), is(true)); + GetResponse getResponse = internalCluster().transportClient().prepareGet("index1", "type", "1").setFields("test").get(); + assertThat((String) getResponse.getField("test").getValue(), equalTo("test")); + + if (randomBoolean()) { + flushAndRefresh(); + } + + // update with a new field + assertThat(internalCluster().transportClient().prepareUpdate("index1", "type", "1").setDoc("{\"not test\": \"not test\"}").get().isCreated(), is(false)); + getResponse = internalCluster().transportClient().prepareGet("index1", "type", "1").setFields("test", "not test").get(); + assertThat((String) getResponse.getField("test").getValue(), equalTo("test")); + assertThat((String) getResponse.getField("not test").getValue(), equalTo("not test")); + + // this part is important. Without this, the document may be read from the translog which would bypass the bug where + // FLS kicks in because the request can't be found and only returns meta fields + flushAndRefresh(); + + // do it in a bulk + BulkResponse response = internalCluster().transportClient().prepareBulk().add(client().prepareUpdate("index1", "type", "1").setDoc("{\"bulk updated\": \"bulk updated\"}")).get(); + assertThat(((UpdateResponse)response.getItems()[0].getResponse()).isCreated(), is(false)); + getResponse = internalCluster().transportClient().prepareGet("index1", "type", "1").setFields("test", "not test", "bulk updated").get(); + assertThat((String) getResponse.getField("test").getValue(), equalTo("test")); + assertThat((String) getResponse.getField("not test").getValue(), equalTo("not test")); + assertThat((String) getResponse.getField("bulk updated").getValue(), equalTo("bulk updated")); + } + + public void testThatBulkUpdateDoesNotLoseFieldsHttp() throws IOException { + final String path = "/index1/type/1"; + final String basicAuthHeader = UsernamePasswordToken.basicAuthHeaderValue(ShieldSettingsSource.DEFAULT_USER_NAME, new SecuredString(ShieldSettingsSource.DEFAULT_PASSWORD.toCharArray())); + + httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("PUT").body("{\"test\":\"test\"}").execute(); + HttpResponse response = httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("GET").execute(); + assertThat(response.getBody(), containsString("\"test\":\"test\"")); + + if (randomBoolean()) { + flushAndRefresh(); + } + + //update with new field + httpClient().path(path + "/_update").addHeader("Authorization", basicAuthHeader).method("POST").body("{\"doc\": {\"not test\": \"not test\"}}").execute(); + response = httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("GET").execute(); + assertThat(response.getBody(), containsString("\"test\":\"test\"")); + assertThat(response.getBody(), containsString("\"not test\":\"not test\"")); + + // this part is important. Without this, the document may be read from the translog which would bypass the bug where + // FLS kicks in because the request can't be found and only returns meta fields + flushAndRefresh(); + + // update with bulk + httpClient().path("/_bulk").addHeader("Authorization", basicAuthHeader).method("POST") + .body("{\"update\": {\"_index\": \"index1\", \"_type\": \"type\", \"_id\": \"1\"}}\n{\"doc\": {\"bulk updated\":\"bulk updated\"}}\n") + .execute(); + response = httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("GET").execute(); + assertThat(response.getBody(), containsString("\"test\":\"test\"")); + assertThat(response.getBody(), containsString("\"not test\":\"not test\"")); + assertThat(response.getBody(), containsString("\"bulk updated\":\"bulk updated\"")); + } +} From 1601ce4fbaff082e59aacb2a07bb77d092a791a4 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 11 Nov 2015 13:40:03 -0500 Subject: [PATCH 08/95] add guava as a test dependency for jimfs Original commit: elastic/x-pack-elasticsearch@331f834729273a592b0bfa390008a40d0c1b0a74 --- shield/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/shield/build.gradle b/shield/build.gradle index 46fe5fb252b..b3c7d085834 100644 --- a/shield/build.gradle +++ b/shield/build.gradle @@ -19,6 +19,7 @@ dependencies { testCompile 'org.slf4j:slf4j-log4j12:1.6.2' testCompile 'org.elasticsearch:securemock:1.1' testCompile 'com.google.jimfs:jimfs:1.0' + testCompile 'com.google.guava:guava:16.0.1' // needed by jimfs } forbiddenPatterns { From 3aa76795f996adaa497cd7f3efee77be753be07c Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 12 Nov 2015 14:11:26 +0100 Subject: [PATCH 09/95] Marvel: Ensure ClusterStatsTests waits enough for marvel documents The should wait a bit more and refreshes index between each assertBusy() Closes elastic/elasticsearch#729 Original commit: elastic/x-pack-elasticsearch@8ec702c1dc11d125b446b64b75454b0a90d45a52 --- .../agent/renderer/cluster/ClusterStatsTests.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsTests.java index 537c3991cea..523ceed3337 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector; +import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.search.SearchHit; @@ -62,16 +63,13 @@ public class ClusterStatsTests extends MarvelIntegTestCase { // ok.. we'll start collecting now... updateMarvelInterval(3L, TimeUnit.SECONDS); - awaitMarvelTemplateInstalled(); + awaitMarvelDocsCount(greaterThan(0L), ClusterStatsCollector.TYPE); assertBusy(new Runnable() { @Override public void run() { - logger.debug("--> searching for marvel [{}] documents", ClusterStatsCollector.TYPE); - SearchResponse response = client().prepareSearch().setTypes(ClusterStatsCollector.TYPE).get(); - assertThat(response.getHits().getTotalHits(), greaterThan(0L)); - logger.debug("--> checking that every document contains the expected fields"); + SearchResponse response = client().prepareSearch().setTypes(ClusterStatsCollector.TYPE).get(); String[] filters = ClusterStatsRenderer.FILTERS; for (SearchHit searchHit : response.getHits().getHits()) { Map fields = searchHit.sourceAsMap(); @@ -80,9 +78,9 @@ public class ClusterStatsTests extends MarvelIntegTestCase { assertContains(filter, fields); } } - - logger.debug("--> cluster stats successfully collected"); } }); + + logger.debug("--> cluster stats successfully collected"); } } From 8c5fdc70230d59f44d114e7bc3e8237e4529cf98 Mon Sep 17 00:00:00 2001 From: Chris Earle Date: Thu, 12 Nov 2015 13:45:38 -0500 Subject: [PATCH 10/95] Fixing references to bin/plugin -i Original commit: elastic/x-pack-elasticsearch@708d38174245a09f7c4163541c11726e18be9d41 --- marvel/docs/installing-marvel.asciidoc | 2 +- marvel/docs/release-notes.asciidoc | 4 ++-- shield/docs/public/installing-shield.asciidoc | 2 +- .../integrating-other-auth-systems.asciidoc | 4 ++-- watcher/docs/installing-watcher.asciidoc | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/marvel/docs/installing-marvel.asciidoc b/marvel/docs/installing-marvel.asciidoc index a466e090e08..b9d597f9ca2 100644 --- a/marvel/docs/installing-marvel.asciidoc +++ b/marvel/docs/installing-marvel.asciidoc @@ -36,7 +36,7 @@ complete list of default paths, see {ref}/setup-dir-layout.html#_deb_and_rpm[Dir the Elasticsearch Reference.) To install the Marvel and License plugins on a DEB/RPM package installation, you need to run -`bin/plugin -i` from the `/usr/share/elasticsearch` directory with superuser permissions, and +`bin/plugin install` from the `/usr/share/elasticsearch` directory with superuser permissions, and specify the location of the configuration files by setting `-Des.path.conf`. For example: [source,shell] diff --git a/marvel/docs/release-notes.asciidoc b/marvel/docs/release-notes.asciidoc index 1c8d5b1f178..491afba07b3 100644 --- a/marvel/docs/release-notes.asciidoc +++ b/marvel/docs/release-notes.asciidoc @@ -44,14 +44,14 @@ curl -XPUT localhost:9200/_cluster/settings -d '{ + [source,sh] -------------------------------------------------- -bin/plugin -r marvel +bin/plugin remove marvel-agent -------------------------------------------------- .. Install the latest version of the Marvel plugin: + [source,sh] -------------------------------------------------- -bin/plugin -i elasticsearch/marvel/latest +bin/plugin install marvel-agent -------------------------------------------------- .. Start Elasticsearch and confirm that the node rejoins the cluster and that there are no errors diff --git a/shield/docs/public/installing-shield.asciidoc b/shield/docs/public/installing-shield.asciidoc index eddef5d8703..f00525bf26a 100644 --- a/shield/docs/public/installing-shield.asciidoc +++ b/shield/docs/public/installing-shield.asciidoc @@ -27,7 +27,7 @@ complete list of default paths, see {ref}/setup-dir-layout.html#_deb_and_rpm[Dir the Elasticsearch Reference.) To install the Shield and License plugins on a DEB/RPM package installation, you need to run -`bin/plugin -i` from the `/usr/share/elasticsearch` directory with superuser permissions. For example: +`bin/plugin install` from the `/usr/share/elasticsearch` directory with superuser permissions. For example: [source,shell] ---------------------------------------------------------- diff --git a/shield/docs/public/setting-up-authentication/integrating-other-auth-systems.asciidoc b/shield/docs/public/setting-up-authentication/integrating-other-auth-systems.asciidoc index 8d326ff63b3..62f305ca949 100644 --- a/shield/docs/public/setting-up-authentication/integrating-other-auth-systems.asciidoc +++ b/shield/docs/public/setting-up-authentication/integrating-other-auth-systems.asciidoc @@ -39,12 +39,12 @@ For more information about Elasticsearch plugins, see https://www.elastic.co/gui To use a custom realm: -. Install the realm plugin on each node in the cluster. You run `bin/plugin` with the `--url` +. Install the realm plugin on each node in the cluster. You run `bin/plugin` with the `install` option and specify the location of the zip file that contains the plugin. For example: + [source,shell] ---------------------------------------- -bin/plugin --url file:////example-realm-plugin-1.0.zip --install example-realm-plugin +bin/plugin install file:////example-realm-plugin-1.0.zip ---------------------------------------- . Add a realm configuration of the appropriate realm type to `elasticsearch.yml` in the diff --git a/watcher/docs/installing-watcher.asciidoc b/watcher/docs/installing-watcher.asciidoc index f103ea9b1ac..624d693a8ca 100644 --- a/watcher/docs/installing-watcher.asciidoc +++ b/watcher/docs/installing-watcher.asciidoc @@ -68,7 +68,7 @@ complete list of default paths, see {ref}/setup-dir-layout.html#_deb_and_rpm[Dir the Elasticsearch Reference.) To install the Watcher and License plugins on a DEB/RPM package installation, you need to run -`bin/plugin -i` from the `/usr/share/elasticsearch` directory with superuser permissions. For example: +`bin/plugin install` from the `/usr/share/elasticsearch` directory with superuser permissions. For example: [source,shell] ---------------------------------------------------------- From 557f0d4f8364ecce451e16b5c98b171147af9e74 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 13 Nov 2015 09:35:46 -0700 Subject: [PATCH 11/95] Fix compilation for `newIndexSettings` arity change Original commit: elastic/x-pack-elasticsearch@f94fabfcc9ecec828630b09c0c1a147a5dafa17e --- .../shield/authz/accesscontrol/DocumentSubsetReaderTests.java | 4 ++-- .../ShieldIndexSearcherWrapperIntegrationTests.java | 4 ++-- .../accesscontrol/ShieldIndexSearcherWrapperUnitTests.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java index 0149ff8efd6..26ffdb789b7 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java @@ -42,7 +42,7 @@ public class DocumentSubsetReaderTests extends ESTestCase { @Before public void before() { directory = newDirectory(); - IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST); + IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY); bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null), new BitsetFilterCache.Listener() { @Override public void onCache(ShardId shardId, Accountable accountable) { @@ -157,7 +157,7 @@ public class DocumentSubsetReaderTests extends ESTestCase { IndexWriterConfig iwc = new IndexWriterConfig(null); IndexWriter iw = new IndexWriter(dir, iwc); iw.close(); - IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST); + IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null), new BitsetFilterCache.Listener() { @Override public void onCache(ShardId shardId, Accountable accountable) { diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java index f1ecc7483b5..001cb715745 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java @@ -80,9 +80,9 @@ public class ShieldIndexSearcherWrapperIntegrationTests extends ESTestCase { RequestContext.setCurrent(new RequestContext(request)); IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, null, singleton(new BytesArray("{}"))); request.putInContext(InternalAuthorizationService.INDICES_PERMISSIONS_KEY, new IndicesAccessControl(true, singletonMap("_index", indexAccessControl))); - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.index(), Settings.EMPTY, Collections.EMPTY_LIST); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.index(), Settings.EMPTY); QueryShardContext queryShardContext = mock(QueryShardContext.class); - IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST); + IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null), new BitsetFilterCache.Listener() { @Override public void onCache(ShardId shardId, Accountable accountable) { diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java index 24143f29380..785b7e20ed3 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java @@ -63,7 +63,7 @@ public class ShieldIndexSearcherWrapperUnitTests extends ESTestCase { @Before public void before() throws Exception { Index index = new Index("_index"); - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY, Collections.emptyList()); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY); AnalysisService analysisService = new AnalysisService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap()); @@ -224,7 +224,7 @@ public class ShieldIndexSearcherWrapperUnitTests extends ESTestCase { public void testDelegateSimilarity() throws Exception { ShardId shardId = new ShardId("_index", 0); - IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.emptyList()); + IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null), new BitsetFilterCache.Listener() { @Override public void onCache(ShardId shardId, Accountable accountable) { From 6e482d1a3d511cb25f82204b3ab8b72dbc118fba Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 12 Nov 2015 12:35:43 +0700 Subject: [PATCH 12/95] shield: Also prohibit update requests inside bulk requests if FLS is enabled. We do this already for update requests, but this was forgotten to be checked for bulk requests. Original commit: elastic/x-pack-elasticsearch@8d864a7c9866007ccebcf822512a9fa4fc69a558 --- .../shield/action/ShieldActionModule.java | 6 +-- .../interceptor/BulkRequestInterceptor.java | 40 +++++++++++++++++++ .../integration/FieldLevelSecurityTests.java | 20 ++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionModule.java b/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionModule.java index 53118d78e0e..15e34bf5925 100644 --- a/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionModule.java +++ b/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionModule.java @@ -7,10 +7,7 @@ package org.elasticsearch.shield.action; import org.elasticsearch.common.inject.multibindings.Multibinder; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.shield.action.interceptor.RealtimeRequestInterceptor; -import org.elasticsearch.shield.action.interceptor.RequestInterceptor; -import org.elasticsearch.shield.action.interceptor.SearchRequestInterceptor; -import org.elasticsearch.shield.action.interceptor.UpdateRequestInterceptor; +import org.elasticsearch.shield.action.interceptor.*; import org.elasticsearch.shield.support.AbstractShieldModule; public class ShieldActionModule extends AbstractShieldModule.Node { @@ -29,5 +26,6 @@ public class ShieldActionModule extends AbstractShieldModule.Node { multibinder.addBinding().to(RealtimeRequestInterceptor.class); multibinder.addBinding().to(SearchRequestInterceptor.class); multibinder.addBinding().to(UpdateRequestInterceptor.class); + multibinder.addBinding().to(BulkRequestInterceptor.class); } } diff --git a/shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java b/shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java new file mode 100644 index 00000000000..520661406c6 --- /dev/null +++ b/shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.shield.action.interceptor; + +import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.transport.TransportRequest; + +/** + * Simular to {@link UpdateRequestInterceptor}, but checks if there are update requests embedded in a bulk request. + */ +public class BulkRequestInterceptor extends FieldSecurityRequestInterceptor { + + @Inject + public BulkRequestInterceptor(Settings settings) { + super(settings); + } + + @Override + protected void disableFeatures(BulkRequest bulkRequest) { + for (ActionRequest actionRequest : bulkRequest.requests()) { + if (actionRequest instanceof UpdateRequest) { + throw new ElasticsearchSecurityException("Can't execute an bulk request with update requests embedded if field level security is enabled", RestStatus.BAD_REQUEST); + } + } + } + + @Override + public boolean supports(TransportRequest request) { + return request instanceof BulkRequest; + } +} diff --git a/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java index 606e0808e28..2726a09d6c3 100644 --- a/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexModule; import org.elasticsearch.indices.cache.request.IndicesRequestCache; @@ -764,6 +765,25 @@ public class FieldLevelSecurityTests extends ShieldIntegTestCase { client().prepareUpdate("test", "type", "1").setDoc("field2", "value2") .get(); assertThat(client().prepareGet("test", "type", "1").get().getSource().get("field2").toString(), equalTo("value2")); + + // With field level security enabled the update in bulk is not allowed: + try { + client().prepareBulk() + .putHeader(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)) + .add(new UpdateRequest("test", "type", "1").doc("field2", "value3")) + .get(); + fail("failed, because bulk request with updates shouldn't be allowed if field level security is enabled"); + } catch (ElasticsearchSecurityException e) { + assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); + assertThat(e.getMessage(), equalTo("Can't execute an bulk request with update requests embedded if field level security is enabled")); + } + assertThat(client().prepareGet("test", "type", "1").get().getSource().get("field2").toString(), equalTo("value2")); + + // With no field level security enabled the update in bulk is allowed: + client().prepareBulk() + .add(new UpdateRequest("test", "type", "1").doc("field2", "value3")) + .get(); + assertThat(client().prepareGet("test", "type", "1").get().getSource().get("field2").toString(), equalTo("value3")); } public void testQuery_withRoleWithFieldWildcards() throws Exception { From 101e4ff7ce5504fa3d9cf4b35bc33fc67982b68a Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 11 Nov 2015 00:05:44 +0700 Subject: [PATCH 13/95] test: added test that ensures that field data cache is not bypassed for segment and global ordinal caching when field subset reader hides the requested field. Original commit: elastic/x-pack-elasticsearch@c4864d18ac35e69fb80f1d43301245be43ee62cb --- ...ldDataCacheWithFieldSubsetReaderTests.java | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java new file mode 100644 index 00000000000..8b8cb4274a3 --- /dev/null +++ b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.shield.authz.accesscontrol; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.*; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.*; +import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; +import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; +import org.junit.Before; + +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +public class FieldDataCacheWithFieldSubsetReaderTests extends ESTestCase { + + private SortedSetDVOrdinalsIndexFieldData sortedSetDVOrdinalsIndexFieldData; + private PagedBytesIndexFieldData pagedBytesIndexFieldData; + + private DirectoryReader ir; + + private long numDocs; + private Directory dir; + private DummyAccountingFieldDataCache indexFieldDataCache; + + @Before + public void setup() throws Exception { + Index index = new Index("_name"); + Settings settings = Settings.EMPTY; + CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); + MappedFieldType.Names names = new MappedFieldType.Names("_field"); + FieldDataType fieldDataType = new StringFieldMapper.StringFieldType().fieldDataType(); + indexFieldDataCache = new DummyAccountingFieldDataCache(); + sortedSetDVOrdinalsIndexFieldData = new SortedSetDVOrdinalsIndexFieldData(createIndexSettings(),indexFieldDataCache, names, circuitBreakerService, fieldDataType); + pagedBytesIndexFieldData = new PagedBytesIndexFieldData(createIndexSettings(), names, fieldDataType, indexFieldDataCache, circuitBreakerService); + + dir = newDirectory(); + IndexWriterConfig iwc = new IndexWriterConfig(null); + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + IndexWriter iw = new IndexWriter(dir, iwc); + numDocs = scaledRandomIntBetween(32, 128); + + for (int i = 1; i <= numDocs; i++) { + Document doc = new Document(); + doc.add(new StringField("_field", String.valueOf(i), Field.Store.NO)); + doc.add(new SortedSetDocValuesField("_field", new BytesRef(String.valueOf(i)))); + iw.addDocument(doc); + if (i % 24 == 0) { + iw.commit(); + } + } + iw.close(); + ir = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(dir), new ShardId(index, 0)); + } + + @After + public void destroy() throws Exception { + ir.close(); + dir.close(); + } + + public void testSortedSetDVOrdinalsIndexFieldData_global() throws Exception { + assertThat(indexFieldDataCache.topLevelBuilds, equalTo(0)); + IndexOrdinalsFieldData global = sortedSetDVOrdinalsIndexFieldData.loadGlobal(ir); + AtomicOrdinalsFieldData atomic = global.load(ir.leaves().get(0)); + assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(numDocs)); + assertThat(indexFieldDataCache.topLevelBuilds, equalTo(1)); + + DirectoryReader ir = FieldSubsetReader.wrap(this.ir, Collections.emptySet()); + global = sortedSetDVOrdinalsIndexFieldData.loadGlobal(ir); + atomic = global.load(ir.leaves().get(0)); + assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(0l)); + assertThat(indexFieldDataCache.topLevelBuilds, equalTo(1)); + } + + public void testSortedSetDVOrdinalsIndexFieldData_segment() throws Exception { + for (LeafReaderContext context : ir.leaves()) { + AtomicOrdinalsFieldData atomic = sortedSetDVOrdinalsIndexFieldData.load(context); + assertThat(atomic.getOrdinalsValues().getValueCount(), greaterThanOrEqualTo(1l)); + } + + DirectoryReader ir = FieldSubsetReader.wrap(this.ir, Collections.emptySet()); + for (LeafReaderContext context : ir.leaves()) { + AtomicOrdinalsFieldData atomic = sortedSetDVOrdinalsIndexFieldData.load(context); + assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(0l)); + } + // dv based field data doesn't use index field data cache, so in the end noting should have been added + assertThat(indexFieldDataCache.leafLevelBuilds, equalTo(0)); + } + + public void testPagedBytesIndexFieldData_global() throws Exception { + assertThat(indexFieldDataCache.topLevelBuilds, equalTo(0)); + IndexOrdinalsFieldData global = pagedBytesIndexFieldData.loadGlobal(ir); + AtomicOrdinalsFieldData atomic = global.load(ir.leaves().get(0)); + assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(numDocs)); + assertThat(indexFieldDataCache.topLevelBuilds, equalTo(1)); + + DirectoryReader ir = FieldSubsetReader.wrap(this.ir, Collections.emptySet()); + global = pagedBytesIndexFieldData.loadGlobal(ir); + atomic = global.load(ir.leaves().get(0)); + assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(0l)); + assertThat(indexFieldDataCache.topLevelBuilds, equalTo(1)); + } + + public void testPagedBytesIndexFieldData_segment() throws Exception { + assertThat(indexFieldDataCache.leafLevelBuilds, equalTo(0)); + for (LeafReaderContext context : ir.leaves()) { + AtomicOrdinalsFieldData atomic = pagedBytesIndexFieldData.load(context); + assertThat(atomic.getOrdinalsValues().getValueCount(), greaterThanOrEqualTo(1l)); + } + assertThat(indexFieldDataCache.leafLevelBuilds, equalTo(ir.leaves().size())); + + DirectoryReader ir = FieldSubsetReader.wrap(this.ir, Collections.emptySet()); + for (LeafReaderContext context : ir.leaves()) { + AtomicOrdinalsFieldData atomic = pagedBytesIndexFieldData.load(context); + assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(0l)); + } + assertThat(indexFieldDataCache.leafLevelBuilds, equalTo(ir.leaves().size())); + } + + private IndexSettings createIndexSettings() { + Settings settings = Settings.EMPTY; + IndexMetaData indexMetaData = IndexMetaData.builder("_name") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(); + return new IndexSettings(indexMetaData, settings, Collections.emptyList()); + } + + private static class DummyAccountingFieldDataCache implements IndexFieldDataCache { + + private int leafLevelBuilds = 0; + private int topLevelBuilds = 0; + + @Override + public > FD load(LeafReaderContext context, IFD indexFieldData) throws Exception { + leafLevelBuilds++; + return indexFieldData.loadDirect(context); + } + + @Override + public > IFD load(DirectoryReader indexReader, IFD indexFieldData) throws Exception { + topLevelBuilds++; + return (IFD) indexFieldData.localGlobalDirect(indexReader); + } + + @Override + public void clear() { + } + + @Override + public void clear(String fieldName) { + } + + } + +} From 46044a4fe021a15064c7674542704888d897b9ce Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 11 Nov 2015 00:05:44 +0700 Subject: [PATCH 14/95] test: use one IndexSetting instance Original commit: elastic/x-pack-elasticsearch@1f35455ebe3d0da135ff6a4506077763a7e8efe1 --- .../FieldDataCacheWithFieldSubsetReaderTests.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java index 8b8cb4274a3..fbb800d0a4e 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java @@ -48,14 +48,13 @@ public class FieldDataCacheWithFieldSubsetReaderTests extends ESTestCase { @Before public void setup() throws Exception { - Index index = new Index("_name"); - Settings settings = Settings.EMPTY; + IndexSettings indexSettings = createIndexSettings(); CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); MappedFieldType.Names names = new MappedFieldType.Names("_field"); FieldDataType fieldDataType = new StringFieldMapper.StringFieldType().fieldDataType(); indexFieldDataCache = new DummyAccountingFieldDataCache(); - sortedSetDVOrdinalsIndexFieldData = new SortedSetDVOrdinalsIndexFieldData(createIndexSettings(),indexFieldDataCache, names, circuitBreakerService, fieldDataType); - pagedBytesIndexFieldData = new PagedBytesIndexFieldData(createIndexSettings(), names, fieldDataType, indexFieldDataCache, circuitBreakerService); + sortedSetDVOrdinalsIndexFieldData = new SortedSetDVOrdinalsIndexFieldData(indexSettings,indexFieldDataCache, names, circuitBreakerService, fieldDataType); + pagedBytesIndexFieldData = new PagedBytesIndexFieldData(indexSettings, names, fieldDataType, indexFieldDataCache, circuitBreakerService); dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(null); @@ -73,7 +72,7 @@ public class FieldDataCacheWithFieldSubsetReaderTests extends ESTestCase { } } iw.close(); - ir = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(dir), new ShardId(index, 0)); + ir = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(dir), new ShardId(indexSettings.getIndex(), 0)); } @After From 0b50bbb5e57235f34dd8cf8bb621ccd1d61f270c Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Mon, 16 Nov 2015 15:34:26 +0100 Subject: [PATCH 15/95] Fix compilation errors with org.elasticsearch.cluster.health.ClusterHealthStatus Original commit: elastic/x-pack-elasticsearch@68988b2163f2a374130667ed3981d93a0b36b212 --- .../agent/collector/cluster/ClusterStateMarvelDoc.java | 2 +- .../marvel/agent/exporter/http/HttpExporterTests.java | 6 ++---- .../marvel/agent/exporter/local/LocalExporterTests.java | 2 +- .../java/org/elasticsearch/test/ShieldIntegTestCase.java | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateMarvelDoc.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateMarvelDoc.java index f1b07c5f448..eec8ccd171e 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateMarvelDoc.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateMarvelDoc.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.marvel.agent.collector.cluster; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; public class ClusterStateMarvelDoc extends MarvelDoc { diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java index 01c31f38d28..7f42009e980 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java @@ -11,9 +11,9 @@ import com.squareup.okhttp.mockwebserver.QueueDispatcher; import com.squareup.okhttp.mockwebserver.RecordedRequest; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; @@ -40,9 +40,7 @@ import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; @ESIntegTestCase.ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class HttpExporterTests extends MarvelIntegTestCase { diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java index c745259fa83..76b0d577ff3 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java @@ -6,7 +6,7 @@ package org.elasticsearch.marvel.agent.exporter.local; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.search.SearchResponse; diff --git a/shield/src/test/java/org/elasticsearch/test/ShieldIntegTestCase.java b/shield/src/test/java/org/elasticsearch/test/ShieldIntegTestCase.java index 18c5b27f0b7..bb79971db9e 100644 --- a/shield/src/test/java/org/elasticsearch/test/ShieldIntegTestCase.java +++ b/shield/src/test/java/org/elasticsearch/test/ShieldIntegTestCase.java @@ -6,7 +6,7 @@ package org.elasticsearch.test; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.Client; From a901aeef86c23de223bb4d66303418d3b9054478 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 13 Nov 2015 12:40:43 +0100 Subject: [PATCH 16/95] Marvel: reset buffer in HttpExporter The BytesStreamOutput buffer is not reset when exporting multiple documents. It causes the same marvel doc to be indexed many times and it creates very large bulk requests. Bug introduced in elastic/x-pack@8575fd91c48999e1cbeb56c3046184662b626b4a Original commit: elastic/x-pack-elasticsearch@adac96677dd1b6b713a3740e6d34ae429bc07bfc --- .../marvel/agent/exporter/http/HttpExporter.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java index ff64d8d18c7..9ac016d8c63 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java @@ -670,10 +670,13 @@ public class HttpExporter extends Exporter { // because the renderer might close the outputstream (ex: XContentBuilder) try (BytesStreamOutput buffer = new BytesStreamOutput()) { for (MarvelDoc marvelDoc : docs) { - render(marvelDoc, buffer); - - // write the result to the connection - out.write(buffer.bytes().toBytes()); + try { + render(marvelDoc, buffer); + // write the result to the connection + out.write(buffer.bytes().toBytes()); + } finally { + buffer.reset(); + } } } } From 891d348dd1951538a93ed729360a88a82a4e4f79 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 13 Nov 2015 21:18:28 +0100 Subject: [PATCH 17/95] Checks the number of actions in bulk request Original commit: elastic/x-pack-elasticsearch@33678eb7f6d2223ba13a04cf354f24f85280ef43 --- .../exporter/http/HttpExporterTests.java | 57 +++++++++++++++---- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java index 7f42009e980..5a7a251bee5 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java @@ -9,9 +9,15 @@ import com.squareup.okhttp.mockwebserver.MockResponse; import com.squareup.okhttp.mockwebserver.MockWebServer; import com.squareup.okhttp.mockwebserver.QueueDispatcher; import com.squareup.okhttp.mockwebserver.RecordedRequest; +import okio.Buffer; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.bytes.BytesArray; @@ -35,7 +41,9 @@ import org.junit.Before; import java.io.IOException; import java.net.BindException; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -84,10 +92,11 @@ public class HttpExporterTests extends MarvelIntegTestCase { String agentNode = internalCluster().startNode(builder); HttpExporter exporter = getExporter(agentNode); - MarvelDoc doc = newRandomMarvelDoc(); - exporter.export(Collections.singletonList(doc)); - assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(4)); + final int nbDocs = randomIntBetween(1, 25); + exporter.export(newRandomMarvelDocs(nbDocs)); + + assertThat(webServer.getRequestCount(), equalTo(4)); RecordedRequest recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -105,6 +114,8 @@ public class HttpExporterTests extends MarvelIntegTestCase { recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("POST")); assertThat(recordedRequest.getPath(), equalTo("/_bulk")); + + assertBulkRequest(recordedRequest.getBody(), nbDocs); } public void testDynamicHostChange() { @@ -149,9 +160,10 @@ public class HttpExporterTests extends MarvelIntegTestCase { logger.info("--> exporting data"); HttpExporter exporter = getExporter(agentNode); - exporter.export(Collections.singletonList(newRandomMarvelDoc())); + final int nbDocs = randomIntBetween(1, 25); + exporter.export(newRandomMarvelDocs(nbDocs)); - assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(4)); + assertThat(webServer.getRequestCount(), equalTo(4)); RecordedRequest recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -169,6 +181,8 @@ public class HttpExporterTests extends MarvelIntegTestCase { recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("POST")); assertThat(recordedRequest.getPath(), equalTo("/_bulk")); + + assertBulkRequest(recordedRequest.getBody(), nbDocs); } public void testHostChangeReChecksTemplate() throws Exception { @@ -194,7 +208,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { exporter.export(Collections.singletonList(newRandomMarvelDoc())); assertThat(exporter.supportedClusterVersion, is(true)); - assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(4)); + assertThat(webServer.getRequestCount(), equalTo(4)); RecordedRequest recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -247,7 +261,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { logger.info("--> exporting a second event"); exporter.export(Collections.singletonList(newRandomMarvelDoc())); - assertThat(secondWebServer.getRequestCount(), greaterThanOrEqualTo(4)); + assertThat(secondWebServer.getRequestCount(), equalTo(4)); recordedRequest = secondWebServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -295,7 +309,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { exporter.export(Collections.singletonList(newRandomMarvelDoc())); assertThat(exporter.supportedClusterVersion, is(true)); - assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(3)); + assertThat(webServer.getRequestCount(), equalTo(3)); RecordedRequest recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -304,6 +318,11 @@ public class HttpExporterTests extends MarvelIntegTestCase { recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("PUT")); + assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); } public void testUnsupportedClusterVersion() throws Exception { @@ -326,7 +345,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { exporter.export(Collections.singletonList(newRandomMarvelDoc())); assertThat(exporter.supportedClusterVersion, is(false)); - assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(1)); + assertThat(webServer.getRequestCount(), equalTo(1)); RecordedRequest recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -354,7 +373,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { MarvelDoc doc = newRandomMarvelDoc(); exporter.export(Collections.singletonList(doc)); - assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(4)); + assertThat(webServer.getRequestCount(), equalTo(4)); RecordedRequest recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -401,7 +420,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { String expectedMarvelIndex = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(newTimeFormat).withZoneUTC().print(doc.timestamp()); - assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(4)); + assertThat(webServer.getRequestCount(), equalTo(4 + 4)); recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -465,6 +484,14 @@ public class HttpExporterTests extends MarvelIntegTestCase { } } + private List newRandomMarvelDocs(int nb) { + List docs = new ArrayList<>(nb); + for (int i = 0; i < nb; i++) { + docs.add(newRandomMarvelDoc()); + } + return docs; + } + private void enqueueGetClusterVersionResponse(Version v) throws IOException { enqueueGetClusterVersionResponse(webServer, v); } @@ -480,4 +507,12 @@ public class HttpExporterTests extends MarvelIntegTestCase { private void enqueueResponse(MockWebServer mockWebServer, int responseCode, String body) throws IOException { mockWebServer.enqueue(new MockResponse().setResponseCode(responseCode).setBody(body)); } + + private void assertBulkRequest(Buffer requestBody, int numberOfActions) throws Exception { + BulkRequest bulkRequest = Requests.bulkRequest().add(new BytesArray(requestBody.readByteArray()), null, null); + assertThat(bulkRequest.numberOfActions(), equalTo(numberOfActions)); + for (ActionRequest actionRequest : bulkRequest.requests()) { + assertThat(actionRequest, instanceOf(IndexRequest.class)); + } + } } From 0d67caa468c4c160c44529330268798cb1859f26 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 16 Nov 2015 12:45:03 -0800 Subject: [PATCH 18/95] Build: Clean buildSrc on gradle clean Same change as elastic/elasticsearchelastic/elasticsearch#14785 but for xplugins. Original commit: elastic/x-pack-elasticsearch@02e824768614b0ead998eee70e697406bf641994 --- build.gradle | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.gradle b/build.gradle index 8b8532b21f9..429996a4fe1 100644 --- a/build.gradle +++ b/build.gradle @@ -16,6 +16,11 @@ if (hasProperty('projectsPrefix') == false) { } } +task clean(type: GradleBuild) { + buildFile = 'buildSrc/build.gradle' + tasks = ['clean'] +} + elasticsearch-releases http://maven.elasticsearch.org/releases From fc1d043c0d4eef24cdbe3b3a4568cab8f3266284 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 16 Nov 2015 13:04:46 -0800 Subject: [PATCH 19/95] Build: Do not cache snapshots Gradle marks dependencies with a version containing "SNAPSHOT" as "changing", and caches them for 24 hours by default. This updates the configuration to not cache them at all. Original commit: elastic/x-pack-elasticsearch@ac95e30a96b5dc666c42c700b754219d335700ef --- build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.gradle b/build.gradle index 8b8532b21f9..75364a1450e 100644 --- a/build.gradle +++ b/build.gradle @@ -8,6 +8,10 @@ allprojects { subprojects { group = 'org.elasticsearch' version = org.elasticsearch.gradle.VersionProperties.elasticsearch + + configurations.all { + resolutionStrategy.cacheChangingModulesFor(0, 'seconds') + } } if (hasProperty('projectsPrefix') == false) { From cc2096b4f9da33de6184b8fc4821351afd9e62bf Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 17 Nov 2015 07:08:35 -0500 Subject: [PATCH 20/95] add the option to disable DLS and FLS completely This commit reverts a previous change where searcher were not wrapped when the RequestContext could not be found. If DLS/FLS is enabled, which is the default, any bulk request that contains an update request will not be permitted. This change also exposes the ability to completely disable DLS and FLS so that users who are not using these features can still use bulk updates. See elastic/elasticsearch#938 Original commit: elastic/x-pack-elasticsearch@513782db1c2808308085c564ca7324bad22468e0 --- shield/docs/public/limitations.asciidoc | 15 +++++++++++++++ shield/docs/public/reference.asciidoc | 7 +++++++ ...up-field-and-document-level-security.asciidoc | 11 ----------- .../org/elasticsearch/shield/ShieldPlugin.java | 13 ++++++++++--- .../interceptor/BulkRequestInterceptor.java | 14 +++++++++++++- .../ShieldIndexSearcherWrapper.java | 4 ++-- .../integration/BulkUpdateTests.java | 2 ++ .../integration/FieldLevelSecurityTests.java | 16 +++++++++++----- 8 files changed, 60 insertions(+), 22 deletions(-) diff --git a/shield/docs/public/limitations.asciidoc b/shield/docs/public/limitations.asciidoc index 348ac62c3b9..a07335bc21e 100644 --- a/shield/docs/public/limitations.asciidoc +++ b/shield/docs/public/limitations.asciidoc @@ -29,6 +29,21 @@ indices, while Elasticsearch with Shield returns exceptions. Note that this beha multiple items will fail the entire set of operations if any one operation throws an exception due to wildcard expansion resulting in an empty set of authorized indices. +[float] +=== Field and Document Level Security Limitations + +Bulk updates do not work when document and field level security is enabled. If you are not using document and field level +security, bulk updates can be enabled by setting `shield.dls_fls.enabled` to `false`. + +When document level security is enabled for an index: + +* The get, multi get, termsvector and multi termsvector APIs aren't executed in real time. The realtime option for these APIs is forcefully set to false. +* Document level security isn't applied for APIs that aren't document based. An example is the field stats API. +* Document level security doesn't affect global index statistics that relevancy scoring uses. So this means that scores are computed without taking the role query into account. + Note that documents not matching with the role query are never returned. +* The `has_child` and `has_parent` queries aren't supported as role query in the `roles.yml` file. + The `has_child` and `has_parent` queries can be used in the search API with document level security enabled. + [float] === Document Expiration (_ttl) diff --git a/shield/docs/public/reference.asciidoc b/shield/docs/public/reference.asciidoc index f6431b48c5d..cb57571bfdb 100644 --- a/shield/docs/public/reference.asciidoc +++ b/shield/docs/public/reference.asciidoc @@ -184,6 +184,13 @@ The parameters listed in this section are configured in the `config/elasticsearc | `shield.authc.anonymous.authz_exception` | `true` | When `true`, a HTTP 403 response will be returned when the anonymous user does not have the appropriate permissions for the requested action. The user will not be prompted to provide credentials to access the requested resource. When set to `false`, a HTTP 401 will be returned allowing for credentials to be provided for a user with the appropriate permissions. |====== +.Shield Document and Field Level Security Settings +[options="header"] +|====== +| Name | Default | Description +| `shield.dls_fls.enabled` | `true` | This setting can be used to completely disable document and field level security regardless of how roles are configured. +|====== + [[ref-realm-settings]] diff --git a/shield/docs/public/setting-up-field-and-document-level-security.asciidoc b/shield/docs/public/setting-up-field-and-document-level-security.asciidoc index cb6ecc82b3a..a6b2c7583fe 100644 --- a/shield/docs/public/setting-up-field-and-document-level-security.asciidoc +++ b/shield/docs/public/setting-up-field-and-document-level-security.asciidoc @@ -158,14 +158,3 @@ customer_care: privileges: read query: '{"term" : {"department_id" : "12"}}'' -------------------------------------------------- - -===== Limitations - -When document level security is enabled for an index: - -* The get, multi get, termsvector and multi termsvector APIs aren't executed in real time. The realtime option for these APIs is forcefully set to false. -* Document level security isn't applied for APIs that aren't document based oriented. For example this is the case for the field stats API. -* Document level security doesn't affect global index statistics that relevancy scoring uses. So this means that scores are computed without taking the role query into account. - Note that documents not matching with the role query are never returned. -* The `has_child` and `has_parent` queries aren't supported as role query in the `roles.yml` file. - The `has_child` and `has_parent` queries can be used in the search API with document level security enabled. \ No newline at end of file diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java index abf384f17d0..2120195f114 100644 --- a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java +++ b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java @@ -62,6 +62,7 @@ public class ShieldPlugin extends Plugin { public static final String NAME = "shield"; public static final String ENABLED_SETTING_NAME = NAME + ".enabled"; public static final String OPT_OUT_QUERY_CACHE = "opt_out_cache"; + public static final String DLS_FLS_ENABLED_SETTING = "shield.dls_fls.enabled"; private static final boolean DEFAULT_ENABLED_SETTING = true; @@ -159,9 +160,11 @@ public class ShieldPlugin extends Plugin { return; } assert shieldLicenseState != null; - module.setSearcherWrapper((indexService) -> new ShieldIndexSearcherWrapper(indexService.getIndexSettings(), - indexService.getQueryShardContext(), indexService.mapperService(), - indexService.cache().bitsetFilterCache(), shieldLicenseState)); + if (flsDlsEnabled(settings)) { + module.setSearcherWrapper((indexService) -> new ShieldIndexSearcherWrapper(indexService.getIndexSettings(), + indexService.getQueryShardContext(), indexService.mapperService(), + indexService.cache().bitsetFilterCache(), shieldLicenseState)); + } if (clientMode == false) { module.registerQueryCache(ShieldPlugin.OPT_OUT_QUERY_CACHE, OptOutQueryCache::new); } @@ -305,6 +308,10 @@ public class ShieldPlugin extends Plugin { return settings.getAsBoolean(ENABLED_SETTING_NAME, DEFAULT_ENABLED_SETTING); } + public static boolean flsDlsEnabled(Settings settings) { + return settings.getAsBoolean(DLS_FLS_ENABLED_SETTING, true); + } + private void failIfShieldQueryCacheIsNotActive(Settings settings, boolean nodeSettings) { String queryCacheImplementation; if (nodeSettings) { diff --git a/shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java b/shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java index 520661406c6..49992f7d7e3 100644 --- a/shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java +++ b/shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java @@ -12,6 +12,8 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.shield.ShieldPlugin; +import org.elasticsearch.shield.User; import org.elasticsearch.transport.TransportRequest; /** @@ -24,11 +26,21 @@ public class BulkRequestInterceptor extends FieldSecurityRequestInterceptor Date: Tue, 17 Nov 2015 16:28:45 -0500 Subject: [PATCH 21/95] shield: disable document and field level security by default This change disables document and field level security by default so that we are able to maintain bulk update functionality. Users that enable DLS/FLS will not have this functionality. Additionally, if a user tries to configure DLS/FLS in a role without enabling it, the role will be skipped during parsing and a log message will be logged at the error level. See elastic/elasticsearch#938 Original commit: elastic/x-pack-elasticsearch@60c75190921c181b90d7db595e2068bbc0fd067d --- .../elasticsearch/shield/ShieldPlugin.java | 2 +- .../shield/authz/store/FileRolesStore.java | 21 +++-- .../integration/BulkUpdateTests.java | 3 +- .../DocumentAndFieldLevelSecurityTests.java | 10 ++ .../DocumentLevelSecurityRandomTests.java | 11 +++ .../DocumentLevelSecurityTests.java | 10 ++ .../FieldLevelSecurityRandomTests.java | 11 +++ .../integration/FieldLevelSecurityTests.java | 9 ++ ...onsWithAliasesWildcardsAndRegexsTests.java | 11 +++ .../authz/store/FileRolesStoreTests.java | 91 +++++++++++++++++-- .../shield/authz/store/roles.yml | 25 ++++- 11 files changed, 185 insertions(+), 19 deletions(-) diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java index 2120195f114..db911ea1e27 100644 --- a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java +++ b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java @@ -309,7 +309,7 @@ public class ShieldPlugin extends Plugin { } public static boolean flsDlsEnabled(Settings settings) { - return settings.getAsBoolean(DLS_FLS_ENABLED_SETTING, true); + return settings.getAsBoolean(DLS_FLS_ENABLED_SETTING, false); } private void failIfShieldQueryCacheIsNotActive(Settings settings, boolean nodeSettings) { diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java b/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java index e95aa782437..4882ab795d5 100644 --- a/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java +++ b/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java @@ -90,7 +90,7 @@ public class FileRolesStore extends AbstractLifecycleComponent imple } catch (IOException e) { throw new ElasticsearchException("failed to setup roles file watcher", e); } - permissions = parseFile(file, reservedRoles, logger); + permissions = parseFile(file, reservedRoles, logger, settings); } @Override @@ -116,18 +116,18 @@ public class FileRolesStore extends AbstractLifecycleComponent imple } public static Set parseFileForRoleNames(Path path, ESLogger logger) { - Map roleMap = parseFile(path, Collections.emptySet(), logger, false); + Map roleMap = parseFile(path, Collections.emptySet(), logger, false, Settings.EMPTY); if (roleMap == null) { return emptySet(); } return roleMap.keySet(); } - public static Map parseFile(Path path, Set reservedRoles, ESLogger logger) { - return parseFile(path, reservedRoles, logger, true); + public static Map parseFile(Path path, Set reservedRoles, ESLogger logger, Settings settings) { + return parseFile(path, reservedRoles, logger, true, settings); } - public static Map parseFile(Path path, Set reservedRoles, ESLogger logger, boolean resolvePermission) { + public static Map parseFile(Path path, Set reservedRoles, ESLogger logger, boolean resolvePermission, Settings settings) { if (logger == null) { logger = NoOpLogger.INSTANCE; } @@ -138,7 +138,7 @@ public class FileRolesStore extends AbstractLifecycleComponent imple try { List roleSegments = roleSegments(path); for (String segment : roleSegments) { - Permission.Global.Role role = parseRole(segment, path, logger, resolvePermission); + Permission.Global.Role role = parseRole(segment, path, logger, resolvePermission, settings); if (role != null) { if (SystemRole.NAME.equals(role.name())) { logger.warn("role [{}] is reserved to the system. the relevant role definition in the mapping file will be ignored", SystemRole.NAME); @@ -164,7 +164,7 @@ public class FileRolesStore extends AbstractLifecycleComponent imple return unmodifiableMap(roles); } - private static Permission.Global.Role parseRole(String segment, Path path, ESLogger logger, boolean resolvePermissions) { + private static Permission.Global.Role parseRole(String segment, Path path, ESLogger logger, boolean resolvePermissions, Settings settings) { String roleName = null; try { XContentParser parser = YamlXContent.yamlXContent.createParser(segment); @@ -301,6 +301,11 @@ public class FileRolesStore extends AbstractLifecycleComponent imple } } if (name != null) { + if ((query != null || (fields != null && fields.isEmpty() == false)) && ShieldPlugin.flsDlsEnabled(settings) == false) { + logger.error("invalid role definition [{}] in roles file [{}]. document and field level security is not enabled. set [{}] to [true] in the configuration file. skipping role...", roleName, path.toAbsolutePath(), ShieldPlugin.DLS_FLS_ENABLED_SETTING); + return null; + } + try { permission.add(fields, query, Privilege.Index.get(name), indices); } catch (IllegalArgumentException e) { @@ -417,7 +422,7 @@ public class FileRolesStore extends AbstractLifecycleComponent imple public void onFileChanged(Path file) { if (file.equals(FileRolesStore.this.file)) { try { - permissions = parseFile(file, reservedRoles, logger); + permissions = parseFile(file, reservedRoles, logger, settings); logger.info("updated roles (roles file [{}] changed)", file.toAbsolutePath()); } catch (Throwable t) { logger.error("could not reload roles file [{}]. Current roles remain unmodified", t, file.toAbsolutePath()); diff --git a/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java b/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java index 31c9d6f9744..1da423c508c 100644 --- a/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.Node; -import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.shield.authc.support.UsernamePasswordToken; import org.elasticsearch.test.ShieldIntegTestCase; @@ -30,7 +29,7 @@ public class BulkUpdateTests extends ShieldIntegTestCase { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(Node.HTTP_ENABLED, true) - .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, false) //FIXME randomize once DLS/FLS works with Bulk updates... + //.put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, false) //FIXME randomize once DLS/FLS works with Bulk updates... .build(); } diff --git a/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java index 54687e44d22..c109ba374ed 100644 --- a/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.integration; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.test.ShieldIntegTestCase; @@ -39,6 +40,7 @@ public class DocumentAndFieldLevelSecurityTests extends ShieldIntegTestCase { "role2:user2\n" + "role3:user3\n"; } + @Override protected String configRoles() { return super.configRoles() + @@ -65,6 +67,14 @@ public class DocumentAndFieldLevelSecurityTests extends ShieldIntegTestCase { " query: '{\"term\" : {\"field2\" : \"value2\"}}'\n"; } + @Override + public Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true) + .build(); + } + public void testSimpleQuery() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type1", "field1", "type=string", "field2", "type=string") diff --git a/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java b/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java index 97982f4bfae..179e19cff69 100644 --- a/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java @@ -8,7 +8,9 @@ package org.elasticsearch.integration; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.test.ShieldIntegTestCase; @@ -49,6 +51,7 @@ public class DocumentLevelSecurityRandomTests extends ShieldIntegTestCase { } return builder.toString(); } + @Override protected String configRoles() { StringBuilder builder = new StringBuilder(super.configRoles()); @@ -66,6 +69,14 @@ public class DocumentLevelSecurityRandomTests extends ShieldIntegTestCase { return builder.toString(); } + @Override + public Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true) + .build(); + } + public void testDuelWithAliasFilters() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type1", "field1", "type=string", "field2", "type=string") diff --git a/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index 3ee8d9a1591..eb51f269881 100644 --- a/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -13,12 +13,14 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.children.Children; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.test.ShieldIntegTestCase; @@ -69,6 +71,14 @@ public class DocumentLevelSecurityTests extends ShieldIntegTestCase { " query: '{\"term\" : {\"field2\" : \"value2\"}}'"; // <-- query defined as json in a string } + @Override + public Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true) + .build(); + } + public void testSimpleQuery() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type1", "field1", "type=string", "field2", "type=string") diff --git a/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java b/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java index d34d988a37d..29ed42bf48a 100644 --- a/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java @@ -7,8 +7,10 @@ package org.elasticsearch.integration; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.test.ShieldIntegTestCase; @@ -47,6 +49,7 @@ public class FieldLevelSecurityRandomTests extends ShieldIntegTestCase { "role3:user3\n" + "role4:user4\n"; } + @Override protected String configRoles() { if (allowedFields == null) { @@ -98,6 +101,14 @@ public class FieldLevelSecurityRandomTests extends ShieldIntegTestCase { " - field3\n"; } + @Override + public Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true) + .build(); + } + public void testRandom() throws Exception { int j = 0; Map doc = new HashMap<>(); diff --git a/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java index 7a060b5142b..81190e3ba0a 100644 --- a/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.test.ESIntegTestCase; @@ -102,6 +103,14 @@ public class FieldLevelSecurityTests extends ShieldIntegTestCase { " fields: 'field*'\n"; } + @Override + public Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true) + .build(); + } + public void testQuery() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type1", "field1", "type=string", "field2", "type=string") diff --git a/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java b/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java index b6ff297885a..0c2db3bcaab 100644 --- a/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java @@ -7,6 +7,8 @@ package org.elasticsearch.integration; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.test.ShieldIntegTestCase; @@ -34,6 +36,7 @@ public class IndicesPermissionsWithAliasesWildcardsAndRegexsTests extends Shield return super.configUsersRoles() + "role1:user1\n"; } + @Override protected String configRoles() { return super.configRoles() + @@ -51,6 +54,14 @@ public class IndicesPermissionsWithAliasesWildcardsAndRegexsTests extends Shield " fields: field3\n"; } + @Override + public Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true) + .build(); + } + public void testResolveWildcardsRegexs() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type1", "field1", "type=string", "field2", "type=string") diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java index 3b4dfe7f941..96914edebd3 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.shield.authz.store; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.audit.logfile.CapturingLogger; import org.elasticsearch.shield.authc.support.RefreshListener; import org.elasticsearch.shield.authz.Permission; @@ -29,6 +30,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import static java.util.Collections.singleton; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -46,9 +48,10 @@ import static org.hamcrest.Matchers.startsWith; public class FileRolesStoreTests extends ESTestCase { public void testParseFile() throws Exception { Path path = getDataPath("roles.yml"); - Map roles = FileRolesStore.parseFile(path, Collections.emptySet(), logger); + Map roles = FileRolesStore.parseFile(path, Collections.emptySet(), + logger, Settings.builder().put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true).build()); assertThat(roles, notNullValue()); - assertThat(roles.size(), is(7)); + assertThat(roles.size(), is(10)); Permission.Global.Role role = roles.get("role1"); assertThat(role, notNullValue()); @@ -140,6 +143,80 @@ public class FileRolesStoreTests extends ESTestCase { assertThat(role.runAs().check("user1"), is(true)); assertThat(role.runAs().check("user2"), is(true)); assertThat(role.runAs().check("user" + randomIntBetween(3, 9)), is(false)); + + role = roles.get("role_fields"); + assertThat(role, notNullValue()); + assertThat(role.name(), equalTo("role_fields")); + assertThat(role.cluster(), notNullValue()); + assertThat(role.cluster(), is(Permission.Cluster.Core.NONE)); + assertThat(role.runAs(), is(Permission.RunAs.Core.NONE)); + assertThat(role.indices(), notNullValue()); + assertThat(role.indices().groups(), notNullValue()); + assertThat(role.indices().groups().length, is(1)); + + group = role.indices().groups()[0]; + assertThat(group.indices(), notNullValue()); + assertThat(group.indices().length, is(1)); + assertThat(group.indices()[0], equalTo("field_idx")); + assertThat(group.privilege(), notNullValue()); + assertThat(group.privilege().isAlias(Privilege.Index.READ), is(true)); + assertThat(group.getFields(), contains("foo", "boo")); + + role = roles.get("role_query"); + assertThat(role, notNullValue()); + assertThat(role.name(), equalTo("role_query")); + assertThat(role.cluster(), notNullValue()); + assertThat(role.cluster(), is(Permission.Cluster.Core.NONE)); + assertThat(role.runAs(), is(Permission.RunAs.Core.NONE)); + assertThat(role.indices(), notNullValue()); + assertThat(role.indices().groups(), notNullValue()); + assertThat(role.indices().groups().length, is(1)); + + group = role.indices().groups()[0]; + assertThat(group.indices(), notNullValue()); + assertThat(group.indices().length, is(1)); + assertThat(group.indices()[0], equalTo("query_idx")); + assertThat(group.privilege(), notNullValue()); + assertThat(group.privilege().isAlias(Privilege.Index.READ), is(true)); + assertThat(group.getFields(), nullValue()); + assertThat(group.getQuery(), notNullValue()); + + role = roles.get("role_query_fields"); + assertThat(role, notNullValue()); + assertThat(role.name(), equalTo("role_query_fields")); + assertThat(role.cluster(), notNullValue()); + assertThat(role.cluster(), is(Permission.Cluster.Core.NONE)); + assertThat(role.runAs(), is(Permission.RunAs.Core.NONE)); + assertThat(role.indices(), notNullValue()); + assertThat(role.indices().groups(), notNullValue()); + assertThat(role.indices().groups().length, is(1)); + + group = role.indices().groups()[0]; + assertThat(group.indices(), notNullValue()); + assertThat(group.indices().length, is(1)); + assertThat(group.indices()[0], equalTo("query_fields_idx")); + assertThat(group.privilege(), notNullValue()); + assertThat(group.privilege().isAlias(Privilege.Index.READ), is(true)); + assertThat(group.getFields(), contains("foo", "boo")); + assertThat(group.getQuery(), notNullValue()); + } + + public void testParseFileWithFLSAndDLSDisabled() throws Exception { + Path path = getDataPath("roles.yml"); + CapturingLogger logger = new CapturingLogger(CapturingLogger.Level.ERROR); + Map roles = FileRolesStore.parseFile(path, Collections.emptySet(), + logger, randomBoolean() ? Settings.builder().put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, false).build() : Settings.EMPTY); + assertThat(roles, notNullValue()); + assertThat(roles.size(), is(7)); + assertThat(roles.get("role_fields"), nullValue()); + assertThat(roles.get("role_query"), nullValue()); + assertThat(roles.get("role_query_fields"), nullValue()); + + List entries = logger.output(CapturingLogger.Level.ERROR); + assertThat(entries, hasSize(3)); + assertThat(entries.get(0).text, startsWith("invalid role definition [role_fields] in roles file [" + path.toAbsolutePath() + "]. document and field level security is not enabled.")); + assertThat(entries.get(1).text, startsWith("invalid role definition [role_query] in roles file [" + path.toAbsolutePath() + "]. document and field level security is not enabled.")); + assertThat(entries.get(2).text, startsWith("invalid role definition [role_query_fields] in roles file [" + path.toAbsolutePath() + "]. document and field level security is not enabled.")); } /** @@ -147,7 +224,7 @@ public class FileRolesStoreTests extends ESTestCase { */ public void testDefaultRolesFile() throws Exception { Path path = getDataPath("default_roles.yml"); - Map roles = FileRolesStore.parseFile(path, Collections.emptySet(), logger); + Map roles = FileRolesStore.parseFile(path, Collections.emptySet(), logger, Settings.EMPTY); assertThat(roles, notNullValue()); assertThat(roles.size(), is(8)); @@ -225,14 +302,14 @@ public class FileRolesStoreTests extends ESTestCase { public void testThatEmptyFileDoesNotResultInLoop() throws Exception { Path file = createTempFile(); Files.write(file, Collections.singletonList("#"), StandardCharsets.UTF_8); - Map roles = FileRolesStore.parseFile(file, Collections.emptySet(), logger); + Map roles = FileRolesStore.parseFile(file, Collections.emptySet(), logger, Settings.EMPTY); assertThat(roles.keySet(), is(empty())); } public void testThatInvalidRoleDefinitions() throws Exception { Path path = getDataPath("invalid_roles.yml"); CapturingLogger logger = new CapturingLogger(CapturingLogger.Level.ERROR); - Map roles = FileRolesStore.parseFile(path, Collections.emptySet(), logger); + Map roles = FileRolesStore.parseFile(path, Collections.emptySet(), logger, Settings.EMPTY); assertThat(roles.size(), is(1)); assertThat(roles, hasKey("valid_role")); Permission.Global.Role role = roles.get("valid_role"); @@ -268,7 +345,7 @@ public class FileRolesStoreTests extends ESTestCase { CapturingLogger logger = new CapturingLogger(CapturingLogger.Level.INFO); Path path = getDataPath("reserved_roles.yml"); - Map roles = FileRolesStore.parseFile(path, reservedRoles, logger); + Map roles = FileRolesStore.parseFile(path, reservedRoles, logger, Settings.EMPTY); assertThat(roles, notNullValue()); assertThat(roles.size(), is(2)); @@ -300,7 +377,7 @@ public class FileRolesStoreTests extends ESTestCase { Path path = createTempFile(); Files.delete(path); assertThat(Files.exists(path), is(false)); - Map roles = FileRolesStore.parseFile(path, reservedRoles, logger); + Map roles = FileRolesStore.parseFile(path, reservedRoles, logger, Settings.EMPTY); assertThat(roles, notNullValue()); assertThat(roles.size(), is(1)); diff --git a/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml b/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml index 2362eaa680e..280922e741c 100644 --- a/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml +++ b/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml @@ -29,4 +29,27 @@ role_run_as: # role with more than run_as role_run_as1: - run_as: [user1, user2] \ No newline at end of file + run_as: [user1, user2] + +role_fields: + indices: + 'field_idx': + privileges: READ + fields: + - foo + - boo + +role_query: + indices: + 'query_idx': + privileges: READ + query: '{ "match_all": {} }' + +role_query_fields: + indices: + 'query_fields_idx': + privileges: READ + query: '{ "match_all": {} }' + fields: + - foo + - boo \ No newline at end of file From 37921036cb61282ae60223ba01d70c0be8ea8f0d Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 10 Nov 2015 10:23:44 +0100 Subject: [PATCH 22/95] Marvel: ignore IndexNotFoundException for _all index when Shield is enabled When Shield and Marvel are installed together and no indices exist in the cluster yet, Shield returns an IndexNotFoundException. This commit ignores and logs at DEBUG level any IndexNotFoundException iff a) Shield is enabled and 2) marvel.agent.indices setting is empty. Closes elastic/elasticsearch#887 Original commit: elastic/x-pack-elasticsearch@5d227d775d62ddde5397b97563f11cc466419f26 --- .../indices/IndexRecoveryCollector.java | 29 ++- .../indices/IndexStatsCollector.java | 37 ++-- .../indices/IndicesStatsCollector.java | 24 +- .../collector/shards/ShardsCollector.java | 5 +- .../collector/AbstractCollectorTestCase.java | 2 +- .../indices/IndexRecoveryCollectorTests.java | 52 ++++- .../indices/IndexStatsCollectorTests.java | 73 +++++-- .../indices/IndicesStatsCollectorTests.java | 205 ++++++++++++++++++ .../marvel/test/MarvelIntegTestCase.java | 9 +- 9 files changed, 375 insertions(+), 61 deletions(-) create mode 100644 marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollectorTests.java diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollector.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollector.java index f0517f92d99..8479973f60b 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollector.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollector.java @@ -9,15 +9,19 @@ import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.marvel.agent.collector.AbstractCollector; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.license.MarvelLicensee; +import org.elasticsearch.marvel.shield.MarvelShieldIntegration; import org.elasticsearch.marvel.shield.SecuredClient; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -36,7 +40,7 @@ public class IndexRecoveryCollector extends AbstractCollector doCollect() throws Exception { List results = new ArrayList<>(1); + try { + RecoveryResponse recoveryResponse = client.admin().indices().prepareRecoveries() + .setIndices(marvelSettings.indices()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setActiveOnly(marvelSettings.recoveryActiveOnly()) + .get(marvelSettings.recoveryTimeout()); - RecoveryResponse recoveryResponse = client.admin().indices().prepareRecoveries() - .setIndices(marvelSettings.indices()) - .setIndicesOptions(IndicesOptions.lenientExpandOpen()) - .setActiveOnly(marvelSettings.recoveryActiveOnly()) - .get(marvelSettings.recoveryTimeout()); - - if (recoveryResponse.hasRecoveries()) { - results.add(new IndexRecoveryMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), recoveryResponse)); + if (recoveryResponse.hasRecoveries()) { + results.add(new IndexRecoveryMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), recoveryResponse)); + } + } catch (IndexNotFoundException e) { + if (MarvelShieldIntegration.enabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) { + logger.debug("collector [{}] - unable to collect data for missing index [{}]", name(), e.getIndex()); + } else { + throw e; + } } return Collections.unmodifiableCollection(results); } diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java index 30aa686fecf..d8d368a1c62 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java @@ -10,18 +10,18 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.marvel.agent.collector.AbstractCollector; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.license.MarvelLicensee; +import org.elasticsearch.marvel.shield.MarvelShieldIntegration; import org.elasticsearch.marvel.shield.SecuredClient; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; +import java.util.*; /** * Collector for indices statistics. @@ -37,7 +37,7 @@ public class IndexStatsCollector extends AbstractCollector private final Client client; @Inject - public IndexStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee, + public IndexStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee, SecuredClient client) { super(settings, NAME, clusterService, marvelSettings, marvelLicensee); this.client = client; @@ -51,17 +51,24 @@ public class IndexStatsCollector extends AbstractCollector @Override protected Collection doCollect() throws Exception { List results = new ArrayList<>(1); + try { + IndicesStatsResponse indicesStats = client.admin().indices().prepareStats() + .setRefresh(true) + .setIndices(marvelSettings.indices()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .get(marvelSettings.indexStatsTimeout()); - IndicesStatsResponse indicesStats = client.admin().indices().prepareStats() - .setRefresh(true) - .setIndices(marvelSettings.indices()) - .setIndicesOptions(IndicesOptions.lenientExpandOpen()) - .get(marvelSettings.indexStatsTimeout()); - - long timestamp = System.currentTimeMillis(); - String clusterUUID = clusterUUID(); - for (IndexStats indexStats : indicesStats.getIndices().values()) { - results.add(new IndexStatsMarvelDoc(clusterUUID, TYPE, timestamp, indexStats)); + long timestamp = System.currentTimeMillis(); + String clusterUUID = clusterUUID(); + for (IndexStats indexStats : indicesStats.getIndices().values()) { + results.add(new IndexStatsMarvelDoc(clusterUUID, TYPE, timestamp, indexStats)); + } + } catch (IndexNotFoundException e) { + if (MarvelShieldIntegration.enabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) { + logger.debug("collector [{}] - unable to collect data for missing index [{}]", name(), e.getIndex()); + } else { + throw e; + } } return Collections.unmodifiableCollection(results); } diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java index cd8227b4033..babfc91b311 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java @@ -6,16 +6,21 @@ package org.elasticsearch.marvel.agent.collector.indices; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.marvel.agent.collector.AbstractCollector; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.license.MarvelLicensee; +import org.elasticsearch.marvel.shield.MarvelShieldIntegration; import org.elasticsearch.marvel.shield.SecuredClient; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -45,11 +50,20 @@ public class IndicesStatsCollector extends AbstractCollector doCollect() throws Exception { - IndicesStatsResponse indicesStats = client.admin().indices().prepareStats() - .setRefresh(true) - .get(marvelSettings.indicesStatsTimeout()); + try { + IndicesStatsResponse indicesStats = client.admin().indices().prepareStats() + .setIndices(marvelSettings.indices()) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setRefresh(true) + .get(marvelSettings.indicesStatsTimeout()); - MarvelDoc result = new IndicesStatsMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), indicesStats); - return Collections.singletonList(result); + return Collections.singletonList(new IndicesStatsMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), indicesStats)); + } catch (IndexNotFoundException e) { + if (MarvelShieldIntegration.enabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) { + logger.debug("collector [{}] - unable to collect data for missing index [{}]", name(), e.getIndex()); + return Collections.emptyList(); + } + throw e; + } } } diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/shards/ShardsCollector.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/shards/ShardsCollector.java index a0c599c50d0..d61d4cb838c 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/shards/ShardsCollector.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/shards/ShardsCollector.java @@ -7,18 +7,19 @@ package org.elasticsearch.marvel.agent.collector.shards; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.marvel.agent.collector.AbstractCollector; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.license.MarvelLicensee; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -72,7 +73,7 @@ public class ShardsCollector extends AbstractCollector { private boolean match(String indexName) { String[] indices = marvelSettings.indices(); - return CollectionUtils.isEmpty(indices) || Regex.simpleMatch(indices, indexName); + return IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices())) || Regex.simpleMatch(indices, indexName); } /** diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java index 775582e9ac4..d2557f58e35 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java @@ -93,7 +93,7 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase { .issuer("test") .maxNodes(Integer.MAX_VALUE) .signature("_signature") - .type("basic") + .type("trial") .uid(String.valueOf(RandomizedTest.systemPropertyAsInt(SysGlobals.CHILDVM_SYSPROP_JVM_ID, 0)) + System.identityHashCode(AbstractCollectorTestCase.class)) .build(); } diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollectorTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollectorTests.java index 3595cb098f2..71cd0508b74 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollectorTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollectorTests.java @@ -7,8 +7,10 @@ package org.elasticsearch.marvel.agent.collector.indices; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; @@ -25,16 +27,11 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; @ClusterScope(numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class IndexRecoveryCollectorTests extends AbstractCollectorTestCase { + private final boolean activeOnly = false; private final String indexName = "test"; @@ -118,10 +115,12 @@ public class IndexRecoveryCollectorTests extends AbstractCollectorTestCase { } } - public void testIndexRecoveryCollectorWithLicensing() { + public void testIndexRecoveryCollectorWithLicensing() throws Exception { + List nodesIds = internalCluster().startNodesAsync(randomIntBetween(2, 5)).get(); + waitForNoBlocksOnNodes(); + try { - String[] nodes = internalCluster().getNodeNames(); - for (String node : nodes) { + for (String node : nodesIds) { logger.debug("--> creating a new instance of the collector"); IndexRecoveryCollector collector = newIndexRecoveryCollector(node); assertNotNull(collector); @@ -156,6 +155,39 @@ public class IndexRecoveryCollectorTests extends AbstractCollectorTestCase { } } + public void testEmptyCluster() throws Exception { + final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, Strings.EMPTY_ARRAY)); + waitForNoBlocksOnNode(node); + + try { + assertThat(newIndexRecoveryCollector(node).doCollect(), hasSize(0)); + } catch (IndexNotFoundException e) { + fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector"); + } + } + + public void testEmptyClusterAllIndices() throws Exception { + final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, MetaData.ALL)); + waitForNoBlocksOnNode(node); + + try { + assertThat(newIndexRecoveryCollector(node).doCollect(), hasSize(0)); + } catch (IndexNotFoundException e) { + fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector"); + } + } + + public void testEmptyClusterMissingIndex() throws Exception { + final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, "unknown")); + waitForNoBlocksOnNode(node); + + try { + assertThat(newIndexRecoveryCollector(node).doCollect(), hasSize(0)); + } catch (IndexNotFoundException e) { + fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector"); + } + } + private IndexRecoveryCollector newIndexRecoveryCollector(String nodeId) { if (!Strings.hasText(nodeId)) { nodeId = randomFrom(internalCluster().getNodeNames()); diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java index 100bbd37481..82b16e41d26 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java @@ -7,39 +7,71 @@ package org.elasticsearch.marvel.agent.collector.indices; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.license.MarvelLicensee; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Before; import java.util.Collection; import java.util.Iterator; +import java.util.List; +import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; -@ClusterScope(numClientNodes = 0) +@ClusterScope(numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class IndexStatsCollectorTests extends AbstractCollectorTestCase { + @Override protected int numberOfReplicas() { return 0; } - @Before - public void beforeIndexStatsCollectorTests() throws Exception { - waitForNoBlocksOnNodes(); + public void testEmptyCluster() throws Exception { + final String node = internalCluster().startNode(); + waitForNoBlocksOnNode(node); + + try { + assertThat(newIndexStatsCollector(node).doCollect(), hasSize(0)); + } catch (IndexNotFoundException e) { + fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector"); + } + } + + public void testEmptyClusterAllIndices() throws Exception { + final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, MetaData.ALL)); + waitForNoBlocksOnNode(node); + + try { + assertThat(newIndexStatsCollector(node).doCollect(), hasSize(0)); + } catch (IndexNotFoundException e) { + fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector"); + } + } + + public void testEmptyClusterMissingIndex() throws Exception { + final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, "unknown")); + waitForNoBlocksOnNode(node); + + try { + assertThat(newIndexStatsCollector(node).doCollect(), hasSize(0)); + } catch (IndexNotFoundException e) { + fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector"); + } } public void testIndexStatsCollectorOneIndex() throws Exception { + final String node = internalCluster().startNode(); + waitForNoBlocksOnNode(node); + final String indexName = "one-index"; + createIndex(indexName); + securedEnsureGreen(indexName); final int nbDocs = randomIntBetween(1, 20); for (int i = 0; i < nbDocs; i++) { @@ -48,7 +80,6 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase { securedFlush(); securedRefresh(); - securedEnsureGreen(indexName); assertHitCount(client().prepareSearch().setSize(0).get(), nbDocs); @@ -77,20 +108,26 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase { } public void testIndexStatsCollectorMultipleIndices() throws Exception { + final String node = internalCluster().startNode(); + waitForNoBlocksOnNode(node); + final String indexPrefix = "multi-indices-"; final int nbIndices = randomIntBetween(1, 5); int[] docsPerIndex = new int[nbIndices]; for (int i = 0; i < nbIndices; i++) { + String index = indexPrefix + i; + createIndex(index); + securedEnsureGreen(index); + docsPerIndex[i] = randomIntBetween(1, 20); for (int j = 0; j < docsPerIndex[i]; j++) { - client().prepareIndex(indexPrefix + i, "test").setSource("num", i).get(); + client().prepareIndex(index, "test").setSource("num", i).get(); } } securedFlush(); securedRefresh(); - securedEnsureGreen(indexPrefix + "*"); for (int i = 0; i < nbIndices; i++) { assertHitCount(client().prepareSearch(indexPrefix + i).setSize(0).get(), docsPerIndex[i]); @@ -134,7 +171,10 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase { } } - public void testIndexStatsCollectorWithLicensing() { + public void testIndexStatsCollectorWithLicensing() throws Exception { + List nodesIds = internalCluster().startNodesAsync(randomIntBetween(2, 5)).get(); + waitForNoBlocksOnNodes(); + try { final int nbDocs = randomIntBetween(1, 20); for (int i = 0; i < nbDocs; i++) { @@ -145,8 +185,7 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase { securedRefresh(); securedEnsureGreen("test"); - String[] nodes = internalCluster().getNodeNames(); - for (String node : nodes) { + for (String node : nodesIds) { logger.debug("--> creating a new instance of the collector"); IndexStatsCollector collector = newIndexStatsCollector(node); assertNotNull(collector); diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollectorTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollectorTests.java new file mode 100644 index 00000000000..fe4e8efc8ad --- /dev/null +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollectorTests.java @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.collector.indices; + +import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase; +import org.elasticsearch.marvel.agent.exporter.MarvelDoc; +import org.elasticsearch.marvel.agent.settings.MarvelSettings; +import org.elasticsearch.marvel.license.MarvelLicensee; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; + +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.*; + + +@ClusterScope(numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) +public class IndicesStatsCollectorTests extends AbstractCollectorTestCase { + + @Override + protected int numberOfReplicas() { + return 0; + } + + public void testEmptyCluster() throws Exception { + final String node = internalCluster().startNode(); + waitForNoBlocksOnNode(node); + + try { + assertThat(newIndicesStatsCollector(node).doCollect(), hasSize(shieldEnabled ? 0 : 1)); + } catch (IndexNotFoundException e) { + fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector"); + } + } + + public void testEmptyClusterAllIndices() throws Exception { + final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, MetaData.ALL)); + waitForNoBlocksOnNode(node); + + try { + assertThat(newIndicesStatsCollector(node).doCollect(), hasSize(shieldEnabled ? 0 : 1)); + } catch (IndexNotFoundException e) { + fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector"); + } + } + + public void testEmptyClusterMissingIndex() throws Exception { + final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, "unknown")); + waitForNoBlocksOnNode(node); + + try { + assertThat(newIndicesStatsCollector(node).doCollect(), hasSize(1)); + } catch (IndexNotFoundException e) { + fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector"); + } + } + + public void testIndicesStatsCollectorOneIndex() throws Exception { + final String node = internalCluster().startNode(); + waitForNoBlocksOnNode(node); + + final String indexName = "one-index"; + createIndex(indexName); + securedEnsureGreen(indexName); + + final int nbDocs = randomIntBetween(1, 20); + for (int i = 0; i < nbDocs; i++) { + client().prepareIndex(indexName, "test").setSource("num", i).get(); + } + + securedFlush(); + securedRefresh(); + + assertHitCount(client().prepareSearch().setSize(0).get(), nbDocs); + + Collection results = newIndicesStatsCollector().doCollect(); + assertThat(results, hasSize(1)); + + MarvelDoc marvelDoc = results.iterator().next(); + assertThat(marvelDoc, instanceOf(IndicesStatsMarvelDoc.class)); + + IndicesStatsMarvelDoc indicesStatsMarvelDoc = (IndicesStatsMarvelDoc) marvelDoc; + IndicesStatsResponse indicesStats = indicesStatsMarvelDoc.getIndicesStats(); + assertNotNull(indicesStats); + assertThat(indicesStats.getIndices().keySet(), hasSize(1)); + + IndexStats indexStats = indicesStats.getIndex(indexName); + assertThat(indexStats.getShards(), arrayWithSize(getNumShards(indexName).totalNumShards)); + } + + public void testIndicesStatsCollectorMultipleIndices() throws Exception { + final String node = internalCluster().startNode(); + waitForNoBlocksOnNode(node); + + final String indexPrefix = "multi-indices-"; + final int nbIndices = randomIntBetween(1, 5); + int[] docsPerIndex = new int[nbIndices]; + + for (int i = 0; i < nbIndices; i++) { + String index = indexPrefix + i; + createIndex(index); + securedEnsureGreen(index); + + docsPerIndex[i] = randomIntBetween(1, 20); + for (int j = 0; j < docsPerIndex[i]; j++) { + client().prepareIndex(index, "test").setSource("num", i).get(); + } + } + + securedFlush(); + securedRefresh(); + + for (int i = 0; i < nbIndices; i++) { + assertHitCount(client().prepareSearch(indexPrefix + i).setSize(0).get(), docsPerIndex[i]); + } + + Collection results = newIndicesStatsCollector().doCollect(); + assertThat(results, hasSize(1)); + + MarvelDoc marvelDoc = results.iterator().next(); + assertThat(marvelDoc, instanceOf(IndicesStatsMarvelDoc.class)); + + IndicesStatsMarvelDoc indicesStatsMarvelDoc = (IndicesStatsMarvelDoc) marvelDoc; + IndicesStatsResponse indicesStats = indicesStatsMarvelDoc.getIndicesStats(); + assertNotNull(indicesStats); + assertThat(indicesStats.getIndices().keySet(), hasSize(nbIndices)); + } + + public void testIndicesStatsCollectorWithLicensing() throws Exception { + List nodesIds = internalCluster().startNodesAsync(randomIntBetween(2, 5)).get(); + waitForNoBlocksOnNodes(); + + try { + final int nbDocs = randomIntBetween(1, 20); + for (int i = 0; i < nbDocs; i++) { + client().prepareIndex("test", "test").setSource("num", i).get(); + } + + securedFlush(); + securedRefresh(); + securedEnsureGreen("test"); + + for (String node : nodesIds) { + logger.debug("--> creating a new instance of the collector"); + IndicesStatsCollector collector = newIndicesStatsCollector(node); + assertNotNull(collector); + + logger.debug("--> enabling license and checks that the collector can collect data if node is master"); + enableLicense(); + if (node.equals(internalCluster().getMasterName())) { + assertCanCollect(collector); + } else { + assertCannotCollect(collector); + } + + logger.debug("--> starting graceful period and checks that the collector can still collect data if node is master"); + beginGracefulPeriod(); + if (node.equals(internalCluster().getMasterName())) { + assertCanCollect(collector); + } else { + assertCannotCollect(collector); + } + + logger.debug("--> ending graceful period and checks that the collector cannot collect data"); + endGracefulPeriod(); + assertCannotCollect(collector); + + logger.debug("--> disabling license and checks that the collector cannot collect data"); + disableLicense(); + assertCannotCollect(collector); + } + } finally { + // Ensure license is enabled before finishing the test + enableLicense(); + } + } + + private IndicesStatsCollector newIndicesStatsCollector() { + // This collector runs on master node only + return newIndicesStatsCollector(internalCluster().getMasterName()); + } + + private IndicesStatsCollector newIndicesStatsCollector(String nodeId) { + if (!Strings.hasText(nodeId)) { + nodeId = randomFrom(internalCluster().getNodeNames()); + } + return new IndicesStatsCollector(internalCluster().getInstance(Settings.class, nodeId), + internalCluster().getInstance(ClusterService.class, nodeId), + internalCluster().getInstance(MarvelSettings.class, nodeId), + internalCluster().getInstance(MarvelLicensee.class, nodeId), + securedClient(nodeId)); + } +} diff --git a/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java b/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java index 1a3b7b77394..2f25f15b2a9 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java @@ -51,10 +51,13 @@ import static org.hamcrest.Matchers.*; */ public abstract class MarvelIntegTestCase extends ESIntegTestCase { - protected Boolean shieldEnabled = enableShield(); + protected static Boolean shieldEnabled; @Override protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { + if (shieldEnabled == null) { + shieldEnabled = enableShield(); + } logger.info("--> shield {}", shieldEnabled ? "enabled" : "disabled"); return super.buildTestCluster(scope, seed); } @@ -105,7 +108,9 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { * Override and returns {@code false} to force running without shield */ protected boolean enableShield() { - return randomBoolean(); + boolean r = randomBoolean(); + logger.info("--> shield is{}", r); + return r; } protected void stopCollection() { From 36dd391d4d06cc9da3886b518e3833ff8d0ad3e1 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 18 Nov 2015 19:04:41 +0100 Subject: [PATCH 23/95] Marvel: mute NodeStats tests Original commit: elastic/x-pack-elasticsearch@f32464f864eb0adc0c585a95b7f8de8d82562ab3 --- .../marvel/agent/renderer/node/MultiNodesStatsTests.java | 2 ++ .../marvel/agent/renderer/node/NodeStatsRendererTests.java | 2 ++ .../marvel/agent/renderer/node/NodeStatsTests.java | 2 ++ 3 files changed, 6 insertions(+) diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java index 01e5f487057..27602daeee9 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.marvel.agent.renderer.node; +import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; @@ -20,6 +21,7 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; import static org.hamcrest.Matchers.*; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/pull/994") @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class MultiNodesStatsTests extends MarvelIntegTestCase { diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRendererTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRendererTests.java index 582fc70042c..e2b13ee1e1d 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRendererTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRendererTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.marvel.agent.renderer.node; +import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.marvel.agent.collector.node.NodeStatsMarvelDoc; import org.elasticsearch.marvel.agent.renderer.Renderer; @@ -13,6 +14,7 @@ import org.elasticsearch.node.service.NodeService; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.StreamsUtils; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/pull/994") public class NodeStatsRendererTests extends ESSingleNodeTestCase { private static final String SAMPLE_FILE = "/samples/node_stats.json"; diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java index d63b4d62d24..de16323b092 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.marvel.agent.renderer.node; +import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector; @@ -21,6 +22,7 @@ import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.greaterThan; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/pull/994") // numClientNodes is set to 0 because Client nodes don't have Filesystem stats @ClusterScope(scope = Scope.TEST, numClientNodes = 0, transportClientRatio = 0.0) public class NodeStatsTests extends MarvelIntegTestCase { From 5b72d1768dd4b0f960d87f8ae6c432bcb7f9d681 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 18 Nov 2015 13:48:54 -0500 Subject: [PATCH 24/95] test: add throws InterruptedException for InternalTestCluster#beforeTest calls Original commit: elastic/x-pack-elasticsearch@b89a58a40866c99b037f6f04af94414737fcfb70 --- .../shield/audit/index/IndexAuditTrailTests.java | 4 ++-- .../audit/index/RemoteIndexAuditTrailStartingTests.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java b/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java index 7d7e2060d85..e60bbbe1664 100644 --- a/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java @@ -119,11 +119,11 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase { return remoteIndexing ? remoteClient : client(); } - private void initialize(String... excludes) throws IOException { + private void initialize(String... excludes) throws IOException, InterruptedException { initialize(null, excludes); } - private void initialize(String[] includes, String[] excludes) throws IOException { + private void initialize(String[] includes, String[] excludes) throws IOException, InterruptedException { rollover = randomFrom(HOURLY, DAILY, WEEKLY, MONTHLY); numReplicas = numberOfReplicas(); numShards = numberOfShards(); diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/index/RemoteIndexAuditTrailStartingTests.java b/shield/src/test/java/org/elasticsearch/shield/audit/index/RemoteIndexAuditTrailStartingTests.java index 5ed09a1259d..decd5f6630b 100644 --- a/shield/src/test/java/org/elasticsearch/shield/audit/index/RemoteIndexAuditTrailStartingTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/audit/index/RemoteIndexAuditTrailStartingTests.java @@ -72,7 +72,7 @@ public class RemoteIndexAuditTrailStartingTests extends ShieldIntegTestCase { } @Before - public void startRemoteCluster() throws IOException { + public void startRemoteCluster() throws IOException, InterruptedException { final List addresses = new ArrayList<>(); // get addresses for current cluster NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet(); From fb4ba6f89d6a0fb455680f3af012253d85a1b7c1 Mon Sep 17 00:00:00 2001 From: debadair Date: Fri, 6 Nov 2015 15:04:41 -0800 Subject: [PATCH 25/95] Watcher Docs: Edited chain input topic & added chain to customizing watches. Original commit: elastic/x-pack-elasticsearch@9e1d7f09c6249e091c71145075d5dd449e6251af --- watcher/docs/customizing-watches.asciidoc | 11 ++++++++-- watcher/docs/reference/input/chain.asciidoc | 24 ++++++++++----------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/watcher/docs/customizing-watches.asciidoc b/watcher/docs/customizing-watches.asciidoc index d343e444e23..4bd7f408049 100644 --- a/watcher/docs/customizing-watches.asciidoc +++ b/watcher/docs/customizing-watches.asciidoc @@ -9,8 +9,8 @@ you can customize a watch by modifying its <>, [[changing-inputs]] === Changing Inputs -Watcher supports three types of inputs <>, -<>, and <>. +Watcher supports four types of inputs <>, +<>, <>, and <>. [[loading-static-data]] ==== Loading Static Data with the Simple Input @@ -115,6 +115,13 @@ during the month of May, 2015 that were tagged with `elasticsearch`. included as part of the path. //////////// +[[chaining-inputs]] +==== Chaining Inputs + +You can create an <> to load data from multiple sources into a watch. +The inputs in a chain are processed in order, so the the data loaded by one input can be used +by subsequent inputs. + [[changing-conditions]] === Changing Conditions diff --git a/watcher/docs/reference/input/chain.asciidoc b/watcher/docs/reference/input/chain.asciidoc index f68035a4252..8340154d8c5 100644 --- a/watcher/docs/reference/input/chain.asciidoc +++ b/watcher/docs/reference/input/chain.asciidoc @@ -1,18 +1,22 @@ [[input-chain]] ==== Chain Input -An <> that enables you to chain several inputs one after the other and have each input populating the execution context. -The `chain` input is useful when the output of one input should change the input of another or you need several inputs to gather all the -information needed for an action.. +An <> that enables you to string together multiple inputs to load data into the watch execution +context. Each input loads data into the execution context and that data is available to subsequent inputs in the chain. -You can define the chained input as following +The `chain` input is useful when you want to perform an action based on data from multiple sources. +You can also use the data collected by one input to load data from another source. + +To reference data loaded by a particular input, you use the input's name, `ctx.payload..`. + +For example, the following chain input loads data from an HTTP server using the path set by a `simple` input. [source,json] -------------------------------------------------- { "input" : { "chain" : { - "inputs" : [ + "inputs" : [ <1> "first" : { "simple" : { "path" : "/_search" } }, @@ -21,7 +25,7 @@ You can define the chained input as following "request" : { "host" : "localhost", "port" : 9200, - "path" : "{{ctx.payload.first.path}}" + "path" : "{{ctx.payload.first.path}}" <2> } } } @@ -31,9 +35,5 @@ You can define the chained input as following ... } -------------------------------------------------- - -As you can see, the name of the input (`first` and `second` in this example) can be used to access values from the context in consecutive inputs. - -In case you are wondering about the structure of this input. The `inputs` must be an array, because JSON does not guarantee the order of arbitrary -objects, one has to use a list. - +<1> The inputs in a chain are specified as an array to guarantee the order in which the inputs are processed. (JSON does not guarantee the order of arbitrary objects.) +<2> Loads the `path` set by the `first` input. \ No newline at end of file From 5809bbdb5de5cd9a900b52b608ac9bbfdb670e5d Mon Sep 17 00:00:00 2001 From: debadair Date: Wed, 18 Nov 2015 17:46:21 -0800 Subject: [PATCH 26/95] Docs: Deleting unused Marvel doc files. Marvel docs are currently in elasticsearch-marvel/docs. Original commit: elastic/x-pack-elasticsearch@be7a7e2243ef865ef72134a910eb4b5f192c8ce3 --- marvel/docs/configuring-marvel.asciidoc | 172 ---------------- marvel/docs/getting-started.asciidoc | 47 ----- marvel/docs/index.asciidoc | 17 -- marvel/docs/installing-marvel.asciidoc | 141 ------------- marvel/docs/introduction.asciidoc | 23 --- marvel/docs/release-notes.asciidoc | 252 ------------------------ 6 files changed, 652 deletions(-) delete mode 100644 marvel/docs/configuring-marvel.asciidoc delete mode 100644 marvel/docs/getting-started.asciidoc delete mode 100644 marvel/docs/index.asciidoc delete mode 100644 marvel/docs/installing-marvel.asciidoc delete mode 100644 marvel/docs/introduction.asciidoc delete mode 100644 marvel/docs/release-notes.asciidoc diff --git a/marvel/docs/configuring-marvel.asciidoc b/marvel/docs/configuring-marvel.asciidoc deleted file mode 100644 index dc4882c0332..00000000000 --- a/marvel/docs/configuring-marvel.asciidoc +++ /dev/null @@ -1,172 +0,0 @@ -[[configuration]] -== Configuring Marvel - -[float] -[[stats-export]] -=== Controlling Marvel Data Collection - -You can set `marvel.agent` options in a node's `elasticsearch.yml` file to control how Marvel data -is collected from the node. - -`marvel.agent.enabled`:: - -Controls whether or not data is collected from the node. Enabled by default. Set -`marvel.agent.enabled` to `false` to disable data collection. Use to disable data collection -on the monitoring nodes when you use a separate monitoring cluster. - -`marvel.agent.exporter.es.hosts`:: - -Specifies where the collected Marvel data should be stored. Defaults to the address of the -local node (typically `localhost:9200`). To send data to a monitoring cluster, -set to a comma-seprarated list of nodes in `hostname:port` format . Marvel attempts to send data -to the hosts in the order they are listed--if the first node is unreachable, the second node is -tried, and so on. added[1.3.0, before the default was `localhost:9200`] -+ -You can update this setting through the Cluster Update Settings API. -+ -added[1.0.2] - HTTP Basic authentication credentials can be specified as part of the host name, - i.e., ["user:pwd@host:9200"] -+ -added[1.3.0] - next to the host and port, you can specify a protocol to use, - i.e., ["https://host:9200"] (defaults to "http") -+ -added[1.3.0] - the `hostname:port` part can be extended with a base path - i.e., ["host:9200/monitor1"] - -`marvel.agent.indices`:: - -Controls which indices Marvel collects data for. Defaults to all indices. Specify the index names -as a comma-separated list, for example `test1,test2,test3`. Names can include wildcards, for -example `test*`. You can explicitly include or exclude indices by prepending -`+` to include the index, or `-` to exclude the index. For example, to include all indices that -start with `test` except `test3`, you could specify `+test*,-test3`. -+ -You can update this setting through the Cluster Update Settings API. - -`marvel.agent.interval`:: - -Controls how often data samples are collected from the node. Defaults to `10s`. Set to -`-1` to temporarily disable data collection. -+ -You can update this setting through the Cluster Update Settings API. - -`marvel.agent.exporter.es.index.timeformat`:: - -Controls the time component in the index name to which data is exported. -Defaults to `"YYYY.MM.dd"`, which produces index names such as -`.marvel-2015.08.23`. Supports date formats as explained -http://joda-time.sourceforge.net/api-release/org/joda/time/format/DateTimeFormat.html[here]. - -`marvel.agent.exporter.es.timeout`:: - -Sets the connection timeout for sending data. Defaults to `6s`. -+ -You can update this setting through the Cluster Update Settings API. - -`marvel.agent.stats.timeout`:: - -added[1.3.1] - -Sets the timeout when collecting statistics from the master node. Defaults to `10m`. - -`marvel.agent.exporter.es.ssl.truststore.path`:: - -added[1.3.0] - -The location of the truststore to use for HTTPS connections. Specified as an absolute path. - -`marvel.agent.exporter.es.ssl.truststore.password`:: - -The password to use to access the truststore. - -`marvel.agent.exporter.es.ssl.truststore.algorithm`:: - -The truststore format. Defaults to SunX509. - -`marvel.agent.exporter.es.ssl.hostname_verification`:: -+ -added[1.3.1] - -Controls whether or not the hostname is verified when using HTTPS. Set to `false` to disable -hostname verification when sending data to a remote host. -+ -You can update this setting through the Cluster Update Settings API. - -[float] -[[marvel-indices]] -=== Configuring Marvel Indices - -Marvel stores it's data using time-based indices. By default, Marvel generates -an index per day, with one shard and one replica. We expect this to be a good -default for most use cases. For very large clusters, you might need to make changes -by overriding the settings in the default Marvel index template. - -[[config-marvel-indices]] - -Marvel uses an {ref}/indices-templates.html[index template] to preconfigure newly created indices. -You can retrieve it with: - -[source,sh] ----------------------------------- -GET /_template/marvel ----------------------------------- - -You can override the default settings by adding your own template. Make sure your template uses -the `.marvel-*` matching pattern and has an order of 1 or higher. For example, the following -template increases the number of shards to 5: - -[source,json] ----------------------------------- -PUT /_template/custom_marvel -{ - "template": ".marvel*", - "order": 1, - "settings": { - "number_of_shards": 5 - } -} ----------------------------------- - -IMPORTANT: We recommend only changing the `settings` section. Other sections are -important for the correct operation of the dashboards. - -For reference, the following snippet shows the `settings` section of the default template. - -[source,json] ----------------------------------- -{ - "template": ".marvel*", - "settings": { - "number_of_shards": 1, - "number_of_replicas": 1, - "analysis": { - "analyzer": { - "default": { - "type": "standard", - "stopwords": "_none_" - } - } - }, - "mapper.dynamic": true, - "marvel.index_format": 1 - } - ..... -} ----------------------------------- - -[float] -[[relevant-settings]] -=== Enabling Automatic Index Creation - -Marvel relies on Elasticsearch's ability to automatically create new indices -when indexing documents. If you have disabled automatic index creation, you -need to configure the `action.auto_create_index` setting to allow the -creation of Marvel indices: - -[source,yaml] ----------------------- -action.auto_create_index: .marvel-* ----------------------- - -For more information see {ref}/docs-index_.html#index-creation[Index Creation] in the Elasticsearch -Reference. diff --git a/marvel/docs/getting-started.asciidoc b/marvel/docs/getting-started.asciidoc deleted file mode 100644 index a4ee053c4a0..00000000000 --- a/marvel/docs/getting-started.asciidoc +++ /dev/null @@ -1,47 +0,0 @@ -[[getting-started]] -== Getting Started - -This getting started guide walks you through installing Marvel -and using the Marvel Kibana app to monitor your Elastisearch cluster. - -To install Marvel: - -. Install the Marvel plugin on each node in your cluster: - -.. Run `bin/plugin install` from `ES_HOME` to install the License plugin: -+ -[source,shell] ----------------------------------------------------------- -bin/plugin install license ----------------------------------------------------------- - -.. Run `bin/plugin install` to install the Marvel plugin. -+ -[source,shell] ----------------------------------------------------------- -bin/plugin install marvel-agent ----------------------------------------------------------- -+ -NOTE: If you are using a <> of Elasticsearch, run the installation with superuser permissions. To perform an offline installation, <>. - -.. Start Elasticsearch. -+ -[source,shell] ----------------------------------------------------------- -bin/elasticsearch ----------------------------------------------------------- - -. Install the Marvel app into Kibana. - -. Run Kibana and open the Marvel app to verify the installation. You should -see an overview of your cluster's status: -+ -// image:images/overview_thumb.png["Overview Dashboard",link="images/overview.png"] - - -Now you're ready to use Marvel to monitor and analyze your cluster! For example, you can: - -* -* - - diff --git a/marvel/docs/index.asciidoc b/marvel/docs/index.asciidoc deleted file mode 100644 index c07f713d15a..00000000000 --- a/marvel/docs/index.asciidoc +++ /dev/null @@ -1,17 +0,0 @@ -= Marvel Documentation - -:ref: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current -:esversion: 2.0 -:licenseversion: 2.0 -:shieldversion: 2.0 -:kibanaversion: 4.2+ - -include::introduction.asciidoc[] - -include::getting-started.asciidoc[] - -include::installing-marvel.asciidoc[] - -include::configuring-marvel.asciidoc[] - -include::release-notes.asciidoc[] \ No newline at end of file diff --git a/marvel/docs/installing-marvel.asciidoc b/marvel/docs/installing-marvel.asciidoc deleted file mode 100644 index b9d597f9ca2..00000000000 --- a/marvel/docs/installing-marvel.asciidoc +++ /dev/null @@ -1,141 +0,0 @@ -[[installing-marvel]] -== Installing Marvel - -To use Marvel, you need to install two components: - -* An Elasticsearch plugin that collects data from each node in your cluster. -This plugin must be installed on every node. -* A Kibana app that provides the Marvel monitoring and management UI. - -By default, the Marvel plugin stores data in the same Elasticsearch cluster -where it is installed. If you are monitoring a production cluster we recommend -that you store the Marvel data in a separate monitoring cluster. Sending the Marvel -data to a monitoring cluster helps ensure that you can continue to monitor your -production cluster if it's in an unhealthy state. - -For basic installation instructions, see <>. For -information about storing Marvel data in a separate monitoring cluster, see <>. - -[float] -[[marvel-prequisites]] -=== Marvel Installation Prequisites - -* Elasticsearch {esversion} or later. -* Kibana {kibanaversion} or later. -* A modern web browser - http://www.elastic.co/subscriptions/matrix#matrix_browsers[Supported -Browsers]. - -[float] -[[package-installation]] -=== Installing Marvel on a DEB/RPM Package Installation - -If you use the DEB/RPM packages to install Elasticsearch, by default Elasticsearch is installed in -`/usr/share/elasticsearch` and the configuration files are stored in `/etc/elasticsearch`. (For the -complete list of default paths, see {ref}/setup-dir-layout.html#_deb_and_rpm[Directory Layout] in -the Elasticsearch Reference.) - -To install the Marvel and License plugins on a DEB/RPM package installation, you need to run -`bin/plugin install` from the `/usr/share/elasticsearch` directory with superuser permissions, and -specify the location of the configuration files by setting `-Des.path.conf`. For example: - -[source,shell] ----------------------------------------------------------- -cd /usr/share/elasticsearch -sudo bin/plugin install license -sudo bin/plugin install marvel-agent ----------------------------------------------------------- - -[float] -[[offline-installation]] -=== Installing Marvel on Offline Machines - -Elasticsearch’s `bin/plugin` script requires direct Internet access to download and install the -License and Marvel plugins. If your server doesn’t have Internet access, you can manually -download and install the plugins. - -To install Marvel on a machine that doesn't have Internet access: - -. Manually download the License and Marvel binaries: -+ -** https://download.elastic.co/elasticsearch/license/license-latest.zip[ -https://download.elastic.co/elasticsearch/license/license-latest.zip] -** https://download.elastic.co/elasticsearch/marvel/marvel-agent-latest.zip[ -https://download.elastic.co/elasticsearch/marvel/marvel-agent-latest.zip] - -. Transfer the zip files to the offline machine. - -. Run `bin/plugin` with the `-u` option to install the plugins using the zip files. For example: -+ -[source,shell] ----------------------------------------------------------- -bin/plugin install file:///path/to/file/license-latest.zip <1> -bin/plugin install file:///path/to/file/marvel-agent-latest.zip ----------------------------------------------------------- -<1> Note that you must specify an absolute path to the zip file after the `file://` protocol. - -[float] -[[monitoring-cluster]] -=== Setting up a Separate Monitoring Cluster - -To store Marvel data in a separate monitoring cluster: - -. Set up the Elasticsearch cluster you want to use for monitoring. For example, a two host cluster -with the nodes `es-mon-1` and `es-mon-2`. - -. Disable Marvel data collection for the nodes in your monitoring cluster by configuring the -`marvel.agent.enabled` setting in their `elasticsearch.yml` configuration files. -+ -[source,yaml] ------------------------- -marvel.agent.enabled: false ------------------------- - -. Install the Marvel and License plugins on the nodes in your monitoring cluster: -+ -[source,sh] ----------------- -bin/plugin install license -bin/plugin install marvel-agent ----------------- - -. Start Elasticsearch on the nodes in your monitoring cluster: -+ -[source,shell] ----------------------------------------------------------- -bin/elasticsearch ----------------------------------------------------------- - -. Configure the nodes in your production cluster to send Marvel data to your monitoring cluster by -configuring the `marvel.agent.exporter.es.hosts` setting in their `elasticsearch.yml` -configuration files: -+ -[source,yaml] ------------------------- -marvel.agent.exporter.es.hosts: ["es-mon-1:9200","es-mon-2:9200"] ------------------------- - -. Install the Marvel and License plugins on the nodes in your production cluster: -+ -[source,sh] ----------------- -bin/plugin install license -bin/plugin install marvel-agent ----------------- - - -. Restart Elasticsearch on the nodes in your production cluster -+ -[source,shell] ----------------------------------------------------------- -bin/elasticsearch ---------------------------------------------------------- -+ -TIP: You may want to temporarily {ref}/modules-cluster.html[disable shard -allocation] before you restart your nodes to avoid unnecessary shard -reallocation during the install process. - -. Install the Marvel app into Kibana. - -. Run Kibana and open the Marvel app to verify the installation. You should -see an overview of your cluster's status: \ No newline at end of file diff --git a/marvel/docs/introduction.asciidoc b/marvel/docs/introduction.asciidoc deleted file mode 100644 index a7d1fb039af..00000000000 --- a/marvel/docs/introduction.asciidoc +++ /dev/null @@ -1,23 +0,0 @@ -[[introduction]] -== Introduction - -_Marvel_ is a plugin for Elasticsearch that enables you to easily monitor and manage -your Elasticsearch cluster. Marvel aggregates cluster-wide statistics and events and provides a -Kibana app that makes it easy to view and analyze the data. - -When you open the Marvel app in Kibana, you see the key metrics that indicate the health of your -cluster. - -From there, you can dive into the details for particular nodes and indices. - -[float] -=== Where to Go Next - -* <> steps through how to install and start using Marvel to -monitor Elasticsearch. - -[float] -=== Have Comments, Questions, or Feedback? - -Head over to our https://discuss.elastic.co/c/marvel[Marvel Discussion Forum] to share your -experience, questions, and suggestions. \ No newline at end of file diff --git a/marvel/docs/release-notes.asciidoc b/marvel/docs/release-notes.asciidoc deleted file mode 100644 index 491afba07b3..00000000000 --- a/marvel/docs/release-notes.asciidoc +++ /dev/null @@ -1,252 +0,0 @@ -[[release-notes]] -== Release Notes - -[float] -[[version-compatibility]] -=== Version Compatibility - -Marvel {marvelversion} is compatible with: - -* Elasticsearch {esversion} -* License {licenseversion} -* Shield {shieldversion} -* Kibana {kibanaversion} - -[float] -[[upgrading]] -=== Upgrading Marvel -When upgrading Marvel, you must upgrade *every node in the cluster*. If you're using a monitoring -cluster, upgrade the nodes in the monitoring cluster before upgrading your production cluster. You -do not need to fully shut down your production or monitoring clusters to perform the upgrade, you -can perform a rolling upgrade. - -To perform a rolling upgrade of Marvel: - -. Disable shard reallocation. While this is optional, it enables a faster startup after cluster -shutdown. If you don't disable shard reallocation, the nodes immediately start trying to -replicate shards to each other on startup and spend a lot of time on wasted I/O. With shard -reallocation disabled, the nodes join the cluster with their indices intact and do not attempt to -rebalance. After startup is complete, you can turn reallocation back on. -+ -[source,sh] --------------------------------------------------- -curl -XPUT localhost:9200/_cluster/settings -d '{ - "transient" : { - "cluster.routing.allocation.enable" : "none" - } -}' --------------------------------------------------- - -. Upgrade each node, one at a time: - -.. Stop Elasticsearch. -.. Remove the old version of the Marvel plugin: -+ -[source,sh] --------------------------------------------------- -bin/plugin remove marvel-agent --------------------------------------------------- - -.. Install the latest version of the Marvel plugin: -+ -[source,sh] --------------------------------------------------- -bin/plugin install marvel-agent --------------------------------------------------- - -.. Start Elasticsearch and confirm that the node rejoins the cluster and that there are no errors -in the logs. - -. When you've upgraded all of the nodes in the cluster, reenable shard allocation: -+ -[source,sh] --------------------------------------------------- -curl -XPUT localhost:9200/_cluster/settings -d '{ - "transient" : { - "cluster.routing.allocation.enable" : "all" - } -}' --------------------------------------------------- - -[float] -[[change_list]] -=== Change List - -[float] -==== 2.0.0-beta2 - -Marvel versions are now aligned with Elasticsearch version. - -- Agent: - * Added: add license expiration support to stop collecting data 7 days - after license expiration - * Added: add integration tests - * Added: add support to index cluster license information - * Added: add support to collect node statistics - * Added: add support to collect index recoveries - * Added: add support to collect index level statistics - * Added: add support to collect cluster statistics - * Added: add support to collect cluster state - * Improved: update project to work with Elasticsearch 2.x - -- Monitoring UI: - * TODO - -- Sense: - * Sense has been removed from Marvel starting version 2.x and is now shipped as - a Kibana application - -[float] -==== 1.3.1 - -- Agent: - * Added: add timeouts to better deal with unresponsive ES nodes / hiccups. - * Added: Allow SSL hostname verification to be disabled. - * Fixed: Node failed to start if HTTP is disabled. - * Fixed: Potential NPE if HTTP server didn't start fast enough. - * Fixed: `marvel.agent.indices` wasn't dynamically updatable when using a single value or a - comma separated list. - * Fixed: unused shield SSL settings caused errors during start up. - -- Monitoring UI: - * Fixed: Upgraded Kibana3 to latest version, fixing a wrap around issue in Safari. - -- Sense: - * Added: Cluster health's level url parameter. - * Added: _recovery API. - * Fixed: trailing space after URL broke request parsing. - * Added: _search_shards API. - -[float] -==== 1.3.0 - -- Agent: - * Added: support for shipping over https. - * Removed: support for optional shard level stats due to an incompatible change in ES 1.4. - * Fixed: an issue causing a tribe node (ES 1.4.0) not to initialize when Marvel is installed. - * Improved: resiliency and error checking around the marvel index template (both checking for it and adding it). - * Improved: logging upon error, supressing repreated logs. - * Added: Automcally detect the local node's port when using not default Marvel settings (previously was always 9200) - * Improved: Change _bulk export command to set the index name in the url param. This is usefull when `rest.action.multi.allow_explicit_index` is set to false. - -- Monitoring UI - * Added: charts for new circuit breakers introduce with ES 1.4.0 - * Added: a chart to plot circuit break limit. - * Added: a charts for query cache. - * Added: charts for index throttling. - * Added: charts to expose memory usage of the index writer and version map. - * Fixed: Network Transport Bytes Received chart actually shows bytes sent - * Fixed: Node Stats dashboard missed some thread pools. - -- Sense: - * Added: a settings to allow disabling mappings and/or indices autocomplete. This is usefull for extremly large deployments where parsing by the browser is unrealistic. - * Added: Custer Reroute API. - * Added: Get Field Mappings API, - * Fixed: Url auto complete failed with completing fully qualified urls (i.e. with protocol and host). - * Added: Query Cache parmaters to the Search API. - * Added: Analyze API. - * Added: Validate Query API. - * Fixed: include_in_parent and include_in_root is missing for nested type mapping. - * Added: Put Percolator API. - * Fixed: Range filter template to use gt, gte, lt and lte. - * Added: cluster.routing.allocation.* settings - * Added: weight param to the Function Score query. - * Added: Flush API. - * Added: show_term_doc_count_error parameter to the Terms Aggregation. - * Added: Update API - * Added: _geo_distance as a sort option. - * Added: Updated the Significant Terms aggregation to 1.4.0 features. - * Added: metadata fields to the Mapping API. - * Added: Get Index API. - * Added: Scripted Metric Aggregation. - * Added: simple_query_string query. - * Added: Updated the More Like This query to 1.4.0 features. - * Added: min_childeren, max_children options to the has_child query and dilter. - * Added: Updated execution hint options in terms and significant terms aggs. - * Added: transform section of Mappings API. - * Added: indexed scripts and templates. - * Added: Geo Bounds aggregation. - * Added: Top Hits aggregation. - * Added: collect_mode option the Terms aggregation. - * Added: Percentiles Rank aggregation. - * Added: Disk Threshold Allocator settings. - * Fixed: Exists filter auto complete. - * Fixed: Snapshot and Restore API failed to autocomplete repository settings. - -[float] -==== 1.2.1 - - Fix a cluster state data shipping for cluster states larger than 16K (in `SMILE` format and without meta data). - -[float] -==== 1.2.0 - - New Shard Allocation Dashboard. - - Simplified navigation and dashboard customization. - - Sense: - * Update the KB to the ES 1.2.0 API, adding the following: - * `_cat/plugins` - * `_cat/segments` - * `_search/template` - * `_count` - * `_snapshot` - * Alias support for index creation. - * Significant terms aggregation. - * Percentiles aggregation. - * Cardinality aggregation. - * Time_zone keyword to date_histogram facet/aggregation. - * Removed deprecated `custom_score` & `custom_boost_factor` from the 1.0 API. - * Fixed a bug causing the query panel to loose focus after running a command. - - - Charts and Dashboards changes: - * Added an information icon next to the status information of Cluster Summary panel. Hovering on it will show a - short explanation of current status. - * The indices stats table in the Overview dashboard now shows an information icon next to red and yellow indices. - Hovering on it will show a short shard level summary. - * Marvel's index template will now be automatically updated upon upgrade. - * Added field data & filter cache eviction charts to Node Stats dashboard and Index Stats dashboard. - * Added field data circuit breaker charts to Node Stats dashboard. - * Added a registration & purchasing form. - * Hidden indexes are now shown by default. - * Default cluster pulse default time span to 7 days. - * Fixed: Split brain detection algorithm didn't fire in some configurations. - - - - `marvel.agent.exporter.es.host` configuration option now defaults to port 9200. - -[float] -==== 1.1.1 - - Fixed: agent did not interpret timeout settings correctly, causing potential connectivity errors when shipping data. - -[float] -==== 1.1.0 - - Improved Sense's autocomplete suggestions: - * Added Snapshot & Restore - * Added Aggregations - * Added support for url query string parameters - * Updated for breaking changes in Elasticsearch 1.0 - - Updated welcome splash screen. - - Sense now uses the last used server when opened (previously used the hostname used to access it). - - The agent's keep-alive thread is now stopped upon errors to reduce log noise. It will be restarted - upon successful connection. - - Improved error reporting for failures of items in the agent's bulk requests. - - Index Statistics Dashboard: Indexing Rate Primaries chart was based on the wrong field. - - Introduced `marvel.agent.shard_stats.enabled` to control exporting of shard level statistics. Defaults to `false`. - - Changed agent's default sampling rate to 10s (was 5s) - - Added a visual indication for the master node at the Nodes section of the Overview Dashboard - - Node and Indices tables visually indicate stale data - - Added error reporting to nodes and indices tables - - Made the following agent settings changeable via the Cluster Update Settings API: - * marvel.agent.interval (also supports setting to -1 to disable exporting) - * marvel.agent.exporter.es.hosts - * marvel.agent.exporter.es.timeout - * marvel.agent.shard_stats.enabled - -[float] -==== 1.0.2 - - Kibana uses `window.location.protocol` (http or https) to make ES calls. - - Added support for basic authentication when sending data from agent. See <>. - - Reduced DEBUG logging verbosity. - -[float] -==== 1.0.1 - - fixed an issue with usage statistics report. - - improve logging message when running on old Elasticsearch versions. From ff1e6ec776dff61422a0c00335bbaa54b9836406 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 19 Nov 2015 16:07:01 +0100 Subject: [PATCH 27/95] Marvel: Fix memory leak when something goes wrong in LocalExporter When the bulk request executed by the LocalExporter has at least 1 failure, an exception is thrown and the BulkRequestBuilder is not reset. Then the next time Marvel collects data it will contain all the previous requests plus the new ones... leading to an OOM. IThis behavior was reproduced with a closed marvel index but it also happen in case of unassigned shards. Original commit: elastic/x-pack-elasticsearch@9c3caa08ebd2dcfe0e73332a8038c2b9e9566723 --- .../marvel/agent/exporter/ExportBulk.java | 5 ++- .../agent/exporter/local/LocalBulk.java | 44 +++++++++++++++--- .../agent/exporter/local/LocalExporter.java | 4 ++ .../exporter/local/LocalExporterTests.java | 45 +++++++++++++------ 4 files changed, 77 insertions(+), 21 deletions(-) diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportBulk.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportBulk.java index 5e2632e9667..b6f108c905b 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportBulk.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportBulk.java @@ -53,9 +53,12 @@ public abstract class ExportBulk { } else { exception = e; } - throw exception; } + // rethrow exception + if (exception != null) { + throw exception; + } } protected void onClose() throws Exception { diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java index db55c428180..6482894ca04 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java @@ -6,6 +6,7 @@ package org.elasticsearch.marvel.agent.exporter.local; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -34,7 +35,7 @@ public class LocalBulk extends ExportBulk { private final RendererRegistry renderers; private BytesStreamOutput buffer = null; - private BulkRequestBuilder requestBuilder; + BulkRequestBuilder requestBuilder; AtomicReference state = new AtomicReference<>(); @@ -94,22 +95,53 @@ public class LocalBulk extends ExportBulk { if (state.get() != State.ACTIVE || requestBuilder == null) { return; } - logger.trace("exporter [{}] - exporting data...", name); - BulkResponse bulkResponse = requestBuilder.get(); - if (bulkResponse.hasFailures()) { - throw new ElasticsearchException(bulkResponse.buildFailureMessage()); + try { + logger.trace("exporter [{}] - exporting {} documents", name, requestBuilder.numberOfActions()); + BulkResponse bulkResponse = requestBuilder.get(); + if (bulkResponse.hasFailures()) { + throw new ElasticsearchException(buildFailureMessage(bulkResponse)); + } + } finally { + requestBuilder = null; + if (buffer != null) { + buffer.reset(); + } } - requestBuilder = null; } void terminate() { state.set(State.TERMINATING); synchronized (this) { requestBuilder = null; + buffer = null; state.compareAndSet(State.TERMINATING, State.TERMINATED); } } + /** + * In case of something goes wrong and there's a lot of shards/indices, + * we limit the number of failures displayed in log. + */ + private String buildFailureMessage(BulkResponse bulkResponse) { + BulkItemResponse[] items = bulkResponse.getItems(); + + if (logger.isDebugEnabled() || (items.length < 100)) { + return bulkResponse.buildFailureMessage(); + } + + StringBuilder sb = new StringBuilder(); + sb.append("failure in bulk execution, only the first 100 failures are printed:"); + for (int i = 0; i < items.length && i < 100; i++) { + BulkItemResponse item = items[i]; + if (item.isFailed()) { + sb.append("\n[").append(i) + .append("]: index [").append(item.getIndex()).append("], type [").append(item.getType()).append("], id [").append(item.getId()) + .append("], message [").append(item.getFailureMessage()).append("]"); + } + } + return sb.toString(); + } + enum State { ACTIVE, TERMINATING, diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java index a83e3ea45c5..7fec1adf988 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java @@ -51,6 +51,10 @@ public class LocalExporter extends Exporter implements ClusterStateListener { clusterService.add(this); } + LocalBulk getBulk() { + return bulk; + } + @Override public void clusterChanged(ClusterChangedEvent event) { LocalBulk currentBulk = bulk; diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java index 76b0d577ff3..52b2268762e 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.marvel.agent.exporter.local; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; -import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.search.SearchResponse; @@ -14,6 +14,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -47,19 +48,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import static org.elasticsearch.marvel.agent.exporter.Exporter.MIN_SUPPORTED_TEMPLATE_VERSION; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.*; +import static org.mockito.Mockito.*; @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class LocalExporterTests extends MarvelIntegTestCase { @@ -284,6 +275,32 @@ public class LocalExporterTests extends MarvelIntegTestCase { assertFalse("null version should not be sufficient", exporter.installedTemplateVersionIsSufficient(Version.CURRENT, null)); } + public void testLocalExporterFlush() throws Exception { + internalCluster().startNode(Settings.builder() + .put("marvel.agent.exporters._local.type", LocalExporter.TYPE) + .put("marvel.agent.exporters._local.enabled", true) + .build()); + securedEnsureGreen(); + + LocalExporter exporter = getLocalExporter("_local"); + + logger.debug("--> exporting a single marvel doc"); + exporter.export(Collections.singletonList(newRandomMarvelDoc())); + awaitMarvelDocsCount(is(1L)); + assertNull(exporter.getBulk().requestBuilder); + + logger.debug("--> closing marvel indices"); + assertAcked(client().admin().indices().prepareClose(MarvelSettings.MARVEL_INDICES_PREFIX + "*").get()); + + try { + logger.debug("--> exporting a second marvel doc"); + exporter.export(Collections.singletonList(newRandomMarvelDoc())); + } catch (ElasticsearchException e) { + assertThat(e.getMessage(), allOf(containsString("failure in bulk execution"), containsString("IndexClosedException[closed]"))); + assertNull(exporter.getBulk().requestBuilder); + } + } + private LocalExporter getLocalExporter(String name) throws Exception { final Exporter exporter = internalCluster().getInstance(Exporters.class).getExporter(name); assertThat(exporter, notNullValue()); From a47adfb270d2d336d21fb9a9c341c18f036b9ce1 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 10 Nov 2015 15:24:58 +0100 Subject: [PATCH 28/95] Marvel: Index Nodes in Time Based Index and Data Index Closes elastic/elasticsearch#946 Original commit: elastic/x-pack-elasticsearch@0f5721ffbf689f3127a1f7b4661006a7a53b73c3 --- .../cluster/ClusterStateCollector.java | 30 ++++- .../cluster/ClusterStateNodeMarvelDoc.java | 29 +++++ .../cluster/DiscoveryNodeMarvelDoc.java | 24 ++++ .../collector/node/NodeStatsCollector.java | 3 +- .../marvel/agent/renderer/RendererModule.java | 6 +- .../cluster/ClusterStateNodeRenderer.java | 35 ++++++ .../cluster/ClusterStateRenderer.java | 1 - .../cluster/DiscoveryNodeRenderer.java | 54 +++++++++ .../main/resources/marvel_index_template.json | 22 +++- .../cluster/ClusterStateCollectorTests.java | 114 ++++++++++++------ .../renderer/cluster/ClusterStateTests.java | 79 ++++++++++++ .../test/resources/samples/cluster_state.json | 11 +- 12 files changed, 349 insertions(+), 59 deletions(-) create mode 100644 marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateNodeMarvelDoc.java create mode 100644 marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/DiscoveryNodeMarvelDoc.java create mode 100644 marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateNodeRenderer.java create mode 100644 marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/DiscoveryNodeRenderer.java diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java index abe7c7330dd..faaf64ff332 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java @@ -9,6 +9,8 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.agent.collector.AbstractCollector; @@ -25,18 +27,20 @@ import java.util.List; /** * Collector for cluster state. *

- * This collector runs on the master node only and collects the {@link ClusterStateMarvelDoc} document + * This collector runs on the master node only and collects {@link ClusterStateMarvelDoc} document * at a given frequency. */ public class ClusterStateCollector extends AbstractCollector { public static final String NAME = "cluster-state-collector"; public static final String TYPE = "cluster_state"; + public static final String NODES_TYPE = "nodes"; + public static final String NODE_TYPE = "node"; private final Client client; @Inject - public ClusterStateCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee, + public ClusterStateCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee, SecuredClient client) { super(settings, NAME, clusterService, marvelSettings, marvelLicensee); this.client = client; @@ -49,12 +53,28 @@ public class ClusterStateCollector extends AbstractCollector doCollect() throws Exception { - List results = new ArrayList<>(1); + List results = new ArrayList<>(3); ClusterState clusterState = clusterService.state(); - ClusterHealthResponse clusterHealth = client.admin().cluster().prepareHealth().get(marvelSettings.clusterStateTimeout()); + String clusterUUID = clusterState.metaData().clusterUUID(); + String stateUUID = clusterState.stateUUID(); + long timestamp = System.currentTimeMillis(); + + // Adds a cluster_state document with associated status + ClusterHealthResponse clusterHealth = client.admin().cluster().prepareHealth().get(marvelSettings.clusterStateTimeout()); + results.add(new ClusterStateMarvelDoc(clusterUUID, TYPE, timestamp, clusterState, clusterHealth.getStatus())); + + DiscoveryNodes nodes = clusterState.nodes(); + if (nodes != null) { + for (DiscoveryNode node : nodes) { + // Adds a document for every node in the marvel timestamped index (type "nodes") + results.add(new ClusterStateNodeMarvelDoc(clusterUUID, NODES_TYPE, timestamp, stateUUID, node.getId())); + + // Adds a document for every node in the marvel data index (type "node") + results.add(new DiscoveryNodeMarvelDoc(MarvelSettings.MARVEL_DATA_INDEX_NAME, NODE_TYPE, node.getId(), clusterUUID, timestamp, node)); + } + } - results.add(new ClusterStateMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), clusterState, clusterHealth.getStatus())); return Collections.unmodifiableCollection(results); } } diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateNodeMarvelDoc.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateNodeMarvelDoc.java new file mode 100644 index 00000000000..3ec6ba6710d --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateNodeMarvelDoc.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.collector.cluster; + +import org.elasticsearch.marvel.agent.exporter.MarvelDoc; + +public class ClusterStateNodeMarvelDoc extends MarvelDoc { + + private final String stateUUID; + private final String nodeId; + + public ClusterStateNodeMarvelDoc(String clusterUUID, String type, long timestamp, String stateUUID, String nodeId) { + super(clusterUUID, type, timestamp); + this.stateUUID = stateUUID; + this.nodeId = nodeId; + } + + public String getStateUUID() { + return stateUUID; + } + + public String getNodeId() { + return nodeId; + } +} + diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/DiscoveryNodeMarvelDoc.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/DiscoveryNodeMarvelDoc.java new file mode 100644 index 00000000000..8af34289165 --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/DiscoveryNodeMarvelDoc.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.collector.cluster; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.marvel.agent.exporter.MarvelDoc; + +public class DiscoveryNodeMarvelDoc extends MarvelDoc { + + private final DiscoveryNode node; + + public DiscoveryNodeMarvelDoc(String index, String type, String id, String clusterUUID, long timestamp, DiscoveryNode node) { + super(index, type, id, clusterUUID, timestamp); + this.node = node; + } + + public DiscoveryNode getNode() { + return node; + } +} + diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsCollector.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsCollector.java index 43fe39cb3ac..8e884b38f6d 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsCollector.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsCollector.java @@ -9,6 +9,7 @@ package org.elasticsearch.marvel.agent.collector.node; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.bootstrap.BootstrapInfo; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -61,7 +62,7 @@ public class NodeStatsCollector extends AbstractCollector { // for nodes that can hold data. Client nodes can collect nodes stats because // elasticsearch correctly handles the nodes stats for client nodes. return super.shouldCollect() - && (clusterService.localNode().isDataNode() == false || nodeEnvironment.hasNodeFile()); + && (DiscoveryNode.nodeRequiresLocalStorage(settings) == false || nodeEnvironment.hasNodeFile()); } @Override diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java index 62c0631bb2e..931ae428612 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java @@ -15,9 +15,7 @@ import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector; import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsCollector; import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector; import org.elasticsearch.marvel.agent.collector.shards.ShardsCollector; -import org.elasticsearch.marvel.agent.renderer.cluster.ClusterInfoRenderer; -import org.elasticsearch.marvel.agent.renderer.cluster.ClusterStateRenderer; -import org.elasticsearch.marvel.agent.renderer.cluster.ClusterStatsRenderer; +import org.elasticsearch.marvel.agent.renderer.cluster.*; import org.elasticsearch.marvel.agent.renderer.indices.IndexRecoveryRenderer; import org.elasticsearch.marvel.agent.renderer.indices.IndexStatsRenderer; import org.elasticsearch.marvel.agent.renderer.indices.IndicesStatsRenderer; @@ -54,6 +52,8 @@ public class RendererModule extends AbstractModule { bind(ClusterStateRenderer.class).asEagerSingleton(); mbinder.addBinding(ClusterStateCollector.TYPE).to(ClusterStateRenderer.class); + mbinder.addBinding(ClusterStateCollector.NODES_TYPE).to(ClusterStateNodeRenderer.class); + mbinder.addBinding(ClusterStateCollector.NODE_TYPE).to(DiscoveryNodeRenderer.class); bind(ShardsRenderer.class).asEagerSingleton(); mbinder.addBinding(ShardsCollector.TYPE).to(ShardsRenderer.class); diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateNodeRenderer.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateNodeRenderer.java new file mode 100644 index 00000000000..b2afa2464d8 --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateNodeRenderer.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer.cluster; + +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateNodeMarvelDoc; +import org.elasticsearch.marvel.agent.renderer.AbstractRenderer; + +import java.io.IOException; + +public class ClusterStateNodeRenderer extends AbstractRenderer { + + public ClusterStateNodeRenderer() { + super(null, false); + } + + @Override + protected void doRender(ClusterStateNodeMarvelDoc marvelDoc, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.field(Fields.STATE_UUID, marvelDoc.getStateUUID()); + builder.startObject(Fields.NODE); + builder.field(Fields.ID, marvelDoc.getNodeId()); + builder.endObject(); + } + + static final class Fields { + static final XContentBuilderString STATE_UUID = new XContentBuilderString("state_uuid"); + static final XContentBuilderString NODE = new XContentBuilderString("node"); + static final XContentBuilderString ID = new XContentBuilderString("id"); + } +} diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRenderer.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRenderer.java index 78f473edbcc..abf759d88b1 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRenderer.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRenderer.java @@ -22,7 +22,6 @@ public class ClusterStateRenderer extends AbstractRenderer { + + public DiscoveryNodeRenderer() { + super(null, false); + } + + @Override + protected void doRender(DiscoveryNodeMarvelDoc marvelDoc, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(Fields.NODE); + + DiscoveryNode node = marvelDoc.getNode(); + if (node != null) { + builder.field(Fields.NAME, node.getName()); + builder.field(Fields.TRANSPORT_ADDRESS, node.getAddress().toString()); + + builder.startObject(Fields.ATTRIBUTES); + for (ObjectObjectCursor attr : node.getAttributes()) { + builder.field(attr.key, attr.value); + } + builder.endObject(); + builder.field(Fields.ID, node.getId()); + } + + builder.endObject(); + } + + static final class Fields { + static final XContentBuilderString NODE = new XContentBuilderString("node"); + static final XContentBuilderString NAME = new XContentBuilderString("name"); + static final XContentBuilderString TRANSPORT_ADDRESS = new XContentBuilderString("transport_address"); + static final XContentBuilderString ATTRIBUTES = new XContentBuilderString("attributes"); + static final XContentBuilderString ID = new XContentBuilderString("id"); + } +} + + + diff --git a/marvel/src/main/resources/marvel_index_template.json b/marvel/src/main/resources/marvel_index_template.json index d367802d90b..30fb5121bae 100644 --- a/marvel/src/main/resources/marvel_index_template.json +++ b/marvel/src/main/resources/marvel_index_template.json @@ -186,9 +186,6 @@ "type": "string", "index": "not_analyzed" }, - "nodes": { - "enabled": false - }, "shards": { "type": "object" } @@ -199,6 +196,25 @@ "cluster_info": { "enabled": false }, + "node": { + "enabled": false + }, + "nodes": { + "properties": { + "state_uuid": { + "type": "string", + "index": "not_analyzed" + }, + "node": { + "properties": { + "id": { + "type": "string", + "index": "not_analyzed" + } + } + } + } + }, "node_stats": { "properties": { "node_stats": { diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java index 57078392eeb..2bf231c7750 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java @@ -14,32 +14,18 @@ import org.elasticsearch.marvel.agent.exporter.MarvelDoc; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.license.MarvelLicensee; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.*; public class ClusterStateCollectorTests extends AbstractCollectorTestCase { + public void testClusterStateCollectorNoIndices() throws Exception { - Collection results = newClusterStateCollector().doCollect(); - assertThat(results, hasSize(1)); - - MarvelDoc marvelDoc = results.iterator().next(); - assertNotNull(marvelDoc); - assertThat(marvelDoc, instanceOf(ClusterStateMarvelDoc.class)); - - ClusterStateMarvelDoc clusterStateMarvelDoc = (ClusterStateMarvelDoc) marvelDoc; - assertThat(clusterStateMarvelDoc.clusterUUID(), equalTo(client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID())); - assertThat(clusterStateMarvelDoc.timestamp(), greaterThan(0L)); - assertThat(clusterStateMarvelDoc.type(), equalTo(ClusterStateCollector.TYPE)); - assertNotNull(clusterStateMarvelDoc.getClusterState()); - - ClusterState clusterState = clusterStateMarvelDoc.getClusterState(); - assertThat(clusterState.getRoutingTable().allShards(), hasSize(0)); + assertMarvelDocs(newClusterStateCollector().doCollect(), 0); } public void testClusterStateCollectorOneIndex() throws Exception { @@ -56,33 +42,20 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase { securedFlush(); securedRefresh(); + assertHitCount(client().prepareSearch().setSize(0).get(), nbDocs); - - Collection results = newClusterStateCollector().doCollect(); - assertThat(results, hasSize(1)); - - MarvelDoc marvelDoc = results.iterator().next(); - assertNotNull(marvelDoc); - assertThat(marvelDoc, instanceOf(ClusterStateMarvelDoc.class)); - - ClusterStateMarvelDoc clusterStateMarvelDoc = (ClusterStateMarvelDoc) marvelDoc; - assertThat(clusterStateMarvelDoc.clusterUUID(), equalTo(client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID())); - assertThat(clusterStateMarvelDoc.timestamp(), greaterThan(0L)); - assertThat(clusterStateMarvelDoc.type(), equalTo(ClusterStateCollector.TYPE)); - - assertNotNull(clusterStateMarvelDoc.getClusterState()); - - ClusterState clusterState = clusterStateMarvelDoc.getClusterState(); - assertThat(clusterState.getRoutingTable().allShards("test"), hasSize(nbShards)); + assertMarvelDocs(newClusterStateCollector().doCollect(), nbShards); } public void testClusterStateCollectorMultipleIndices() throws Exception { int nbIndices = randomIntBetween(1, 5); int[] docsPerIndex = new int[nbIndices]; int[] shardsPerIndex = new int[nbIndices]; + int nbShards = 0; for (int i = 0; i < nbIndices; i++) { shardsPerIndex[i] = randomIntBetween(1, 5); + nbShards += shardsPerIndex[i]; assertAcked(prepareCreate("test-" + i).setSettings(Settings.settingsBuilder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shardsPerIndex[i]) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) @@ -96,11 +69,14 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase { securedFlush(); securedRefresh(); + for (int i = 0; i < nbIndices; i++) { assertHitCount(client().prepareSearch("test-" + i).setSize(0).get(), docsPerIndex[i]); } Collection results = newClusterStateCollector().doCollect(); + assertMarvelDocs(results, nbShards); + MarvelDoc marvelDoc = results.iterator().next(); assertNotNull(marvelDoc); assertThat(marvelDoc, instanceOf(ClusterStateMarvelDoc.class)); @@ -168,4 +144,70 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase { internalCluster().getInstance(MarvelLicensee.class, nodeId), securedClient(nodeId)); } + + private void assertMarvelDocs(Collection results, final int nbShards) { + assertThat("expecting 1 document for the cluster state and 2 documents per node", results, hasSize(1 + internalCluster().size() * 2)); + + final ClusterState clusterState = securedClient().admin().cluster().prepareState().get().getState(); + final String clusterUUID = clusterState.getMetaData().clusterUUID(); + final String stateUUID = clusterState.stateUUID(); + + List clusterStateNodes = new ArrayList<>(); + List discoveryNodes = new ArrayList<>(); + + for (MarvelDoc marvelDoc : results) { + assertThat(marvelDoc.clusterUUID(), equalTo(clusterUUID)); + assertThat(marvelDoc.timestamp(), greaterThan(0L)); + assertThat(marvelDoc, anyOf(instanceOf(ClusterStateMarvelDoc.class), instanceOf(ClusterStateNodeMarvelDoc.class), instanceOf(DiscoveryNodeMarvelDoc.class))); + + switch (marvelDoc.type()) { + case ClusterStateCollector.TYPE: + ClusterStateMarvelDoc clusterStateMarvelDoc = (ClusterStateMarvelDoc) marvelDoc; + assertThat(clusterStateMarvelDoc.getClusterState().getRoutingTable().allShards(), hasSize(nbShards)); + break; + + case ClusterStateCollector.NODES_TYPE: + ClusterStateNodeMarvelDoc clusterStateNodeMarvelDoc = (ClusterStateNodeMarvelDoc) marvelDoc; + assertThat(clusterStateNodeMarvelDoc.getStateUUID(), equalTo(stateUUID)); + assertThat(clusterStateNodeMarvelDoc.getNodeId(), not(isEmptyOrNullString())); + clusterStateNodes.add(clusterStateNodeMarvelDoc); + break; + + case ClusterStateCollector.NODE_TYPE: + DiscoveryNodeMarvelDoc discoveryNodeMarvelDoc = (DiscoveryNodeMarvelDoc) marvelDoc; + assertThat(discoveryNodeMarvelDoc.index(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_NAME)); + assertThat(discoveryNodeMarvelDoc.id(), not(isEmptyOrNullString())); + assertNotNull(discoveryNodeMarvelDoc.getNode()); + discoveryNodes.add(discoveryNodeMarvelDoc); + break; + default: + fail("unknown marvel document type " + marvelDoc.type()); + } + } + + assertThat(clusterStateNodes, hasSize(internalCluster().size())); + assertThat(discoveryNodes, hasSize(internalCluster().size())); + + for (final String nodeName : internalCluster().getNodeNames()) { + final String nodeId = internalCluster().clusterService(nodeName).localNode().getId(); + + boolean found = false; + for (ClusterStateNodeMarvelDoc doc : clusterStateNodes) { + if (nodeId.equals(doc.getNodeId())) { + found = true; + break; + } + } + assertTrue("Could not find node id [" + nodeName + "]", found); + + found = false; + for (DiscoveryNodeMarvelDoc doc : discoveryNodes) { + if (nodeName.equals(doc.getNode().getName())) { + found = true; + break; + } + } + assertTrue("Could not find node name [" + nodeName + "]", found); + } + } } diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java index ef2d1fe00d2..2405147adac 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector; +import org.elasticsearch.marvel.agent.renderer.AbstractRenderer; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.search.SearchHit; @@ -23,6 +24,8 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.core.Is.is; @ClusterScope(scope = Scope.TEST) public class ClusterStateTests extends MarvelIntegTestCase { @@ -89,4 +92,80 @@ public class ClusterStateTests extends MarvelIntegTestCase { .setTypes(ClusterStateCollector.TYPE) .setQuery(QueryBuilders.matchQuery("cluster_state.nodes." + nodes.masterNodeId() + ".name", nodes.masterNode().name())).get(), 0L); } + + public void testClusterStateNodes() throws Exception { + final long nbNodes = internalCluster().size(); + + logger.debug("--> waiting for documents to be collected"); + awaitMarvelDocsCount(greaterThanOrEqualTo(nbNodes), ClusterStateCollector.NODES_TYPE); + + logger.debug("--> searching for marvel documents of type [{}]", ClusterStateCollector.NODES_TYPE); + SearchResponse response = client().prepareSearch().setTypes(ClusterStateCollector.NODES_TYPE).get(); + assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes)); + + logger.debug("--> checking that every document contains the expected fields"); + String[] filters = { + AbstractRenderer.Fields.CLUSTER_UUID.underscore().toString(), + AbstractRenderer.Fields.TIMESTAMP.underscore().toString(), + ClusterStateNodeRenderer.Fields.STATE_UUID.underscore().toString(), + ClusterStateNodeRenderer.Fields.NODE.underscore().toString(), + ClusterStateNodeRenderer.Fields.NODE.underscore().toString() + "." + ClusterStateNodeRenderer.Fields.ID.underscore().toString(), + }; + + for (SearchHit searchHit : response.getHits().getHits()) { + Map fields = searchHit.sourceAsMap(); + + for (String filter : filters) { + assertContains(filter, fields); + } + } + + logger.debug("--> cluster state nodes successfully collected"); + } + + public void testClusterStateNode() throws Exception { + final long nbNodes = internalCluster().size(); + + logger.debug("--> waiting for documents to be collected"); + awaitMarvelDocsCount(greaterThanOrEqualTo(nbNodes), ClusterStateCollector.NODE_TYPE); + + logger.debug("--> searching for marvel documents of type [{}]", ClusterStateCollector.NODE_TYPE); + SearchResponse response = client().prepareSearch().setTypes(ClusterStateCollector.NODE_TYPE).get(); + assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes)); + + logger.debug("--> checking that every document contains the expected fields"); + String[] filters = { + AbstractRenderer.Fields.CLUSTER_UUID.underscore().toString(), + AbstractRenderer.Fields.TIMESTAMP.underscore().toString(), + DiscoveryNodeRenderer.Fields.NODE.underscore().toString(), + DiscoveryNodeRenderer.Fields.NODE.underscore().toString() + "." + DiscoveryNodeRenderer.Fields.ID.underscore().toString(), + DiscoveryNodeRenderer.Fields.NODE.underscore().toString() + "." + DiscoveryNodeRenderer.Fields.NAME.underscore().toString(), + DiscoveryNodeRenderer.Fields.NODE.underscore().toString() + "." + DiscoveryNodeRenderer.Fields.ATTRIBUTES.underscore().toString(), + DiscoveryNodeRenderer.Fields.NODE.underscore().toString() + "." + DiscoveryNodeRenderer.Fields.TRANSPORT_ADDRESS.underscore().toString(), + }; + + for (SearchHit searchHit : response.getHits().getHits()) { + Map fields = searchHit.sourceAsMap(); + + for (String filter : filters) { + assertContains(filter, fields); + } + } + + for (final String nodeName : internalCluster().getNodeNames()) { + final String nodeId = internalCluster().clusterService(nodeName).localNode().getId(); + + logger.debug("--> getting marvel document for node id [{}]", nodeId); + assertThat(client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterStateCollector.NODE_TYPE, nodeId).get().isExists(), is(true)); + + // checks that document is not indexed + assertHitCount(client().prepareSearch().setSize(0) + .setTypes(ClusterStateCollector.NODE_TYPE) + .setQuery(QueryBuilders.boolQuery() + .should(QueryBuilders.matchQuery("node.id", nodeId)) + .should(QueryBuilders.matchQuery("node.name", nodeName))).get(), 0); + } + + logger.debug("--> cluster state node successfully collected"); + } } diff --git a/marvel/src/test/resources/samples/cluster_state.json b/marvel/src/test/resources/samples/cluster_state.json index 7066face4ab..24ec82416a9 100644 --- a/marvel/src/test/resources/samples/cluster_state.json +++ b/marvel/src/test/resources/samples/cluster_state.json @@ -5,15 +5,6 @@ "status": "yellow", "version": 14, "state_uuid": "lj0hNoO9QaeNa1eR2ukktQ", - "master_node": "__node_id__", - "nodes": { - "__node_id__": { - "name": "Box", - "transport_address": "inet[/127.0.0.1:9300]", - "attributes": { - "local": "true" - } - } - } + "master_node": "__node_id__" } } \ No newline at end of file From f2900f71c56d61f9d92b3dafaf2990f957c2fb61 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 20 Nov 2015 12:56:56 +0100 Subject: [PATCH 29/95] test: unmuted slow watches test. Removed the dependency on groovy and added a script impl. that allows to run Thread.sleep(...) in order to simulate a slow watch. Relates to elastic/elasticsearch#724 Original commit: elastic/x-pack-elasticsearch@b18dd89b464ae102b1682069c2faa8c093d9e1b1 --- .../script/SleepScriptEngine.java | 97 +++++++++++++++++++ .../action/delete/ForceDeleteWatchTests.java | 19 +++- .../action/stats/SlowWatchStatsTests.java | 21 +++- 3 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 watcher/src/test/java/org/elasticsearch/script/SleepScriptEngine.java diff --git a/watcher/src/test/java/org/elasticsearch/script/SleepScriptEngine.java b/watcher/src/test/java/org/elasticsearch/script/SleepScriptEngine.java new file mode 100644 index 00000000000..064fa1a83d6 --- /dev/null +++ b/watcher/src/test/java/org/elasticsearch/script/SleepScriptEngine.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.script; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + * A dummy script engine used for testing. Scripts must be a number. Running the script + */ +public class SleepScriptEngine implements ScriptEngineService { + public static final String NAME = "sleep"; + + public static class TestPlugin extends Plugin { + + public TestPlugin() { + } + + @Override + public String name() { + return NAME; + } + + @Override + public String description() { + return "Mock script engine for integration tests"; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(SleepScriptEngine.class); + } + + } + + @Override + public String[] types() { + return new String[]{ NAME }; + } + + @Override + public String[] extensions() { + return types(); + } + + @Override + public boolean sandboxed() { + return true; + } + + @Override + public Object compile(String script) { + return script; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map vars) { + return new AbstractSearchScript() { + @Override + public Object run() { + try { + Thread.sleep(((Number) vars.get("millis")).longValue()); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return true; + } + }; + } + + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map vars) { + return null; + } + + @Override + public void scriptRemoved(@Nullable CompiledScript script) { + } + + @Override + public void close() throws IOException { + } + + public static org.elasticsearch.watcher.support.Script sleepScript(long millis) { + return new org.elasticsearch.watcher.support.Script.Builder.Inline("") + .lang("sleep") + .params(Collections.singletonMap("millis", millis)).build(); + } + +} diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/ForceDeleteWatchTests.java b/watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/ForceDeleteWatchTests.java index 379671420ef..704910a5a50 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/ForceDeleteWatchTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/ForceDeleteWatchTests.java @@ -6,13 +6,22 @@ package org.elasticsearch.watcher.transport.action.delete; import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.SleepScriptEngine; import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.watcher.condition.Condition; +import org.elasticsearch.watcher.condition.ConditionBuilders; import org.elasticsearch.watcher.support.Script; import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; import org.elasticsearch.watcher.transport.actions.delete.DeleteWatchResponse; import org.elasticsearch.watcher.transport.actions.put.PutWatchResponse; import org.elasticsearch.watcher.transport.actions.service.WatcherServiceResponse; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + import static org.elasticsearch.watcher.actions.ActionBuilders.loggingAction; import static org.elasticsearch.watcher.client.WatchSourceBuilders.watchBuilder; import static org.elasticsearch.watcher.condition.ConditionBuilders.scriptCondition; @@ -23,7 +32,6 @@ import static org.hamcrest.Matchers.is; /** */ -@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") public class ForceDeleteWatchTests extends AbstractWatcherIntegrationTestCase { //Disable time warping for the force delete long running watch test @Override @@ -36,11 +44,18 @@ public class ForceDeleteWatchTests extends AbstractWatcherIntegrationTestCase { return false; } + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(SleepScriptEngine.TestPlugin.class); + return plugins; + } + @TestLogging("_root:DEBUG") public void testForceDeleteLongRunningWatch() throws Exception { PutWatchResponse putResponse = watcherClient().preparePutWatch("_name").setSource(watchBuilder() .trigger(schedule(interval("3s"))) - .condition(scriptCondition(Script.inline("sleep 5000; return true"))) + .condition(scriptCondition(SleepScriptEngine.sleepScript(5000))) .addAction("_action1", loggingAction("executed action: {{ctx.id}}"))) .get(); assertThat(putResponse.getId(), equalTo("_name")); diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/SlowWatchStatsTests.java b/watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/SlowWatchStatsTests.java index 8beef6d405e..157a8220d99 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/SlowWatchStatsTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/SlowWatchStatsTests.java @@ -8,17 +8,26 @@ package org.elasticsearch.watcher.transport.action.stats; import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.SleepScriptEngine; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.watcher.WatcherState; import org.elasticsearch.watcher.actions.ActionBuilders; +import org.elasticsearch.watcher.condition.Condition; import org.elasticsearch.watcher.condition.ConditionBuilders; import org.elasticsearch.watcher.execution.ExecutionPhase; import org.elasticsearch.watcher.execution.QueuedWatch; import org.elasticsearch.watcher.input.InputBuilders; +import org.elasticsearch.watcher.support.Script; import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; import org.elasticsearch.watcher.transport.actions.stats.WatcherStatsResponse; import org.joda.time.DateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; @@ -33,13 +42,19 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = TEST, numClientNodes = 0, transportClientRatio = 0, randomDynamicTemplates = false, numDataNodes = 2) -@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") public class SlowWatchStatsTests extends AbstractWatcherIntegrationTestCase { @Override protected boolean timeWarped() { return false; } + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(SleepScriptEngine.TestPlugin.class); + return plugins; + } + @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() @@ -53,7 +68,7 @@ public class SlowWatchStatsTests extends AbstractWatcherIntegrationTestCase { watcherClient().preparePutWatch("_id").setSource(watchBuilder() .trigger(schedule(interval("1s"))) .input(InputBuilders.simpleInput("key", "value")) - .condition(ConditionBuilders.scriptCondition("sleep 10000; return true")) + .condition(ConditionBuilders.scriptCondition(SleepScriptEngine.sleepScript(10000))) .addAction("_action", ActionBuilders.loggingAction("hello {{ctx.watch_id}}!")) ).get(); @@ -78,7 +93,7 @@ public class SlowWatchStatsTests extends AbstractWatcherIntegrationTestCase { watcherClient().preparePutWatch("_id" + i).setSource(watchBuilder() .trigger(schedule(interval("1s"))) .input(InputBuilders.simpleInput("key", "value")) - .condition(ConditionBuilders.scriptCondition("sleep 10000; return true")) + .condition(ConditionBuilders.scriptCondition(SleepScriptEngine.sleepScript(10000))) .addAction("_action", ActionBuilders.loggingAction("hello {{ctx.watch_id}}!")) ).get(); } From 8119451a7baa8430e88d25d5419e7f02406ca5ed Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 20 Nov 2015 12:02:25 -0800 Subject: [PATCH 30/95] Build: Remove project attachment and use ES extra-plugins instead This change removes the project attachment support, and instead relies on x-plugins being checked out under extra-plugins/x-plugins within an elasticsearch checkout. The only real change, other than removing unnecessary code, was to rename the license/core project because gradle project substitution has a bug which causes it to try and use ES core as the substitution. (Unfortunately this is not reproduceable with a simple example, so I have not yet filed an issue with gradle). Original commit: elastic/x-pack-elasticsearch@fa315ffcb5e142f779a366e34c21f43799f8b973 --- README.asciidoc | 12 +++------ build.gradle | 53 +++++++--------------------------------- buildSrc/build.gradle | 41 ------------------------------- buildSrc/settings.gradle | 13 ---------- marvel/build.gradle | 6 ++--- settings.gradle | 50 ------------------------------------- shield/build.gradle | 6 ++--- watcher/build.gradle | 6 ++--- 8 files changed, 21 insertions(+), 166 deletions(-) delete mode 100644 buildSrc/build.gradle delete mode 100644 buildSrc/settings.gradle delete mode 100644 settings.gradle diff --git a/README.asciidoc b/README.asciidoc index 03fd29e8382..242dc1dd174 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -7,12 +7,6 @@ A set of Elastic's commercial plugins: - Watcher - Marvel -= Testing with Elasticsearch -Sometimes it is useful to use your local elasticsearch checkout with x-plugins. To do this, run the following commands: -cd buildSrc -gradle attach --name elasticsearch --path /path/to/elasticsearch/buildSrc -cd .. -gradle attach --name elasticsearch --path /path/to/elasticsearch - -This will cause building x-plugins to reflect any changes in your elasticsearch repo. For example, if you make a change to elasticsearch core, building x-plugins will first re-build elasticsearch core, and use that when building x-plugins. - += Setup +You must checkout x-plugins within an elasticsearch checkout. It must be +called x-plugins, and must be inside the extra-plugins directory. diff --git a/build.gradle b/build.gradle index 29a9ad69a78..559bf2f9c28 100644 --- a/build.gradle +++ b/build.gradle @@ -1,30 +1,14 @@ -apply plugin: 'elasticsearch.project-attachment' - -allprojects { - apply plugin: 'idea' - apply plugin: 'eclipse' -} - -subprojects { - group = 'org.elasticsearch' - version = org.elasticsearch.gradle.VersionProperties.elasticsearch - - configurations.all { - resolutionStrategy.cacheChangingModulesFor(0, 'seconds') - } -} - -if (hasProperty('projectsPrefix') == false) { - allprojects { - project.ext['projectsPrefix'] = '' - } -} - -task clean(type: GradleBuild) { - buildFile = 'buildSrc/build.gradle' - tasks = ['clean'] +if (project.name != 'x-plugins') { + throw new GradleException('You must checkout x-plugins to a directory named x-plugins') +} +if (project.projectDir.parentFile.name != 'extra-plugins') { + throw new GradleException('You must place the x-plugins checkout in a directory named extra-plugins') +} +if (project.rootProject.projectDir != project.projectDir.parentFile.parentFile) { + throw new GradleException('You must place the extra-plugins directory inside the root of an elasticsearch checkout') } +/* elasticsearch-releases http://maven.elasticsearch.org/releases @@ -64,22 +48,3 @@ subprojects { } } */ - -// ================= Local Elasticsearch attachment =============== -if (hasProperty('attachments') && 'elasticsearch' in attachments) { - subprojects { - configurations { - all { - resolutionStrategy { - dependencySubstitution { - substitute module("org.elasticsearch:rest-api-spec:${version}") with project(":elasticsearch:rest-api-spec") - substitute module("org.elasticsearch:elasticsearch:${version}") with project(":elasticsearch:core") - substitute module("org.elasticsearch:test-framework:${version}") with project(":elasticsearch:test-framework") - substitute module("org.elasticsearch.distribution.zip:elasticsearch:${version}") with project(":elasticsearch:distribution:zip") - substitute module("org.elasticsearch.distribution.tar:elasticsearch:${version}") with project(":elasticsearch:distribution:tar") - } - } - } - } - } -} diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle deleted file mode 100644 index 892df99d63c..00000000000 --- a/buildSrc/build.gradle +++ /dev/null @@ -1,41 +0,0 @@ -apply plugin: 'groovy' - -buildscript { - repositories { - maven { - name 'sonatype-snapshots' - url 'http://oss.sonatype.org/content/repositories/snapshots/' - } - } - dependencies { - classpath 'org.elasticsearch.gradle:project-attachment-plugin:1.0.0-SNAPSHOT' - } -} -apply plugin: 'elasticsearch.project-attachment' - -repositories { - mavenCentral() - maven { - name 'sonatype-snapshots' - url "https://oss.sonatype.org/content/repositories/snapshots/" - } - jcenter() -} - -dependencies { - runtime 'org.elasticsearch.gradle:project-attachment-plugin:1.0.0-SNAPSHOT' - runtime 'org.elasticsearch.gradle:build-tools:3.0.0-SNAPSHOT' -} - -if ('elasticsearch' in attachments) { - configurations { - all { - resolutionStrategy { - dependencySubstitution { - substitute module('org.elasticsearch.gradle:build-tools') with project(':elasticsearch') - } - } - } - } -} - diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle deleted file mode 100644 index 65f7417ae07..00000000000 --- a/buildSrc/settings.gradle +++ /dev/null @@ -1,13 +0,0 @@ -buildscript { - repositories { - maven { - name 'sonatype-snapshots' - url 'http://oss.sonatype.org/content/repositories/snapshots/' - } - } - dependencies { - classpath 'org.elasticsearch.gradle:project-attachment-plugin:1.0.0-SNAPSHOT' - } -} -apply plugin: 'elasticsearch.project-settings-attachment' - diff --git a/marvel/build.gradle b/marvel/build.gradle index 245f21fc837..df4f9675ed1 100644 --- a/marvel/build.gradle +++ b/marvel/build.gradle @@ -17,9 +17,9 @@ ext.versions = [ ] dependencies { - licensePluginZip project(path: "${projectsPrefix}:license:plugin") // zip - provided project(path: "${projectsPrefix}:license:plugin", configuration: 'runtime') - provided project(path: "${projectsPrefix}:shield", configuration: 'runtime') + licensePluginZip project(path: ':x-plugins:license:plugin') // zip + provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') + provided project(path: ':x-plugins:shield', configuration: 'runtime') testCompile 'org.elasticsearch:securemock:1.1' // mock web server diff --git a/settings.gradle b/settings.gradle deleted file mode 100644 index 666256810f7..00000000000 --- a/settings.gradle +++ /dev/null @@ -1,50 +0,0 @@ -rootProject.name = 'x-plugins' -apply plugin: 'elasticsearch.project-settings-attachment' - -String prefix = '' -if (rootProject.children.isEmpty() == false) { - // we are attached to another project, make a fake root - rootProject.name = 'x-plugins and attachments' - rootProject.projectDir = new File(settingsDir, 'build/fake-root') - if (rootProject.projectDir.exists()) { - rootProject.projectDir.delete() - } - rootProject.projectDir.mkdirs() - - // the fake root's build file just needs to know about attachments - File rootBuild = new File(rootProject.projectDir, 'build.gradle') - rootBuild.setText('apply plugin: "elasticsearch.project-attachment"', 'UTF-8') - String attachPrefix = org.elasticsearch.gradle.attachment.ProjectAttachmentPlugin.ATTACHMENT_PREFIX - settingsDir.eachFile { file -> - if (file.getName().startsWith(org.elasticsearch.gradle.attachment.ProjectAttachmentPlugin.ATTACHMENT_PREFIX)) { - new File(rootProject.projectDir, file.getName()).setText(file.getText('UTF-8'), 'UTF-8') - } - } - - // add x-plugins as an attachment - File xpluginsAttachment = new File(rootProject.projectDir, "${attachPrefix}x-plugins") - xpluginsAttachment.setText(settingsDir.getPath(), 'UTF-8') - - // and add x-plugins root project - prefix = 'x-plugins:' - include 'x-plugins' - project(':x-plugins').projectDir = settingsDir -} - -String[] projects = [ - 'license:core2', - 'license:licensor', - 'license:plugin-api', - 'license:plugin', - 'license:found-plugin', - 'shield', - 'watcher', - 'marvel' -] -if (prefix.isEmpty() == false) { - projects = projects.collect { "${prefix}${it}" } -} -include projects - -project(":${prefix}license:core2").projectDir = new File(project(":${prefix}license").projectDir, 'core') - diff --git a/shield/build.gradle b/shield/build.gradle index b3c7d085834..c314b390739 100644 --- a/shield/build.gradle +++ b/shield/build.gradle @@ -11,9 +11,9 @@ configurations { } dependencies { - licensePluginZip project(path: "${projectsPrefix}:license:plugin") // zip - provided project(path: "${projectsPrefix}:license:plugin", configuration: 'runtime') - compile project("${projectsPrefix}:license:plugin-api") + licensePluginZip project(path: ':x-plugins:license:plugin') // zip + provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') + compile project(':x-plugins:license:plugin-api') compile 'dk.brics.automaton:automaton:1.11-8' compile 'com.unboundid:unboundid-ldapsdk:2.3.8' testCompile 'org.slf4j:slf4j-log4j12:1.6.2' diff --git a/watcher/build.gradle b/watcher/build.gradle index 01eca7c392c..06b53faea89 100644 --- a/watcher/build.gradle +++ b/watcher/build.gradle @@ -17,9 +17,9 @@ ext.versions = [ ] dependencies { - licensePluginZip project(path: "${projectsPrefix}:license:plugin") // zip - provided project(path: "${projectsPrefix}:license:plugin", configuration: 'runtime') - provided project(path: "${projectsPrefix}:shield", configuration: 'runtime') + licensePluginZip project(path: ':x-plugins:license:plugin') // zip + provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') + provided project(path: ':x-plugins:shield', configuration: 'runtime') compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239' compile 'com.google.guava:guava:19.0-rc2' From f1a9b50e9e066bd580cf77c5ee995f1547d26ceb Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sat, 21 Nov 2015 23:01:37 -0500 Subject: [PATCH 31/95] Ban write access to system properties Followup to https://github.com/elastic/elasticsearch/pull/14914 Shield has to request read-write access to all system properties due to silliness in UnboundID sdk (https://github.com/UnboundID/ldapsdk/blob/556a203094a4239e0ad09c3acaa0704f21f5aa23/src/com/unboundid/util/Debug.java#L166) We should followup with a pull request to them, to not use System.getProperties() here which returns a mutable map (hence: read-write to "*"). Furthermore, the hack has to be wrapped in another hack because gradle doesn't add shield's plugin metadata to the classpath. Of course, if we weren't testing with two plugins in the classpath (which is not very realistic) this would be a non-issue. Original commit: elastic/x-pack-elasticsearch@612cacde6a37bc1a86078cc8ae9df58a1c440ef3 --- .../elasticsearch/shield/ShieldPlugin.java | 32 +++++++++++++++++++ .../plugin-metadata/plugin-security.policy | 4 +++ .../org/elasticsearch/watcher/WatcherF.java | 2 ++ .../bench/WatcherScheduleEngineBenchmark.java | 2 ++ 4 files changed, 40 insertions(+) create mode 100644 shield/src/main/plugin-metadata/plugin-security.policy diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java index db911ea1e27..8d267678831 100644 --- a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java +++ b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.shield; +import org.elasticsearch.SpecialPermission; import org.elasticsearch.action.ActionModule; import org.elasticsearch.client.Client; import org.elasticsearch.client.support.Headers; @@ -53,6 +54,8 @@ import org.elasticsearch.transport.TransportModule; import java.io.Closeable; import java.nio.file.Path; import java.util.*; +import java.security.AccessController; +import java.security.PrivilegedAction; /** * @@ -71,6 +74,35 @@ public class ShieldPlugin extends Plugin { private final boolean clientMode; private ShieldLicenseState shieldLicenseState; + // TODO: clean up this library to not ask for write access to all system properties! + static { + // invoke this clinit in unbound with permissions to access all system properties + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + try { + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Void run() { + try { + Class.forName("com.unboundid.util.Debug"); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + return null; + } + }); + // TODO: fix gradle to add all shield resources (plugin metadata) to test classpath + // of watcher plugin, which depends on it directly. This prevents these plugins + // from being initialized correctly by the test framework, and means we have to + // have this leniency. + } catch (ExceptionInInitializerError bogus) { + if (bogus.getCause() instanceof SecurityException == false) { + throw bogus; // some other bug + } + } + } public ShieldPlugin(Settings settings) { this.settings = settings; diff --git a/shield/src/main/plugin-metadata/plugin-security.policy b/shield/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 00000000000..b104d276aae --- /dev/null +++ b/shield/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,4 @@ +grant { + // needed because of problems in unbound LDAP library + permission java.util.PropertyPermission "*", "read,write"; +}; diff --git a/watcher/src/test/java/org/elasticsearch/watcher/WatcherF.java b/watcher/src/test/java/org/elasticsearch/watcher/WatcherF.java index 25032ff0b63..e651477eb8a 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/WatcherF.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/WatcherF.java @@ -7,6 +7,7 @@ package org.elasticsearch.watcher; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.license.plugin.LicensePlugin; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; @@ -23,6 +24,7 @@ import java.util.concurrent.CountDownLatch; */ public class WatcherF { + @SuppressForbidden(reason = "not really code or a test") public static void main(String[] args) throws Throwable { Settings.Builder settings = Settings.builder(); settings.put("http.cors.enabled", "true"); diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java b/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java index 0c3361a2b93..f4c2c1bd8bb 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; @@ -63,6 +64,7 @@ public class WatcherScheduleEngineBenchmark { .put("http.cors.enabled", true) .build(); + @SuppressForbidden(reason = "not really code or a test") public static void main(String[] args) throws Exception { System.setProperty("es.logger.prefix", ""); From b752b79a299504029218392d814a29d9a4785836 Mon Sep 17 00:00:00 2001 From: uboness Date: Sun, 22 Nov 2015 21:21:54 +0100 Subject: [PATCH 32/95] [docs] fixed `chain` input example Original commit: elastic/x-pack-elasticsearch@6ac4ac4ac9216d3e4b38969d964cf24732587bf8 --- watcher/docs/reference/input/chain.asciidoc | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/watcher/docs/reference/input/chain.asciidoc b/watcher/docs/reference/input/chain.asciidoc index 8340154d8c5..900c25f419c 100644 --- a/watcher/docs/reference/input/chain.asciidoc +++ b/watcher/docs/reference/input/chain.asciidoc @@ -17,15 +17,19 @@ For example, the following chain input loads data from an HTTP server using the "input" : { "chain" : { "inputs" : [ <1> - "first" : { - "simple" : { "path" : "/_search" } + { + "first" : { + "simple" : { "path" : "/_search" } + } }, - "second" : { - "http" : { - "request" : { - "host" : "localhost", - "port" : 9200, - "path" : "{{ctx.payload.first.path}}" <2> + { + "second" : { + "http" : { + "request" : { + "host" : "localhost", + "port" : 9200, + "path" : "{{ctx.payload.first.path}}" <2> + } } } } From 0a78332c3ed1f0e1f6eb3c74c7bb0cfacb6e07fa Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 23 Nov 2015 00:09:47 -0800 Subject: [PATCH 33/95] Build: Remove leftover ant and maven files A number of unused files still exist for maven assemblies and ant integration test overrides. This change removes them. There are still some files left which need to be examined more, for instance, the shield overrides ant file. Original commit: elastic/x-pack-elasticsearch@1d9b277ff04916c564ea8215ff1de87557689d89 --- marvel/dev-tools/integration-tests.xml | 11 ----- marvel/src/main/assemblies/plugin.xml | 50 ---------------------- shield/dev-tools/integration-tests.xml | 42 ------------------ shield/src/main/assemblies/plugin.xml | 46 -------------------- watcher/dev-tools/integration-tests.xml | 11 ----- watcher/src/main/assemblies/plugin.xml | 57 ------------------------- 6 files changed, 217 deletions(-) delete mode 100644 marvel/dev-tools/integration-tests.xml delete mode 100644 marvel/src/main/assemblies/plugin.xml delete mode 100644 shield/dev-tools/integration-tests.xml delete mode 100644 shield/src/main/assemblies/plugin.xml delete mode 100644 watcher/dev-tools/integration-tests.xml delete mode 100644 watcher/src/main/assemblies/plugin.xml diff --git a/marvel/dev-tools/integration-tests.xml b/marvel/dev-tools/integration-tests.xml deleted file mode 100644 index fca64eebcbe..00000000000 --- a/marvel/dev-tools/integration-tests.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - diff --git a/marvel/src/main/assemblies/plugin.xml b/marvel/src/main/assemblies/plugin.xml deleted file mode 100644 index 2d4209a09af..00000000000 --- a/marvel/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - - plugin - - zip - - false - - - / - true - false - - org.elasticsearch.plugin:marvel-agent - - - - - - LICENSE.txt - / - - - NOTICE.txt - / - - - ${elasticsearch.tools.directory}/plugin-metadata/plugin-descriptor.properties - - true - - - diff --git a/shield/dev-tools/integration-tests.xml b/shield/dev-tools/integration-tests.xml deleted file mode 100644 index 42a11f74b18..00000000000 --- a/shield/dev-tools/integration-tests.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - Waiting for elasticsearch to become available on port @{port}... - - - - - - - - - - - - - - - - - - - - - - - Checking we can connect with basic auth on port ${integ.http.port}... - - - - - diff --git a/shield/src/main/assemblies/plugin.xml b/shield/src/main/assemblies/plugin.xml deleted file mode 100644 index c6d7b362671..00000000000 --- a/shield/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - plugin - - zip - - false - - - false - bin/shield - bin - - - config/shield - config - - - - - / - true - false - - org.elasticsearch.plugin:shield - dk.brics.automaton:automaton - com.unboundid:unboundid-ldapsdk - - - - - - LICENSE.txt - / - - - NOTICE.txt - / - - - ${elasticsearch.tools.directory}/plugin-metadata/plugin-descriptor.properties - - true - - - diff --git a/watcher/dev-tools/integration-tests.xml b/watcher/dev-tools/integration-tests.xml deleted file mode 100644 index fca64eebcbe..00000000000 --- a/watcher/dev-tools/integration-tests.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - diff --git a/watcher/src/main/assemblies/plugin.xml b/watcher/src/main/assemblies/plugin.xml deleted file mode 100644 index cc0b2676d32..00000000000 --- a/watcher/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - plugin - - zip - - false - - - false - bin/watcher - bin - - - ${project.basedir}/src/main/plugin-metadata - - plugin-security.policy - - - false - - - - - / - true - true - - org.elasticsearch:elasticsearch - - - - / - false - - com.sun.mail:javax.mail - javax.activation:activation - com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer - - - - - - ${elasticsearch.tools.directory}/plugin-metadata/plugin-descriptor.properties - - true - - - LICENSE.txt - / - - - NOTICE.txt - / - - - From 0a009e355cbef2bedef76ddde3bc67a4393a5370 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Mon, 23 Nov 2015 11:01:10 +0000 Subject: [PATCH 34/95] [TEST] Provides generics for action in ESUsersRealmTests Although the build passes on the command line the Eclipse compiler complains that the client.execute() call on line 216 does not have correct arguments because of the lack of generics. This changes adds the generics to the action variable to solve the error in Eclipse. The change is very low risk and should not adversely affect the build on the command line nor in intelliJ IDEA Original commit: elastic/x-pack-elasticsearch@08f354855651693d896e8d5850eafa17a8a74000 --- .../elasticsearch/shield/authc/esusers/ESUsersRealmTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/esusers/ESUsersRealmTests.java b/shield/src/test/java/org/elasticsearch/shield/authc/esusers/ESUsersRealmTests.java index e469d375047..50481a15240 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authc/esusers/ESUsersRealmTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authc/esusers/ESUsersRealmTests.java @@ -196,7 +196,7 @@ public class ESUsersRealmTests extends ESTestCase { } }; RestRequest restRequest = mock(RestRequest.class); - final Action action = mock(Action.class); + final Action action = mock(Action.class); final ActionListener listener = mock(ActionListener.class); BaseRestHandler handler = new BaseRestHandler(Settings.EMPTY, restController, client) { @Override From 9f6398127c4affec2f04edefc54d3deec8e53685 Mon Sep 17 00:00:00 2001 From: debadair Date: Mon, 23 Nov 2015 10:34:26 -0800 Subject: [PATCH 35/95] Shield Docs: Added release notes for 2.0.1. Original commit: elastic/x-pack-elasticsearch@a35092bc0658ec70308ed588d8f78f53e77c1b35 --- shield/docs/public/release-notes.asciidoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/shield/docs/public/release-notes.asciidoc b/shield/docs/public/release-notes.asciidoc index fab1404968f..46e2c7f1c55 100644 --- a/shield/docs/public/release-notes.asciidoc +++ b/shield/docs/public/release-notes.asciidoc @@ -44,6 +44,12 @@ version of Shield. We recommend copying the changes listed below to your `roles. [[changelist]] === Change List +[float] +==== 2.0.1 + +.Breaking Changes +* <> is now disabled by default. Set `shield.dls_fls.enabled` to `true` in `elasticsearch.yml` to enable it. You cannot submit `_bulk` update requests when document and field level security is enabled. + [float] ==== 2.0.0 From a973cbcd72b989fbff7586c7aaf323722a153d32 Mon Sep 17 00:00:00 2001 From: debadair Date: Mon, 23 Nov 2015 10:46:08 -0800 Subject: [PATCH 36/95] Shield Docs: Added release notes for 2.1. Original commit: elastic/x-pack-elasticsearch@042904968b0233e3c0e946adac0c7499959a0071 --- shield/docs/public/release-notes.asciidoc | 67 +++++++++++++---------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/shield/docs/public/release-notes.asciidoc b/shield/docs/public/release-notes.asciidoc index 46e2c7f1c55..b47bba1d4ad 100644 --- a/shield/docs/public/release-notes.asciidoc +++ b/shield/docs/public/release-notes.asciidoc @@ -30,7 +30,7 @@ On upgrade, your current configuration files will remain untouched. The configur of Shield will be added with a `.new` extension. [float] -==== updated role definitions +==== Updated Role Definitions The default role definitions in the `roles.yml` file may need to be changed to ensure proper functionality with other applications such as Marvel and Kibana. Any role changes will be found in `roles.yml.new` after upgrading to the new version of Shield. We recommend copying the changes listed below to your `roles.yml` file. @@ -44,6 +44,15 @@ version of Shield. We recommend copying the changes listed below to your `roles. [[changelist]] === Change List +[float] +==== 2.1.0 + +.Breaking Changes +* Same as 2.0.1. <> is now disabled by default. Set `shield.dls_fls.enabled` to `true` in `elasticsearch.yml` to enable it. You cannot submit `_bulk` update requests when document and field level security is enabled. + +.Enhancements +* Adds support for Elasticsearch 2.1.0. + [float] ==== 2.0.1 @@ -53,27 +62,27 @@ version of Shield. We recommend copying the changes listed below to your `roles. [float] ==== 2.0.0 -.new features +.Breaking Changes +* All files that Shield uses must be kept in the <> due to the enhanced security of Elasticsearch 2.0. +* The network format has been changed from all previous versions of Shield and a full cluster restart is required to upgrade to Shield 2.0. + +.New Features * <> support has been added and can be configured per role. * Support for <> has been added, allowing Shield to integrate with more authentication sources and methods. * <> has also been added, which allows a user to send a request to elasticsearch that will be run with the specified user's permissions. -.bug fixes +.Bug Fixes * <> now captures requests from nodes using a different system key as tampered requests. * The <> stores the type of request when available. * `esusers` and `syskeygen` work when spaces are in the elasticsearch installation path. * Fixed a rare issue where authentication fails even when the username and password are correct. -.breaking changes -* All files that Shield uses must be kept in the <> due to the enhanced security of Elasticsearch 2.0. -* The network format has been changed from all previous versions of Shield and a full cluster restart is required to upgrade to Shield 2.0. - [float] ==== 1.3.2 -.bug fixes +.Bug Fixes * When using the <> mechanism, connection errors during startup no longer cause the node to stop. * The <> no longer generates invalid JSON. * The <> starts properly when forwarding the audit events to a remote cluster and uses @@ -82,7 +91,7 @@ the correct user to index the audit events. [float] ==== 1.3.1 -.bug fixes +.Bug Fixes * Fixes <> serialization to work with Shield 1.2.1 and earlier. ** NOTE: if you are upgrading from Shield 1.3.0 or Shield 1.2.2 a {ref-17}/setup-upgrade.html#restart-upgrade[cluster restart upgrade] will be necessary. When upgrading from other versions of Shield, follow the normal <>. @@ -90,25 +99,25 @@ will be necessary. When upgrading from other versions of Shield, follow the norm [float] ==== 1.3.0 -.new features -* <>: Adds Public Key Infrastructure (PKI) authentication through the use of X.509 certificates in place of - username and password credentials. -* <>: An index based output has been added for storing audit events in an Elasticsearch index. - -.breaking changes +.Breaking Changes * The `sha2` and `apr1` hashing algorithms have been removed as options for the <>. If your existing Shield installation uses either of these options, remove the setting and use the default `ssha256` algorithm. * The `users` file now only supports `bcrypt` password hashing. All existing passwords stored using the `esusers` tool have been hashed with `bcrypt` and are not affected. -.enhancements +.New Features +* <>: Adds Public Key Infrastructure (PKI) authentication through the use of X.509 certificates in place of + username and password credentials. +* <>: An index based output has been added for storing audit events in an Elasticsearch index. + +.Enhancements * TLS 1.2 is now the default protocol. * Clients that do not support pre-emptive basic authentication can now support both anonymous and authenticated access by specifying the `shield.authc.anonymous.authz_exception` <> with a value of `false`. * Reduced logging for common SSL exceptions, such as a client closing the connection during a handshake. -.bug fixes +.Bug Fixes * The `esusers` and `syskeygen` tools now work correctly with environment variables in the RPM and DEB installation environment files `/etc/sysconfig/elasticsearch` and `/etc/default/elasticsearch`. * Default ciphers no longer include `TLS_DHE_RSA_WITH_AES_128_CBC_SHA`. @@ -116,7 +125,7 @@ will be necessary. When upgrading from other versions of Shield, follow the norm [float] ==== 1.2.3 -.bug fixes +.Bug Fixes * Fixes <> serialization to work with Shield 1.2.1 and earlier. ** NOTE: if you are upgrading from Shield 1.2.2 a {ref-17}/setup-upgrade.html#restart-upgrade[cluster restart upgrade] will be necessary. When upgrading from other versions of Shield, follow the normal <>. @@ -124,7 +133,7 @@ will be necessary. When upgrading from other versions of Shield, follow the norm [float] ==== 1.2.2 -.bug fixes +.Bug Fixes * The `esusers` tool no longer warns about missing roles that are properly defined in the `roles.yml` file. * The period character, `.`, is now allowed in usernames and role names. * The {ref-17}/query-dsl-terms-filter.html#_caching_19[terms filter lookup cache] has been disabled to ensure all requests @@ -136,27 +145,27 @@ will be necessary. When upgrading from other versions of Shield, follow the norm [float] ==== 1.2.1 -.bug fixes -* Several bug fixes including a fix to ensure that {ref-17}/disk.html[Disk-based Shard Allocation] +.Bug Fixes +* Several bug fixes including a fix to ensure that {ref}/disk.html[Disk-based Shard Allocation] works properly with Shield [float] ==== 1.2.0 -.enhancements +.Enhancements * Adds support for Elasticsearch 1.5 [float] ==== 1.1.1 -.bug fixes -* Several bug fixes including a fix to ensure that {ref-17}/disk.html[Disk-based Shard Allocation] +.Bug Fixes +* Several bug fixes including a fix to ensure that {ref}/disk.html[Disk-based Shard Allocation] works properly with Shield [float] ==== 1.1.0 -.new features +.New Features * LDAP: ** Add the ability to bind as a specific user for LDAP searches, which removes the need to specify `user_dn_templates`. This mode of operation also makes use of connection pooling for better performance. Please see <> @@ -167,27 +176,27 @@ for more information. * IP Filtering: ** IP Filtering settings can now be <> using the {ref}/cluster-update-settings.html[Cluster Update Settings API]. -.enhancements +.Enhancements * Significant memory footprint reduction of internal data structures * Test if SSL/TLS ciphers are supported and warn if any of the specified ciphers are not supported * Reduce the amount of logging when a non-encrypted connection is opened and `https` is being used * Added the <>, which is a role that contains the minimum set of permissions required for the Kibana 4 server. * In-memory user credential caching hash algorithm defaults now to salted SHA-256 (see <> -.bug fixes +.Bug Fixes * Filter out sensitive settings from the settings APIs [float] ==== 1.0.2 -.bug fixes +.Bug Fixes * Filter out sensitive settings from the settings APIs * Significant memory footprint reduction of internal data structures [float] ==== 1.0.1 -.bug fixes +.Bug Fixes * Fixed dependency issues with Elasticsearch 1.4.3 and (Lucene 4.10.3 that comes with it) * Fixed bug in how user roles were handled. When multiple roles were defined for a user, and one of the roles only had cluster permissions, not all privileges were properly evaluated. From c365da861f0f8cd7c8fbbdb0fbee2bf4ef4d6ebf Mon Sep 17 00:00:00 2001 From: debadair Date: Mon, 23 Nov 2015 11:19:09 -0800 Subject: [PATCH 37/95] Shield Docs: Added information about enabling DLS/FLS. Original commit: elastic/x-pack-elasticsearch@23f9ad66d474c995750ebd9cbbda249b3da85e94 --- shield/docs/public/reference.asciidoc | 3 ++- .../setting-up-field-and-document-level-security.asciidoc | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/shield/docs/public/reference.asciidoc b/shield/docs/public/reference.asciidoc index cb57571bfdb..65fde4e4f52 100644 --- a/shield/docs/public/reference.asciidoc +++ b/shield/docs/public/reference.asciidoc @@ -188,7 +188,8 @@ The parameters listed in this section are configured in the `config/elasticsearc [options="header"] |====== | Name | Default | Description -| `shield.dls_fls.enabled` | `true` | This setting can be used to completely disable document and field level security regardless of how roles are configured. +| `shield.dls_fls.enabled` | `false` | Set to `true` to enable document and field level security. +You cannot submit `_bulk` update requests when document and field level security is enabled. |====== diff --git a/shield/docs/public/setting-up-field-and-document-level-security.asciidoc b/shield/docs/public/setting-up-field-and-document-level-security.asciidoc index a6b2c7583fe..873693efb1c 100644 --- a/shield/docs/public/setting-up-field-and-document-level-security.asciidoc +++ b/shield/docs/public/setting-up-field-and-document-level-security.asciidoc @@ -8,6 +8,8 @@ Document level security permissions restrict access to particular documents with Field and document level permissions are specified separately, but a role can define both field and document level permissions. Field and document level security permissions can be configured on a per-index basis. +IMPORTANT: Document and Field Level Security is disabled by default. Set `shield.dls_fls.enabled` to `true` in `elasticsearch.yml` to enable it. You cannot submit `_bulk` update requests when document and field level security is enabled. + ==== Field Level Security To enable field level security, you specify the fields that each role can access in the `roles.yml` file. From 90cb7d38b3f988dd9a8aa7b3120e234c9ced00cd Mon Sep 17 00:00:00 2001 From: debadair Date: Mon, 23 Nov 2015 13:15:54 -0800 Subject: [PATCH 38/95] Docs: Added dates to Shield & Watcher release notes. Original commit: elastic/x-pack-elasticsearch@2d42762b84c56333228dfd6798cb133fb64ccb3c --- shield/docs/public/release-notes.asciidoc | 19 ++++++++++++--- watcher/docs/release-notes.asciidoc | 28 +++++++++++++++++------ 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/shield/docs/public/release-notes.asciidoc b/shield/docs/public/release-notes.asciidoc index b47bba1d4ad..00580a970c8 100644 --- a/shield/docs/public/release-notes.asciidoc +++ b/shield/docs/public/release-notes.asciidoc @@ -4,10 +4,9 @@ [float] [[version-compatibility]] === Version Compatibility -Shield {version} is compatible with: -* Elasticsearch: {version} -* License plugin: {version} +You must run the version of Shield that matches the version of Elasticsearch you are running. For +example, Shield {version} requires Elasticsearch {version}. [float] [[upgrade-instructions]] @@ -46,6 +45,7 @@ version of Shield. We recommend copying the changes listed below to your `roles. [float] ==== 2.1.0 +November 24, 2015 .Breaking Changes * Same as 2.0.1. <> is now disabled by default. Set `shield.dls_fls.enabled` to `true` in `elasticsearch.yml` to enable it. You cannot submit `_bulk` update requests when document and field level security is enabled. @@ -55,12 +55,14 @@ version of Shield. We recommend copying the changes listed below to your `roles. [float] ==== 2.0.1 +November 24, 2015 .Breaking Changes * <> is now disabled by default. Set `shield.dls_fls.enabled` to `true` in `elasticsearch.yml` to enable it. You cannot submit `_bulk` update requests when document and field level security is enabled. [float] ==== 2.0.0 +October 28, 2015 .Breaking Changes * All files that Shield uses must be kept in the <> due to the enhanced security of Elasticsearch 2.0. @@ -81,6 +83,7 @@ with the specified user's permissions. [float] ==== 1.3.2 +August 10, 2015 .Bug Fixes * When using the <> mechanism, connection errors during startup no longer cause the node to stop. @@ -90,6 +93,7 @@ the correct user to index the audit events. [float] ==== 1.3.1 +July 21, 2015 .Bug Fixes * Fixes <> serialization to work with Shield 1.2.1 and earlier. @@ -98,6 +102,7 @@ will be necessary. When upgrading from other versions of Shield, follow the norm [float] ==== 1.3.0 +June 24, 2015 .Breaking Changes * The `sha2` and `apr1` hashing algorithms have been removed as options for the <>. @@ -124,6 +129,7 @@ will be necessary. When upgrading from other versions of Shield, follow the norm [float] ==== 1.2.3 +July 21, 2015 .Bug Fixes * Fixes <> serialization to work with Shield 1.2.1 and earlier. @@ -132,6 +138,7 @@ will be necessary. When upgrading from other versions of Shield, follow the norm [float] ==== 1.2.2 +June 24, 2015 .Bug Fixes * The `esusers` tool no longer warns about missing roles that are properly defined in the `roles.yml` file. @@ -144,6 +151,7 @@ will be necessary. When upgrading from other versions of Shield, follow the norm [float] ==== 1.2.1 +April 29, 2015 .Bug Fixes * Several bug fixes including a fix to ensure that {ref}/disk.html[Disk-based Shard Allocation] @@ -151,12 +159,14 @@ works properly with Shield [float] ==== 1.2.0 +March 24, 2015 .Enhancements * Adds support for Elasticsearch 1.5 [float] ==== 1.1.1 +April 29, 2015 .Bug Fixes * Several bug fixes including a fix to ensure that {ref}/disk.html[Disk-based Shard Allocation] @@ -164,6 +174,7 @@ works properly with Shield [float] ==== 1.1.0 +March 24, 2015 .New Features * LDAP: @@ -188,6 +199,7 @@ for more information. [float] ==== 1.0.2 +March 24, 2015 .Bug Fixes * Filter out sensitive settings from the settings APIs @@ -195,6 +207,7 @@ for more information. [float] ==== 1.0.1 +February 13, 2015 .Bug Fixes * Fixed dependency issues with Elasticsearch 1.4.3 and (Lucene 4.10.3 that comes with it) diff --git a/watcher/docs/release-notes.asciidoc b/watcher/docs/release-notes.asciidoc index 9e9f4ed0fa6..7d5948b7e2f 100644 --- a/watcher/docs/release-notes.asciidoc +++ b/watcher/docs/release-notes.asciidoc @@ -5,11 +5,8 @@ [[version-compatibility]] === Version Compatibility -Watcher {version} is compatible with: - -* Elasticsearch: {version} -* License {version} -* Shield {version} +You must run the version of Watcher that matches the version of Elasticsearch you are running. For +example, Watcher {version} requires Elasticsearch {version}. [float] [[upgrade-instructions]] @@ -38,12 +35,14 @@ bin/plugin remove watcher [float] ==== 2.1.0 +November 24, 2015 .New Features -* Added support for <> +* Adds support for <> .Enhancement -* Support for configuring a proxy in the webhook action, http input and configuring a default proxy (which is also used by the slack action), using the `watcher.http.proxy.host` and `watcher.http.proxy.port` settings. +* Adds support for Elasticsearch 2.1.0. +* Adds support for configuring a proxy in the webhook action, http input and configuring a default proxy (which is also used by the slack action), using the `watcher.http.proxy.host` and `watcher.http.proxy.port` settings. .Bug fixes * Fixed an issue where the scheduler may get stuck during Watcher startup. This caused no watches to ever fire. @@ -53,6 +52,7 @@ bin/plugin remove watcher [float] ==== 2.0.1 +November 24, 2015 .Bug fixes * Fixed an issue where under specific conditions Watcher would not start if there are not finished watch executions from the @@ -61,6 +61,7 @@ bin/plugin remove watcher [float] ==== 2.0.0 +October 28, 2015 .Breaking Changes * The dynamic index names support has been removed and Elasticsearch's date math index names support should be used instead. @@ -99,8 +100,19 @@ bin/plugin remove watcher * Fixed an issue where the scheduler may get stuck during Watcher startup. This caused no watches to ever fire. * Fixed url encoding issue in http input and webhook output. The url params were url encoded twice. +[float] +==== 1.0.1 +July 29, 2015 + +.Enhancements +* Dynamic index names now support specifying a time zone to be used when computing the names of the indices. The default is UTC. Previously, the computation was fixed to always use UTC when computing the names of the indices. + +.Bug Fixes +* Fixed a compatibility issue with Elasticsearch 1.6.1 and 1.7.2, which were released earlier today. + [float] ==== 1.0.0 +June 25, 2015 .Enhancements * Added execution time aware dynamic index names support to `index` @@ -113,6 +125,7 @@ bin/plugin remove watcher [float] ==== 1.0.0-rc1 +June 19, 2015 .New Features * Added <> support to the Execute API @@ -125,6 +138,7 @@ bin/plugin remove watcher [float] ==== 1.0.0-Beta2 +June 10, 2015 .New Features * <> are now applied at the action level rather than From 2f4dca744f2f255ceee75414dbdb49df4542b41c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 23 Nov 2015 14:18:07 -0800 Subject: [PATCH 39/95] Build: Change x-plugins setup with elasticsearch to a sibling directory This change is the x-plugins side of elastic/elasticsearchelastic/elasticsearch#14952. It now requires x-plugins to be checked out as a sibling of elasticsearch called extra-plugin-x-plugins. Original commit: elastic/x-pack-elasticsearch@f69b16740783af9d481df3b3ddb9331ff45c3027 --- build.gradle | 10 ++-------- buildSrc/settings.gradle | 6 ++++++ settings.gradle | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 buildSrc/settings.gradle create mode 100644 settings.gradle diff --git a/build.gradle b/build.gradle index 559bf2f9c28..1a8c392fab8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,11 +1,5 @@ -if (project.name != 'x-plugins') { - throw new GradleException('You must checkout x-plugins to a directory named x-plugins') -} -if (project.projectDir.parentFile.name != 'extra-plugins') { - throw new GradleException('You must place the x-plugins checkout in a directory named extra-plugins') -} -if (project.rootProject.projectDir != project.projectDir.parentFile.parentFile) { - throw new GradleException('You must place the extra-plugins directory inside the root of an elasticsearch checkout') +if (project.projectDir.name != 'extra-plugin-x-plugins') { + throw new GradleException('You must checkout x-plugins in a directory named extra-plugin-x-plugins') } /* diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle new file mode 100644 index 00000000000..fede2f180e7 --- /dev/null +++ b/buildSrc/settings.gradle @@ -0,0 +1,6 @@ +File elasticsearchDir = new File(settingsDir, '../../elasticsearch') +if (elasticsearchDir.exists() == false) { + throw new GradleException('Elasticsearch must be checked out as a sibling directory of x-plugins') +} + +project(':').projectDir = new File(elasticsearchDir, 'buildSrc') diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 00000000000..651d9aca508 --- /dev/null +++ b/settings.gradle @@ -0,0 +1,2 @@ +project(':').projectDir = new File('../elasticsearch') +apply from: '../elasticsearch/settings.gradle' From c53390cbdf82a936dab001ed29e2889056f5c0c5 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 23 Nov 2015 14:45:24 -0800 Subject: [PATCH 40/95] Change to x-plugins directory name instead of extra-plugin-x-plugins Original commit: elastic/x-pack-elasticsearch@2b609414b7bdcb7567d08557f55ac9dddd76fae2 --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 1a8c392fab8..38be8951a45 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ -if (project.projectDir.name != 'extra-plugin-x-plugins') { - throw new GradleException('You must checkout x-plugins in a directory named extra-plugin-x-plugins') +if (project.projectDir.name != 'x-plugins') { + throw new GradleException('You must checkout x-plugins in a directory named x-plugins next to elasticsearch') } /* From 415dff083fef1e7de50834a4a4482b97175a38ef Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 23 Nov 2015 19:52:01 -0500 Subject: [PATCH 41/95] Watcher and Shield should depend on same guava version. This causes hellaciousness at least with eclipse, possibly qa tests too. the sanitiser just wants 11+, so make it the same version as shield's (dragged in by jimfs). Also remove unnecessary usage of guava in a test Original commit: elastic/x-pack-elasticsearch@ea9ac88fdc428afb2ea181026c7abaa8b74f28f2 --- watcher/build.gradle | 2 +- .../watcher/transform/search/SearchTransformTests.java | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/watcher/build.gradle b/watcher/build.gradle index 06b53faea89..ef997debfa9 100644 --- a/watcher/build.gradle +++ b/watcher/build.gradle @@ -22,7 +22,7 @@ dependencies { provided project(path: ':x-plugins:shield', configuration: 'runtime') compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239' - compile 'com.google.guava:guava:19.0-rc2' + compile 'com.google.guava:guava:16.0.1' compile 'com.google.code.findbugs:jsr305:3.0.1' compile 'com.sun.mail:javax.mail:1.5.3' compile 'javax.activation:activation:1.1.1' diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java b/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java index 5142007fcfe..0f66815ba2e 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java @@ -5,8 +5,6 @@ */ package org.elasticsearch.watcher.transform.search; -import com.google.common.base.Charsets; - import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -53,6 +51,7 @@ import org.joda.time.chrono.ISOChronology; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -199,7 +198,7 @@ public class SearchTransformTests extends ESIntegTestCase { assertThat(map.get("query"), instanceOf(String.class)); String queryAsBase64 = (String) map.get("query"); - String decodedQuery = new String(Base64.decode(queryAsBase64), Charsets.UTF_8); + String decodedQuery = new String(Base64.decode(queryAsBase64), StandardCharsets.UTF_8); assertThat(decodedQuery, containsString("_unknown_query_")); } From f8ab6f0fb557ea93261b622695f020d7aab36337 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 24 Nov 2015 09:28:28 +0100 Subject: [PATCH 42/95] Fix compilation of ShieldIndexSearcherWrapperUnitTests. The break was introduced in elastic/elasticsearchelastic/elasticsearch#14896. Original commit: elastic/x-pack-elasticsearch@07810b2d2b2b8df798cb13d1968ab172359b4259 --- .../accesscontrol/ShieldIndexSearcherWrapperUnitTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java index 785b7e20ed3..5177e10e701 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.mapper.internal.ParentFieldMapper; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.shield.authz.InternalAuthorizationService; @@ -67,7 +68,7 @@ public class ShieldIndexSearcherWrapperUnitTests extends ESTestCase { AnalysisService analysisService = new AnalysisService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap()); - mapperService = new MapperService(indexSettings, analysisService, similarityService); + mapperService = new MapperService(indexSettings, analysisService, similarityService, new IndicesModule().getMapperRegistry()); ShardId shardId = new ShardId(index, 0); licenseState = mock(ShieldLicenseState.class); From 9da4b6160caa068b81be07c8263101ba6c2dbd1d Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 24 Nov 2015 00:45:06 -0800 Subject: [PATCH 43/95] Build: Get shield qa test with core rest tests working This adds back the shield qa rest tests module with gradle. There is also a small fix in ShieldPlugin for a bug that was discovered around checking for a custom query cache (which was using the node settings instead of index settings). Original commit: elastic/x-pack-elasticsearch@28c6d58f3755eae76c23ba5a2a984a1b1017d29d --- marvel/build.gradle | 2 +- qa/build.gradle | 0 qa/shield-core-rest-tests/build.gradle | 56 +++++++++++++++++++ .../integration-tests.xml | 8 --- .../elasticsearch/shield/ShieldPlugin.java | 2 +- 5 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 qa/build.gradle create mode 100644 qa/shield-core-rest-tests/build.gradle delete mode 100644 qa/shield-core-rest-tests/integration-tests.xml diff --git a/marvel/build.gradle b/marvel/build.gradle index df4f9675ed1..2abc375e372 100644 --- a/marvel/build.gradle +++ b/marvel/build.gradle @@ -35,7 +35,7 @@ compileTestJava.options.compilerArgs << '-Xlint:-rawtypes,-unchecked' ext.expansions = [ 'project.version': version, - 'integ.http.port': integTest.cluster.httpPort + 'integ.http.port': integTest.cluster.baseHttpPort ] processResources { diff --git a/qa/build.gradle b/qa/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/shield-core-rest-tests/build.gradle b/qa/shield-core-rest-tests/build.gradle new file mode 100644 index 00000000000..36b3b67473c --- /dev/null +++ b/qa/shield-core-rest-tests/build.gradle @@ -0,0 +1,56 @@ +apply plugin: 'elasticsearch.rest-test' + +configurations { + licensePluginZip + shieldPluginZip +} + +dependencies { + licensePluginZip project(path: ':x-plugins:license:plugin') + shieldPluginZip project(path: ':x-plugins:shield') + testCompile project(path: ':x-plugins:shield', configuration: 'runtime') +} + +integTest { + dependsOn configurations.licensePluginZip, configurations.shieldPluginZip + includePackaged true + systemProperty 'tests.rest.blacklist', + ['indices.get/10_basic/*allow_no_indices*', + 'cat.count/10_basic/Test cat count output', + 'cat.aliases/10_basic/Empty cluster', + 'indices.segments/10_basic/no segments test', + 'indices.clear_cache/10_basic/clear_cache test', + 'indices.status/10_basic/Indices status test', + 'cat.indices/10_basic/Test cat indices output', + 'cat.recovery/10_basic/Test cat recovery output', + 'cat.shards/10_basic/Test cat shards output', + 'termvector/20_issue7121/*', + 'index/10_with_id/Index with ID', + 'indices.get_alias/20_emtpy/*', + 'cat.segments/10_basic/Test cat segments output', + 'indices.put_settings/10_basic/Test indices settings allow_no_indices', + 'indices.put_settings/10_basic/Test indices settings ignore_unavailable', + 'indices.refresh/10_basic/Indices refresh test no-match wildcard', + 'indices.stats/10_index/Index - star*', + 'indices.recovery/10_basic/Indices recovery test*', + 'indices.shard_stores/10_basic/no indices test', + 'cat.nodeattrs/10_basic/Test cat nodes attrs output', + 'bulk/40_fields/Fields'].join(',') + + cluster { + plugin 'license', configurations.licensePluginZip + plugin 'shield', configurations.shieldPluginZip + setupCommand 'setupDummyUser', + 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' + waitCondition = { node, ant -> + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://localhost:${node.httpPort()}", + dest: tmpFile.toString(), + username: 'test_user', + password: 'changeme', + ignoreerrors: true) + return tmpFile.exists() + } + } +} + diff --git a/qa/shield-core-rest-tests/integration-tests.xml b/qa/shield-core-rest-tests/integration-tests.xml deleted file mode 100644 index 4da32222a8f..00000000000 --- a/qa/shield-core-rest-tests/integration-tests.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java index 8d267678831..095b971ccd1 100644 --- a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java +++ b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java @@ -182,7 +182,7 @@ public class ShieldPlugin extends Plugin { @Override public void onIndexService(IndexService indexService) { if (enabled && clientMode == false) { - failIfShieldQueryCacheIsNotActive(settings, false); + failIfShieldQueryCacheIsNotActive(indexService.getIndexSettings().getSettings(), false); } } From c1908942283b550356d07ea05a445d3314731e44 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 18 Nov 2015 18:57:55 +0100 Subject: [PATCH 44/95] Marvel: Update Node Stats with os.cpu.load_average Original commit: elastic/x-pack-elasticsearch@fa8ed35105c26b801d06d2e6d57a0dd41b027836 --- .../marvel/agent/renderer/node/NodeStatsRenderer.java | 2 +- marvel/src/main/resources/marvel_index_template.json | 3 +++ marvel/src/test/resources/samples/node_stats.json | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRenderer.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRenderer.java index 77ed4173363..3c5d63efbb5 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRenderer.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRenderer.java @@ -36,7 +36,7 @@ public class NodeStatsRenderer extends AbstractRenderer { "node_stats.fs.total.total_in_bytes", "node_stats.fs.total.free_in_bytes", "node_stats.fs.total.available_in_bytes", - "node_stats.os.load_average", + "node_stats.os.cpu.load_average", "node_stats.process.cpu.percent", "node_stats.process.max_file_descriptors", "node_stats.process.open_file_descriptors", diff --git a/marvel/src/main/resources/marvel_index_template.json b/marvel/src/main/resources/marvel_index_template.json index 30fb5121bae..c6cb56e3421 100644 --- a/marvel/src/main/resources/marvel_index_template.json +++ b/marvel/src/main/resources/marvel_index_template.json @@ -241,6 +241,9 @@ "fs": { "type": "object" }, + "os": { + "type": "object" + }, "process": { "type": "object" }, diff --git a/marvel/src/test/resources/samples/node_stats.json b/marvel/src/test/resources/samples/node_stats.json index 7a8161cd97a..590c57e511c 100644 --- a/marvel/src/test/resources/samples/node_stats.json +++ b/marvel/src/test/resources/samples/node_stats.json @@ -29,7 +29,9 @@ } }, "os": { - "load_average" : 0.66 + "cpu": { + "load_average": 0.66 + } }, "process": { "open_file_descriptors": 235, From 01383849816b366b467429b74e9b992cfd764f77 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 24 Nov 2015 12:44:15 +0100 Subject: [PATCH 45/95] Marvel: Unmute nodes stats tests Original commit: elastic/x-pack-elasticsearch@9244d6e15a603499d2800e1b933d4b1d73de302f --- .../marvel/agent/renderer/node/MultiNodesStatsTests.java | 4 +--- .../marvel/agent/renderer/node/NodeStatsRendererTests.java | 2 -- .../marvel/agent/renderer/node/NodeStatsTests.java | 2 -- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java index 27602daeee9..5d6ebfe54b5 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.marvel.agent.renderer.node; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; @@ -19,9 +18,8 @@ import org.junit.After; import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.greaterThan; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/pull/994") @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class MultiNodesStatsTests extends MarvelIntegTestCase { diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRendererTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRendererTests.java index e2b13ee1e1d..582fc70042c 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRendererTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRendererTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.marvel.agent.renderer.node; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.marvel.agent.collector.node.NodeStatsMarvelDoc; import org.elasticsearch.marvel.agent.renderer.Renderer; @@ -14,7 +13,6 @@ import org.elasticsearch.node.service.NodeService; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.StreamsUtils; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/pull/994") public class NodeStatsRendererTests extends ESSingleNodeTestCase { private static final String SAMPLE_FILE = "/samples/node_stats.json"; diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java index de16323b092..d63b4d62d24 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.marvel.agent.renderer.node; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector; @@ -22,7 +21,6 @@ import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.greaterThan; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/pull/994") // numClientNodes is set to 0 because Client nodes don't have Filesystem stats @ClusterScope(scope = Scope.TEST, numClientNodes = 0, transportClientRatio = 0.0) public class NodeStatsTests extends MarvelIntegTestCase { From d9363085c3599bb6c286fabfcaf5f29ad7d8a0e1 Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 23 Nov 2015 14:15:37 -0500 Subject: [PATCH 46/95] docs: add Shield 1.3.3 release notes Original commit: elastic/x-pack-elasticsearch@69a525677b3f71064a3d0887b1c628ca8a4add81 --- shield/docs/public/release-notes.asciidoc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/shield/docs/public/release-notes.asciidoc b/shield/docs/public/release-notes.asciidoc index 00580a970c8..e650f1afd62 100644 --- a/shield/docs/public/release-notes.asciidoc +++ b/shield/docs/public/release-notes.asciidoc @@ -81,6 +81,16 @@ with the specified user's permissions. * `esusers` and `syskeygen` work when spaces are in the elasticsearch installation path. * Fixed a rare issue where authentication fails even when the username and password are correct. +[float] +==== 1.3.3 + +.Bug Fixes +* Fixed a rare issue where authentication fails even when the username and password are correct. +* The <> stores the type of request when available. + +.Enhancements +* Tampered requests with a bad header are now audited. + [float] ==== 1.3.2 August 10, 2015 From b0114903207d51fb8facc982b7a99eee9e639ca6 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 24 Nov 2015 08:24:57 -0800 Subject: [PATCH 47/95] Add more retries for wait condition, in case jenkins is slow Original commit: elastic/x-pack-elasticsearch@04e5648cd90a5f78bda5f9719aab704564098206 --- qa/shield-core-rest-tests/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qa/shield-core-rest-tests/build.gradle b/qa/shield-core-rest-tests/build.gradle index 36b3b67473c..926d18bb2a1 100644 --- a/qa/shield-core-rest-tests/build.gradle +++ b/qa/shield-core-rest-tests/build.gradle @@ -48,7 +48,8 @@ integTest { dest: tmpFile.toString(), username: 'test_user', password: 'changeme', - ignoreerrors: true) + ignoreerrors: true, + retries: 10) return tmpFile.exists() } } From 9b32a4e7c4ed61aa082c675ad223e29580582bfd Mon Sep 17 00:00:00 2001 From: debadair Date: Tue, 24 Nov 2015 09:45:07 -0800 Subject: [PATCH 48/95] Docs: Updated offline install links to show version numbers. Original commit: elastic/x-pack-elasticsearch@a00efeea1c8dd1afd2f7fb8460d49295c12dd6dc --- watcher/docs/index.asciidoc | 2 +- watcher/docs/installing-watcher.asciidoc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/watcher/docs/index.asciidoc b/watcher/docs/index.asciidoc index 58feeef43fc..e9c54285b92 100644 --- a/watcher/docs/index.asciidoc +++ b/watcher/docs/index.asciidoc @@ -7,7 +7,7 @@ :java-client-ref: http://www.elastic.co/guide/en/elasticsearch/client/java-api/master :blog-ref: https://www.elastic.co/blog/ :forum: https://discuss.elastic.co/c/watcher -:version: 2.0.0-beta2 +:version: 2.1.0 include::introduction.asciidoc[] diff --git a/watcher/docs/installing-watcher.asciidoc b/watcher/docs/installing-watcher.asciidoc index 624d693a8ca..64091a4e9e4 100644 --- a/watcher/docs/installing-watcher.asciidoc +++ b/watcher/docs/installing-watcher.asciidoc @@ -98,8 +98,8 @@ https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/watch + [source,shell] ---------------------------------------------------------- -bin/plugin install file:///path/to/file/license-2.0.0.zip <1> -bin/plugin install file:///path/to/file/watcher-2.0.0.zip +bin/plugin install file:///path/to/file/license-2.1.0.zip <1> +bin/plugin install file:///path/to/file/watcher-2.1.0.zip ---------------------------------------------------------- <1> Note that you must specify an absolute path to the zip file after the `file://` protocol. From 59b4177b95731d7c89552c56c0bf2a73c0591ecf Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 24 Nov 2015 14:39:16 -0500 Subject: [PATCH 49/95] Mark the slowest slowest tests @BadApple See elastic/elasticsearch#1007 Original commit: elastic/x-pack-elasticsearch@87ebfb1eabb1d173933045e6144d83a00c8f0e00 --- .../agent/collector/cluster/ClusterInfoCollectorTests.java | 3 +++ .../agent/collector/cluster/ClusterStatsCollectorTests.java | 3 +++ .../marvel/agent/renderer/cluster/ClusterStateTests.java | 3 +++ .../marvel/agent/renderer/shards/ShardsTests.java | 3 +++ .../marvel/agent/settings/MarvelSettingsTests.java | 4 ++++ .../org/elasticsearch/integration/IndexPrivilegeTests.java | 3 +++ .../elasticsearch/integration/PermissionPrecedenceTests.java | 3 +++ .../shield/audit/index/IndexAuditTrailEnabledTests.java | 3 +++ .../shield/audit/index/IndexAuditTrailTests.java | 3 +++ .../authc/support/CachingUsernamePasswordRealmTests.java | 3 +++ .../watcher/test/integration/NoMasterNodeTests.java | 3 +++ .../watcher/transport/action/ack/WatchAckTests.java | 3 +++ .../action/execute/ExecuteWatchWithDateMathTests.java | 3 +++ .../trigger/schedule/engine/SchedulerScheduleEngineTests.java | 4 ++++ .../trigger/schedule/engine/TickerScheduleEngineTests.java | 4 ++++ 15 files changed, 48 insertions(+) diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollectorTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollectorTests.java index a8bee4baa90..f0c457e6d8a 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollectorTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollectorTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.marvel.agent.collector.cluster; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -22,6 +23,8 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") public class ClusterInfoCollectorTests extends AbstractCollectorTestCase { public void testClusterInfoCollector() throws Exception { Collection results = newClusterInfoCollector().doCollect(); diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollectorTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollectorTests.java index 6cd50a6741e..80b9cd4f043 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollectorTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollectorTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.marvel.agent.collector.cluster; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase; @@ -19,6 +20,8 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") public class ClusterStatsCollectorTests extends AbstractCollectorTestCase { public void testClusterStatsCollector() throws Exception { Collection results = newClusterStatsCollector().doCollect(); diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java index 2405147adac..94cbe3e09f2 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.marvel.agent.renderer.cluster; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; @@ -27,6 +28,8 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.core.Is.is; +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") @ClusterScope(scope = Scope.TEST) public class ClusterStateTests extends MarvelIntegTestCase { @Override diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsTests.java index 45aefddace0..0a5a65579d0 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.marvel.agent.renderer.shards; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -30,6 +31,8 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") @ClusterScope(scope = Scope.TEST) public class ShardsTests extends MarvelIntegTestCase { private static final String INDEX_PREFIX = "test-shards-"; diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingsTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingsTests.java index 525f94b90d5..f0711a5731a 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingsTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingsTests.java @@ -16,6 +16,10 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import org.apache.lucene.util.LuceneTestCase.BadApple; + +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1, numClientNodes = 0) public class MarvelSettingsTests extends MarvelIntegTestCase { private final TimeValue interval = newRandomTimeValue(); diff --git a/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java b/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java index 56d38509316..6be87088adf 100644 --- a/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.integration; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.Node; import org.elasticsearch.test.rest.client.http.HttpResponse; @@ -17,6 +18,8 @@ import java.util.Map; import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.is; +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { private String jsonDoc = "{ \"name\" : \"elasticsearch\"}"; diff --git a/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java b/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java index 26660d39673..8873726dfa0 100644 --- a/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.integration; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; @@ -30,6 +31,8 @@ import static org.hamcrest.Matchers.hasSize; * actions that are normally categorized as index actions as cluster actions - for example, * index template actions. */ +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") public class PermissionPrecedenceTests extends ShieldIntegTestCase { protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("test123".toCharArray()))); diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailEnabledTests.java b/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailEnabledTests.java index c4ae5f2261d..41c24e4b038 100644 --- a/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailEnabledTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailEnabledTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.shield.audit.index; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.search.SearchResponse; @@ -22,6 +23,8 @@ import java.util.Set; import static org.hamcrest.Matchers.is; +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") @ClusterScope(scope = Scope.TEST, randomDynamicTemplates = false) public class IndexAuditTrailEnabledTests extends ShieldIntegTestCase { IndexNameResolver.Rollover rollover = randomFrom(IndexNameResolver.Rollover.values()); diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java b/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java index e60bbbe1664..78214265f81 100644 --- a/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.shield.audit.index; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -58,6 +59,8 @@ import static org.mockito.Mockito.*; /** * */ +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") @ESIntegTestCase.ClusterScope(scope = SUITE, numDataNodes = 1) public class IndexAuditTrailTests extends ShieldIntegTestCase { public static final String SECOND_CLUSTER_NODE_PREFIX = "remote_" + SUITE_CLUSTER_NODE_PREFIX; diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java b/shield/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java index 63e55e264da..ed880eb29f9 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.shield.authc.support; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.shield.User; @@ -24,6 +25,8 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") public class CachingUsernamePasswordRealmTests extends ESTestCase { private Settings globalSettings; diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/NoMasterNodeTests.java b/watcher/src/test/java/org/elasticsearch/watcher/test/integration/NoMasterNodeTests.java index df6126aeec7..a33add93404 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/NoMasterNodeTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/test/integration/NoMasterNodeTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.watcher.test.integration; import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.Client; @@ -52,6 +53,8 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.core.Is.is; +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") @TestLogging("discovery:TRACE,watcher:TRACE") @ClusterScope(scope = TEST, numClientNodes = 0, transportClientRatio = 0, randomDynamicTemplates = false, numDataNodes = 0) @SuppressLocalMode diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/ack/WatchAckTests.java b/watcher/src/test/java/org/elasticsearch/watcher/transport/action/ack/WatchAckTests.java index 13de098821f..a1f9a1a7a2c 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/ack/WatchAckTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/transport/action/ack/WatchAckTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.watcher.transport.action.ack; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; @@ -48,6 +49,8 @@ import static org.hamcrest.core.IsEqual.equalTo; /** */ +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") public class WatchAckTests extends AbstractWatcherIntegrationTestCase { private IndexResponse indexTestDoc() { createIndex("actions", "events"); diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchWithDateMathTests.java b/watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchWithDateMathTests.java index 81f0b174caf..3faa7e31adc 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchWithDateMathTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchWithDateMathTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.watcher.transport.action.execute; +import org.apache.lucene.util.LuceneTestCase.BadApple; import org.elasticsearch.watcher.client.WatcherClient; import org.elasticsearch.watcher.execution.Wid; import org.elasticsearch.watcher.support.WatcherDateTimeUtils; @@ -29,6 +30,8 @@ import static org.hamcrest.Matchers.notNullValue; /** * */ +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") public class ExecuteWatchWithDateMathTests extends AbstractWatcherIntegrationTestCase { @Override protected boolean timeWarped() { diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleEngineTests.java b/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleEngineTests.java index a5bdcafcf22..2ff9f3ee843 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleEngineTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleEngineTests.java @@ -12,6 +12,10 @@ import org.elasticsearch.watcher.trigger.schedule.ScheduleRegistry; import static org.mockito.Mockito.mock; +import org.apache.lucene.util.LuceneTestCase.BadApple; + +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") public class SchedulerScheduleEngineTests extends BaseTriggerEngineTestCase { protected TriggerEngine createEngine() { diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java b/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java index 8be9366585c..1b37414c4c0 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java @@ -12,8 +12,12 @@ import org.elasticsearch.watcher.trigger.schedule.ScheduleRegistry; import static org.mockito.Mockito.mock; +import org.apache.lucene.util.LuceneTestCase.BadApple; + /** */ +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") public class TickerScheduleEngineTests extends BaseTriggerEngineTestCase { @Override From ec3ba81be36ce0323d81dff1e24a121ee2f4505b Mon Sep 17 00:00:00 2001 From: uboness Date: Tue, 24 Nov 2015 21:02:28 +0100 Subject: [PATCH 50/95] Create GRADLE.CHEATSHEET Original commit: elastic/x-pack-elasticsearch@d5bbad8873388d6db027f4ef5ce86807289fda3b --- GRADLE.CHEATSHEET | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 GRADLE.CHEATSHEET diff --git a/GRADLE.CHEATSHEET b/GRADLE.CHEATSHEET new file mode 100644 index 00000000000..d3f433e25c4 --- /dev/null +++ b/GRADLE.CHEATSHEET @@ -0,0 +1,9 @@ +As a quick helper, below are the equivalent commands from maven to gradle. You can also run `gradle tasks` to see all tasks that are available to run. + +| Maven | Gradle | Description | +| ----------------------------| ------------|---------------------| +| `clean` | `clean` | | +| `verify` | `check` | | +| `verify -Dskip.unit.tests` | `integTest` | | +| `package -DskipTests` | `assemble` | | +| `install -DskipTests` | `install` | | From 4f44ccedb5d375fde6a9eda361ab98539dc9e1c1 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 24 Nov 2015 12:54:11 -0800 Subject: [PATCH 51/95] Build: Simplify plugin installs for integTests This is the xplugins side of elastic/elasticsearchelastic/elasticsearch#14986, making use of the simplification in configuration. Original commit: elastic/x-pack-elasticsearch@a24ad7b08ab07350c609b1bf2bd7573de92592e4 --- marvel/build.gradle | 8 +------- qa/shield-core-rest-tests/build.gradle | 12 ++---------- shield/build.gradle | 5 ----- watcher/build.gradle | 20 +------------------- 4 files changed, 4 insertions(+), 41 deletions(-) diff --git a/marvel/build.gradle b/marvel/build.gradle index 2abc375e372..7eda0dacc35 100644 --- a/marvel/build.gradle +++ b/marvel/build.gradle @@ -8,16 +8,11 @@ esplugin { isolated false } -configurations { - licensePluginZip -} - ext.versions = [ okhttp: '2.3.0' ] dependencies { - licensePluginZip project(path: ':x-plugins:license:plugin') // zip provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') provided project(path: ':x-plugins:shield', configuration: 'runtime') testCompile 'org.elasticsearch:securemock:1.1' @@ -49,9 +44,8 @@ processTestResources { } integTest { - dependsOn configurations.licensePluginZip cluster { - plugin 'license', configurations.licensePluginZip + plugin 'license', project(':x-plugins:license:plugin') } } diff --git a/qa/shield-core-rest-tests/build.gradle b/qa/shield-core-rest-tests/build.gradle index 926d18bb2a1..8d2d2411db2 100644 --- a/qa/shield-core-rest-tests/build.gradle +++ b/qa/shield-core-rest-tests/build.gradle @@ -1,18 +1,10 @@ apply plugin: 'elasticsearch.rest-test' -configurations { - licensePluginZip - shieldPluginZip -} - dependencies { - licensePluginZip project(path: ':x-plugins:license:plugin') - shieldPluginZip project(path: ':x-plugins:shield') testCompile project(path: ':x-plugins:shield', configuration: 'runtime') } integTest { - dependsOn configurations.licensePluginZip, configurations.shieldPluginZip includePackaged true systemProperty 'tests.rest.blacklist', ['indices.get/10_basic/*allow_no_indices*', @@ -38,8 +30,8 @@ integTest { 'bulk/40_fields/Fields'].join(',') cluster { - plugin 'license', configurations.licensePluginZip - plugin 'shield', configurations.shieldPluginZip + plugin 'license', project(':x-plugins:license:plugin') + plugin 'shield', project(':x-plugins:shield') setupCommand 'setupDummyUser', 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' waitCondition = { node, ant -> diff --git a/shield/build.gradle b/shield/build.gradle index c314b390739..0d2c5999458 100644 --- a/shield/build.gradle +++ b/shield/build.gradle @@ -6,12 +6,7 @@ esplugin { isolated false } -configurations { - licensePluginZip -} - dependencies { - licensePluginZip project(path: ':x-plugins:license:plugin') // zip provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') compile project(':x-plugins:license:plugin-api') compile 'dk.brics.automaton:automaton:1.11-8' diff --git a/watcher/build.gradle b/watcher/build.gradle index ef997debfa9..3a64e30bb84 100644 --- a/watcher/build.gradle +++ b/watcher/build.gradle @@ -8,16 +8,11 @@ esplugin { isolated false } -configurations { - licensePluginZip -} - ext.versions = [ okhttp: '2.3.0' ] dependencies { - licensePluginZip project(path: ':x-plugins:license:plugin') // zip provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') provided project(path: ':x-plugins:shield', configuration: 'runtime') @@ -44,22 +39,9 @@ dependencies { compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked" compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked" -/*ext.expansions = [ - 'integ.http.port': integTest.cluster.httpPort -] - -processTestResources { - inputs.properties(expansions) - with copySpec { - MavenFilteringHack.filter(it, expansions) - } -} -*/ - integTest { - dependsOn configurations.licensePluginZip cluster { - plugin 'installLicensePlugin', configurations.licensePluginZip + plugin 'license', project(':x-plugins:license:plugin') } } From a4f596b20430020da2ed970e0686182f096dca36 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 24 Nov 2015 17:41:21 -0500 Subject: [PATCH 52/95] get watcher+groovy QA test working again (without hack) Original commit: elastic/x-pack-elasticsearch@843a5ea6e47239419f9f6c37ef58ba9b5f195673 --- .../build.gradle | 15 +++++++++ .../integration-tests.xml | 32 ------------------- .../test/watcher_groovy/10_basic.yaml | 0 .../test/watcher_groovy/20_minimal_body.yaml | 0 .../test/watcher_groovy/30_inline_watch.yaml | 0 5 files changed, 15 insertions(+), 32 deletions(-) create mode 100644 qa/smoke-test-watcher-with-groovy/build.gradle delete mode 100644 qa/smoke-test-watcher-with-groovy/integration-tests.xml rename qa/smoke-test-watcher-with-groovy/{ => src/test/resources}/rest-api-spec/test/watcher_groovy/10_basic.yaml (100%) rename qa/smoke-test-watcher-with-groovy/{ => src/test/resources}/rest-api-spec/test/watcher_groovy/20_minimal_body.yaml (100%) rename qa/smoke-test-watcher-with-groovy/{ => src/test/resources}/rest-api-spec/test/watcher_groovy/30_inline_watch.yaml (100%) diff --git a/qa/smoke-test-watcher-with-groovy/build.gradle b/qa/smoke-test-watcher-with-groovy/build.gradle new file mode 100644 index 00000000000..bfbdef34180 --- /dev/null +++ b/qa/smoke-test-watcher-with-groovy/build.gradle @@ -0,0 +1,15 @@ +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':x-plugins:watcher', configuration: 'runtime') + testCompile project(path: ':plugins:lang-groovy', configuration: 'runtime') +} + +integTest { + cluster { + plugin 'license', project(':x-plugins:license:plugin') + plugin 'watcher', project(':x-plugins:watcher') + plugin 'groovy', project(':plugins:lang-groovy') + systemProperty 'es.script.inline', 'on' + } +} diff --git a/qa/smoke-test-watcher-with-groovy/integration-tests.xml b/qa/smoke-test-watcher-with-groovy/integration-tests.xml deleted file mode 100644 index db4e3d96a6b..00000000000 --- a/qa/smoke-test-watcher-with-groovy/integration-tests.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/qa/smoke-test-watcher-with-groovy/rest-api-spec/test/watcher_groovy/10_basic.yaml b/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/10_basic.yaml similarity index 100% rename from qa/smoke-test-watcher-with-groovy/rest-api-spec/test/watcher_groovy/10_basic.yaml rename to qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/10_basic.yaml diff --git a/qa/smoke-test-watcher-with-groovy/rest-api-spec/test/watcher_groovy/20_minimal_body.yaml b/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/20_minimal_body.yaml similarity index 100% rename from qa/smoke-test-watcher-with-groovy/rest-api-spec/test/watcher_groovy/20_minimal_body.yaml rename to qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/20_minimal_body.yaml diff --git a/qa/smoke-test-watcher-with-groovy/rest-api-spec/test/watcher_groovy/30_inline_watch.yaml b/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/30_inline_watch.yaml similarity index 100% rename from qa/smoke-test-watcher-with-groovy/rest-api-spec/test/watcher_groovy/30_inline_watch.yaml rename to qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/30_inline_watch.yaml From feb247d2a0f649835cd1c67511a2200097e564cf Mon Sep 17 00:00:00 2001 From: debadair Date: Tue, 24 Nov 2015 15:32:13 -0800 Subject: [PATCH 53/95] Docs: Tweaked release notes for consistency. Original commit: elastic/x-pack-elasticsearch@bb94d42cbf490b969ecf90084858ce8eb8aec6a2 --- shield/docs/public/release-notes.asciidoc | 3 +++ watcher/docs/release-notes.asciidoc | 3 +++ 2 files changed, 6 insertions(+) diff --git a/shield/docs/public/release-notes.asciidoc b/shield/docs/public/release-notes.asciidoc index e650f1afd62..4512f716fde 100644 --- a/shield/docs/public/release-notes.asciidoc +++ b/shield/docs/public/release-notes.asciidoc @@ -60,6 +60,9 @@ November 24, 2015 .Breaking Changes * <> is now disabled by default. Set `shield.dls_fls.enabled` to `true` in `elasticsearch.yml` to enable it. You cannot submit `_bulk` update requests when document and field level security is enabled. +.Enhancement +* Adds support for Elasticsearch 2.0.1. + [float] ==== 2.0.0 October 28, 2015 diff --git a/watcher/docs/release-notes.asciidoc b/watcher/docs/release-notes.asciidoc index 7d5948b7e2f..9af55b191b0 100644 --- a/watcher/docs/release-notes.asciidoc +++ b/watcher/docs/release-notes.asciidoc @@ -54,6 +54,9 @@ November 24, 2015 ==== 2.0.1 November 24, 2015 +.Enhancement +* Adds support for Elasticsearch 2.0.1. + .Bug fixes * Fixed an issue where under specific conditions Watcher would not start if there are not finished watch executions from the previous time that watcher was running and those watch execution are unable the execute during the current start process. From 4b354075100f8355cb45b34e68da0c793f8bf866 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 24 Nov 2015 18:49:15 -0500 Subject: [PATCH 54/95] re-enable smoke-test-watcher-with-shield qa test Original commit: elastic/x-pack-elasticsearch@2710eb67efde3713a9b188a4d786ada084a52863 --- .../build.gradle | 45 +++++++ .../integration-tests.xml | 114 ------------------ ...atcher-with-shield-roles.yml => roles.yml} | 0 3 files changed, 45 insertions(+), 114 deletions(-) create mode 100644 qa/smoke-test-watcher-with-shield/build.gradle delete mode 100644 qa/smoke-test-watcher-with-shield/integration-tests.xml rename qa/smoke-test-watcher-with-shield/{watcher-with-shield-roles.yml => roles.yml} (100%) diff --git a/qa/smoke-test-watcher-with-shield/build.gradle b/qa/smoke-test-watcher-with-shield/build.gradle new file mode 100644 index 00000000000..eb1568c4ce5 --- /dev/null +++ b/qa/smoke-test-watcher-with-shield/build.gradle @@ -0,0 +1,45 @@ +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':x-plugins:shield', configuration: 'runtime') + testCompile project(path: ':x-plugins:watcher', configuration: 'runtime') +} + +// bring in watcher rest test suite +task copyWatcherRestTests(type: Copy) { + into project.sourceSets.test.output.resourcesDir + from project(':x-plugins:watcher').sourceSets.test.resources.srcDirs + include 'rest-api-spec/test/**' +} + +integTest.dependsOn(copyWatcherRestTests) + +integTest { + systemProperty 'tests.rest.blacklist', + ['hijack/10_basic/*', + 'array_compare_watch/10_basic/Basic array_compare watch'].join(',') + + cluster { + plugin 'license', project(':x-plugins:license:plugin') + plugin 'shield', project(':x-plugins:shield') + plugin 'watcher', project(':x-plugins:watcher') + extraConfigFile 'shield', 'roles.yml' + setupCommand 'setupTestAdminUser', + 'bin/shield/esusers', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'admin' + setupCommand 'setupWatcherManagerUser', + 'bin/shield/esusers', 'useradd', 'watcher_manager', '-p', 'changeme', '-r', 'watcher_manager' + setupCommand 'setupPowerlessUser', + 'bin/shield/esusers', 'useradd', 'powerless_user', '-p', 'changeme', '-r', 'crapy_role' + waitCondition = { node, ant -> + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://localhost:${node.httpPort()}", + dest: tmpFile.toString(), + username: 'test_admin', + password: 'changeme', + ignoreerrors: true, + retries: 10) + return tmpFile.exists() + } + } +} + diff --git a/qa/smoke-test-watcher-with-shield/integration-tests.xml b/qa/smoke-test-watcher-with-shield/integration-tests.xml deleted file mode 100644 index 8b0d6aedf22..00000000000 --- a/qa/smoke-test-watcher-with-shield/integration-tests.xml +++ /dev/null @@ -1,114 +0,0 @@ - - - - - - - - - - - - - Waiting for elasticsearch to become available on port @{port}... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Adding roles.yml with watcher roles - - - Adding shield users... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Checking we can connect with basic auth on port ${integ.http.port}... - - - - - diff --git a/qa/smoke-test-watcher-with-shield/watcher-with-shield-roles.yml b/qa/smoke-test-watcher-with-shield/roles.yml similarity index 100% rename from qa/smoke-test-watcher-with-shield/watcher-with-shield-roles.yml rename to qa/smoke-test-watcher-with-shield/roles.yml From f96a6700c41af285cd82add9d7d314e574d1da59 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 24 Nov 2015 16:10:04 -0800 Subject: [PATCH 55/95] Build: Remove hack in shield+watcher rest test for copying config file Original commit: elastic/x-pack-elasticsearch@e31ef685d0d2c2fedcdf6f5f6863f57d7830338f --- qa/smoke-test-watcher-with-shield/build.gradle | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qa/smoke-test-watcher-with-shield/build.gradle b/qa/smoke-test-watcher-with-shield/build.gradle index eb1568c4ce5..64db01497b7 100644 --- a/qa/smoke-test-watcher-with-shield/build.gradle +++ b/qa/smoke-test-watcher-with-shield/build.gradle @@ -12,9 +12,8 @@ task copyWatcherRestTests(type: Copy) { include 'rest-api-spec/test/**' } -integTest.dependsOn(copyWatcherRestTests) - integTest { + dependsOn copyWatcherRestTests systemProperty 'tests.rest.blacklist', ['hijack/10_basic/*', 'array_compare_watch/10_basic/Basic array_compare watch'].join(',') @@ -23,7 +22,7 @@ integTest { plugin 'license', project(':x-plugins:license:plugin') plugin 'shield', project(':x-plugins:shield') plugin 'watcher', project(':x-plugins:watcher') - extraConfigFile 'shield', 'roles.yml' + extraConfigFile 'shield/roles.yml', 'roles.yml' setupCommand 'setupTestAdminUser', 'bin/shield/esusers', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'admin' setupCommand 'setupWatcherManagerUser', From 19b7cad39c5454572893494e288e5f0f249770c9 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 24 Nov 2015 16:40:05 -0800 Subject: [PATCH 56/95] Build: Add back shield client qa tests Original commit: elastic/x-pack-elasticsearch@6cecea3992aec994cbbfdbe992354168a7aecdab --- qa/shield-client-tests/build.gradle | 27 ++++++ qa/shield-client-tests/integration-tests.xml | 95 -------------------- 2 files changed, 27 insertions(+), 95 deletions(-) create mode 100644 qa/shield-client-tests/build.gradle delete mode 100644 qa/shield-client-tests/integration-tests.xml diff --git a/qa/shield-client-tests/build.gradle b/qa/shield-client-tests/build.gradle new file mode 100644 index 00000000000..4c3cba8ea59 --- /dev/null +++ b/qa/shield-client-tests/build.gradle @@ -0,0 +1,27 @@ +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':x-plugins:shield', configuration: 'runtime') +} + +integTest { + cluster { + plugin 'license', project(':x-plugins:license:plugin') + plugin 'shield', project(':x-plugins:shield') + setupCommand 'setupDummyUser', + 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' + setupCommand 'setupTransportClientUser', + 'bin/shield/esusers', 'useradd', 'transport', '-p', 'changeme', '-r', 'transport_client' + waitCondition = { node, ant -> + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://localhost:${node.httpPort()}", + dest: tmpFile.toString(), + username: 'test_user', + password: 'changeme', + ignoreerrors: true, + retries: 10) + return tmpFile.exists() + } + } +} + diff --git a/qa/shield-client-tests/integration-tests.xml b/qa/shield-client-tests/integration-tests.xml deleted file mode 100644 index 56f8290b01c..00000000000 --- a/qa/shield-client-tests/integration-tests.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - - - - - - - - Waiting for elasticsearch to become available on port @{port}... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Adding shield users... - - - - - - - - - - - - - - - - - - - - - - - - Checking we can connect with basic auth on port ${integ.http.port}... - - - - - From 71d50ec0588b3616f21cbf4a224ad6db7986cff7 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 24 Nov 2015 19:41:50 -0500 Subject: [PATCH 57/95] add back shield audit qa tests Original commit: elastic/x-pack-elasticsearch@f34b2c99e9a6c710ed263433f456bc7d77133a52 --- qa/shield-audit-tests/build.gradle | 28 +++++++ qa/shield-audit-tests/integration-tests.xml | 90 --------------------- 2 files changed, 28 insertions(+), 90 deletions(-) create mode 100644 qa/shield-audit-tests/build.gradle delete mode 100644 qa/shield-audit-tests/integration-tests.xml diff --git a/qa/shield-audit-tests/build.gradle b/qa/shield-audit-tests/build.gradle new file mode 100644 index 00000000000..9ebb5cc83c9 --- /dev/null +++ b/qa/shield-audit-tests/build.gradle @@ -0,0 +1,28 @@ +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':x-plugins:shield', configuration: 'runtime') +} + +integTest { + cluster { + plugin 'license', project(':x-plugins:license:plugin') + plugin 'shield', project(':x-plugins:shield') + // TODO: these should be settings? + systemProperty 'es.shield.audit.enabled', 'true' + systemProperty 'es.shield.audit.outputs', 'index' + setupCommand 'setupDummyUser', + 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' + waitCondition = { node, ant -> + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://localhost:${node.httpPort()}", + dest: tmpFile.toString(), + username: 'test_user', + password: 'changeme', + ignoreerrors: true, + retries: 10) + return tmpFile.exists() + } + } +} + diff --git a/qa/shield-audit-tests/integration-tests.xml b/qa/shield-audit-tests/integration-tests.xml deleted file mode 100644 index 24ee45c2cdc..00000000000 --- a/qa/shield-audit-tests/integration-tests.xml +++ /dev/null @@ -1,90 +0,0 @@ - - - - - - - - - - - - - Waiting for elasticsearch to become available on port @{port}... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Adding shield users... - - - - - - - - - - - - - - - - - - - Checking we can connect with basic auth on port ${integ.http.port}... - - - - - From 66f3d18af0cf7f18828606c1c1f0bb62ce33e740 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 24 Nov 2015 17:07:04 -0800 Subject: [PATCH 58/95] Build: Add back smoke test plugins for xplugins This checks that all ES plugins and xplugins are installed. I also changed the rest check to be a simple plugin count, so it does not fail when new plugins are added. Original commit: elastic/x-pack-elasticsearch@eaab182e4390f95e42755d9374fa165ef83b330d --- qa/smoke-test-plugins/build.gradle | 50 +++++++++++++++++++ qa/smoke-test-plugins/integration-tests.xml | 8 --- .../test/smoke_test_plugins/10_basic.yaml | 16 ------ .../test/smoke_test_plugins/10_basic.yaml | 14 ++++++ 4 files changed, 64 insertions(+), 24 deletions(-) create mode 100644 qa/smoke-test-plugins/build.gradle delete mode 100644 qa/smoke-test-plugins/integration-tests.xml delete mode 100644 qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml create mode 100644 qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle new file mode 100644 index 00000000000..a3deb67646c --- /dev/null +++ b/qa/smoke-test-plugins/build.gradle @@ -0,0 +1,50 @@ +import org.elasticsearch.gradle.MavenFilteringHack + +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':x-plugins:shield', configuration: 'runtime') +} + +ext.pluginCount = 0 +// this loop must be outside of a configuration closure, otherwise it may get executed multiple times +for (Project subproj : project.rootProject.subprojects) { + if (subproj.path.startsWith(':plugins:')) { + // need to get a non-decorated project object, so must re-lookup the project by path + integTest.clusterConfig.plugin(subproj.name, project(subproj.path)) + project.pluginCount += 1 + } +} + +project.pluginCount += 4 +integTest { + cluster { + plugin 'licence', project(':x-plugins:license:plugin') + plugin 'shield', project(':x-plugins:shield') + plugin 'watcher', project(':x-plugins:watcher') + plugin 'marvel-agent', project(':x-plugins:marvel') + + setupCommand 'setupDummyUser', + 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' + waitCondition = { node, ant -> + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://localhost:${node.httpPort()}", + dest: tmpFile.toString(), + username: 'test_user', + password: 'changeme', + ignoreerrors: true, + retries: 10) + return tmpFile.exists() + } + } +} + +ext.expansions = [ + 'expected.plugin.count': pluginCount +] + +processTestResources { + inputs.properties(expansions) + MavenFilteringHack.filter(it, expansions) +} + diff --git a/qa/smoke-test-plugins/integration-tests.xml b/qa/smoke-test-plugins/integration-tests.xml deleted file mode 100644 index 4da32222a8f..00000000000 --- a/qa/smoke-test-plugins/integration-tests.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - diff --git a/qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml b/qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml deleted file mode 100644 index c1454200368..00000000000 --- a/qa/smoke-test-plugins/rest-api-spec/test/smoke_test_plugins/10_basic.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Integration tests for smoke testing plugins -# -"Plugins are actually installed": - - do: - cluster.state: {} - - # Get master node id - - set: { master_node: master } - - - do: - nodes.info: {} - - - match: { nodes.$master.plugins.15.name: license } - - match: { nodes.$master.plugins.18.name: marvel-agent } - - match: { nodes.$master.plugins.21.name: shield } - - match: { nodes.$master.plugins.24.name: watcher } diff --git a/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml b/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml new file mode 100644 index 00000000000..f87af6475a4 --- /dev/null +++ b/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml @@ -0,0 +1,14 @@ +# Integration tests for smoke testing plugins +# +"Plugins are actually installed": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - length: { nodes.$master.plugins: ${expected.plugin.count} } + # TODO: check that every plugin is installed From ac898ef4f3303c85378ea39fa9cfb74522dfb5a0 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 24 Nov 2015 20:10:46 -0500 Subject: [PATCH 59/95] re-enable shield example realm QA test Original commit: elastic/x-pack-elasticsearch@98fd46f3aa838bc54a9dc9c2c30ab1664929fa3a --- qa/shield-example-realm/build.gradle | 41 +++++++++ qa/shield-example-realm/integration-tests.xml | 92 ------------------- 2 files changed, 41 insertions(+), 92 deletions(-) create mode 100644 qa/shield-example-realm/build.gradle delete mode 100644 qa/shield-example-realm/integration-tests.xml diff --git a/qa/shield-example-realm/build.gradle b/qa/shield-example-realm/build.gradle new file mode 100644 index 00000000000..debd3aa8008 --- /dev/null +++ b/qa/shield-example-realm/build.gradle @@ -0,0 +1,41 @@ +apply plugin: 'elasticsearch.esplugin' + +esplugin { + description 'a very basic implementation of a custom realm to validate it works' + classname 'org.elasticsearch.example.ExampleRealmPlugin' + isolated false +} + +dependencies { + provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') + provided project(path: ':x-plugins:shield', configuration: 'runtime') +} + +compileJava.options.compilerArgs << "-Xlint:-rawtypes" +//compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" + +integTest { + cluster { + plugin 'license', project(':x-plugins:license:plugin') + plugin 'shield', project(':x-plugins:shield') + // TODO: these should be settings? + systemProperty 'es.shield.authc.realms.custom.order', '0' + systemProperty 'es.shield.authc.realms.custom.type', 'custom' + systemProperty 'es.shield.authc.realms.esusers.order', '1' + systemProperty 'es.shield.authc.realms.esusers.type', 'esusers' + + setupCommand 'setupDummyUser', + 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' + waitCondition = { node, ant -> + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://localhost:${node.httpPort()}", + dest: tmpFile.toString(), + username: 'test_user', + password: 'changeme', + ignoreerrors: true, + retries: 10) + return tmpFile.exists() + } + } +} + diff --git a/qa/shield-example-realm/integration-tests.xml b/qa/shield-example-realm/integration-tests.xml deleted file mode 100644 index e4c3afb2b37..00000000000 --- a/qa/shield-example-realm/integration-tests.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - - - - - - - - - - Waiting for elasticsearch to become available on port @{port}... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Adding shield users... - - - - - - - - - - - - - - - - - - - - - Checking we can connect with basic auth on port ${integ.http.port}... - - - - - From 7ed4ea56b21fab4ed51ac5ae18534ccb22456c35 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 24 Nov 2015 23:19:04 -0500 Subject: [PATCH 60/95] re-enable smoke-test-plugins with ssl Note, its a bit crazy/hackish, but it works. Original commit: elastic/x-pack-elasticsearch@377113c1c209bd9edbd9d8ad9f4da977525810cb --- qa/smoke-test-plugins-ssl/build.gradle | 88 ++++++++++++++++++ .../integration-tests.xml | 90 ------------------- .../smoketest/SmokeTestPluginsSslIT.java | 3 + .../test/smoke_test_plugins_ssl/10_basic.yaml | 6 +- 4 files changed, 94 insertions(+), 93 deletions(-) create mode 100644 qa/smoke-test-plugins-ssl/build.gradle delete mode 100644 qa/smoke-test-plugins-ssl/integration-tests.xml rename qa/smoke-test-plugins-ssl/{ => src/test/resources}/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml (60%) diff --git a/qa/smoke-test-plugins-ssl/build.gradle b/qa/smoke-test-plugins-ssl/build.gradle new file mode 100644 index 00000000000..1d4728c4be3 --- /dev/null +++ b/qa/smoke-test-plugins-ssl/build.gradle @@ -0,0 +1,88 @@ +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':x-plugins:shield', configuration: 'runtime') +} + +// ssl setup, it reuses the ssl-setup.xml from ant, for now. + +// location of target keystore +File keystore = new File(project.buildDir, 'keystore/test-node.jks') + +// we touch keystore because otherwise it fails, extraConfigFile does not exist +// this tricks some broken compile-time check into just moving along: we nuke this stuff before we actually generate +keystore.parentFile.mkdirs() +keystore.createNewFile() + +// add keystore to test classpath: it expects it there +sourceSets.test.resources.srcDir(keystore.parentFile) + +configurations { + antcontrib { + description = 'ant-contrib' + transitive = false + } +} + +dependencies { + antcontrib "ant-contrib:ant-contrib:1.0b3" +} + +// this loop must be outside of a configuration closure, otherwise it may get executed multiple times +for (Project subproj : project.rootProject.subprojects) { + if (subproj.path.startsWith(':plugins:')) { + // need to get a non-decorated project object, so must re-lookup the project by path + integTest.clusterConfig.plugin(subproj.name, project(subproj.path)) + } +} + +// we should be able to taskdef, but gradle has *the worst* classloader management +// so just do a hack, jam ant-contrib directly into gradle's ant's classloader +ClassLoader antClassLoader = org.apache.tools.ant.Project.class.classLoader +configurations.antcontrib.each { File f -> + antClassLoader.addURL(f.toURI().toURL()) +} + +// suck in ssl-setup.xml, defining matching tasks in gradle +ant.property(name: 'integ.scratch', location: project.buildDir) +ant.property(name: 'keystore.path', keystore) +ant.importBuild 'ssl-setup.xml' + +// clean all intermediate/keystore files before regenerating it +task cleanKeystore(type: Delete) { + delete new File(project.buildDir, 'keystore'), + new File(project.buildDir, 'cert'), + new File(project.buildDir, 'ca') +} + +// wipe and regenerate keystore so its available as a test dep +processTestResources.dependsOn('cleanKeystore') +processTestResources.dependsOn('generate-keystore') + +integTest { + cluster { + // TODO: use some variable here for port number + systemProperty 'es.marvel.agent.exporter.es.hosts', 'https://marvel_export:changeme@localhost:9400' + systemProperty 'es.marvel.agent.exporter.es.ssl.truststore.path', 'test-node.jks' + systemProperty 'es.marvel.agent.exporter.es.ssl.truststore.password', 'keypass' + systemProperty 'es.shield.transport.ssl', 'true' + systemProperty 'es.shield.http.ssl', 'true' + systemProperty 'es.shield.ssl.keystore.path', 'test-node.jks' + systemProperty 'es.shield.ssl.keystore.password', 'keypass' + plugin 'licence', project(':x-plugins:license:plugin') + plugin 'shield', project(':x-plugins:shield') + plugin 'watcher', project(':x-plugins:watcher') + plugin 'marvel-agent', project(':x-plugins:marvel') + + // copy keystore into config/ + extraConfigFile 'test-node.jks', keystore + setupCommand 'setupTestUser', + 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' + setupCommand 'setupMarvelUser', + 'bin/shield/esusers', 'useradd', 'marvel_export', '-p', 'changeme', '-r', 'marvel_agent' + waitCondition = { node, ant -> + // we just return true, doing an https check is tricky here + return true + } + } +} diff --git a/qa/smoke-test-plugins-ssl/integration-tests.xml b/qa/smoke-test-plugins-ssl/integration-tests.xml deleted file mode 100644 index a23691500ec..00000000000 --- a/qa/smoke-test-plugins-ssl/integration-tests.xml +++ /dev/null @@ -1,90 +0,0 @@ - - - - - - - - - - - - - Waiting for elasticsearch to become available on port @{port}... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Setting up Shield auth - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Checking we can connect with basic auth on port ${integ.http.port}... - - - - - - diff --git a/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java b/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java index 65ad838e67e..f33264fb239 100644 --- a/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java +++ b/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java @@ -11,11 +11,14 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.client.support.Headers; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.test.rest.ESRestTestCase; diff --git a/qa/smoke-test-plugins-ssl/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml b/qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml similarity index 60% rename from qa/smoke-test-plugins-ssl/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml rename to qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml index c1454200368..6ae125ab28b 100644 --- a/qa/smoke-test-plugins-ssl/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml +++ b/qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml @@ -11,6 +11,6 @@ nodes.info: {} - match: { nodes.$master.plugins.15.name: license } - - match: { nodes.$master.plugins.18.name: marvel-agent } - - match: { nodes.$master.plugins.21.name: shield } - - match: { nodes.$master.plugins.24.name: watcher } + - match: { nodes.$master.plugins.19.name: marvel-agent } + - match: { nodes.$master.plugins.22.name: shield } + - match: { nodes.$master.plugins.25.name: watcher } From 2fd35ea57547d3a5bb6b8e72ca0a00d99dcb1875 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 25 Nov 2015 10:39:08 -0500 Subject: [PATCH 61/95] Gradle daemon is a demon This commit adds a property that will prevent the Gradle daemon from being used for builds (even if one is running). This is to avoid some nasty issues (e.g., SIGBUS faults and other mmap diasters) that result from class loaders not being closed properly. Relates elastic/elasticsearchelastic/elasticsearch#15018 Original commit: elastic/x-pack-elasticsearch@0786013a81e5b4d4a5254e0c062ea68aa9bb8fd4 --- gradle.properties | 1 + 1 file changed, 1 insertion(+) create mode 100644 gradle.properties diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000000..6b1823d86a6 --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +org.gradle.daemon=false From ae24881484fbd45c42e58cb1f7591d8d63d87e86 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 25 Nov 2015 13:29:00 -0500 Subject: [PATCH 62/95] Move disabled watcher+groovy "unit" tests to qa/messy-test-watcher-with-groovy This is all the tests disabled from https://github.com/elastic/x-plugins/issues/724 At least, they will be running in the build in some way. If we can fix gradle to add plugin metadata from lang-groovy to the test classpath, security manager can be re-enabled for these as well. But its also only 8 tests, maybe its easier to fix them? Original commit: elastic/x-pack-elasticsearch@a5c407b80f84218e834a1ed68c35de72308313aa --- .../build.gradle | 20 +++ .../tests}/ExecutionVarsIntegrationTests.java | 14 ++- .../tests/GroovyManualExecutionTests.java | 118 ++++++++++++++++++ ...HistoryTemplateTransformMappingsTests.java | 15 ++- .../tests}/IndexActionIntegrationTests.java | 16 ++- .../messy/tests/MessyTestUtils.java | 44 +++++++ .../tests}/ScriptConditionSearchTests.java | 20 ++- .../messy/tests}/ScriptConditionTests.java | 12 +- .../tests}/TransformIntegrationTests.java | 15 ++- watcher/build.gradle | 13 ++ .../execution/ManualExecutionTests.java | 62 +-------- 11 files changed, 267 insertions(+), 82 deletions(-) create mode 100644 qa/messy-test-watcher-with-groovy/build.gradle rename {watcher/src/test/java/org/elasticsearch/watcher/test/integration => qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests}/ExecutionVarsIntegrationTests.java (96%) create mode 100644 qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/GroovyManualExecutionTests.java rename {watcher/src/test/java/org/elasticsearch/watcher/history => qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests}/HistoryTemplateTransformMappingsTests.java (93%) rename {watcher/src/test/java/org/elasticsearch/watcher/actions/index => qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests}/IndexActionIntegrationTests.java (96%) create mode 100644 qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/MessyTestUtils.java rename {watcher/src/test/java/org/elasticsearch/watcher/condition/script => qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests}/ScriptConditionSearchTests.java (88%) rename {watcher/src/test/java/org/elasticsearch/watcher/condition/script => qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests}/ScriptConditionTests.java (96%) rename {watcher/src/test/java/org/elasticsearch/watcher/transform => qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests}/TransformIntegrationTests.java (96%) diff --git a/qa/messy-test-watcher-with-groovy/build.gradle b/qa/messy-test-watcher-with-groovy/build.gradle new file mode 100644 index 00000000000..26d927a16cd --- /dev/null +++ b/qa/messy-test-watcher-with-groovy/build.gradle @@ -0,0 +1,20 @@ + +/* + * Messy tests that depend on groovy directly. Fix these! + * https://github.com/elastic/x-plugins/issues/724 + */ + +apply plugin: 'elasticsearch.standalone-test' + +dependencies { + testCompile project(path: ':x-plugins:license:plugin', configuration: 'runtime') + testCompile project(path: ':x-plugins:shield', configuration: 'runtime') + testCompile project(path: ':x-plugins:watcher', configuration: 'testArtifacts') + testCompile project(path: ':plugins:lang-groovy', configuration: 'runtime') +} + +// TODO: remove this, its because gradle does not bring in plugin-metadata for lang-groovy +// into the test classpath: if it did, then things will work +test { + systemProperty 'tests.security.manager', 'false' +} diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/ExecutionVarsIntegrationTests.java b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ExecutionVarsIntegrationTests.java similarity index 96% rename from watcher/src/test/java/org/elasticsearch/watcher/test/integration/ExecutionVarsIntegrationTests.java rename to qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ExecutionVarsIntegrationTests.java index 404c1acc4bf..6ba797fa02e 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/ExecutionVarsIntegrationTests.java +++ b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ExecutionVarsIntegrationTests.java @@ -3,12 +3,13 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.watcher.test.integration; +package org.elasticsearch.messy.tests; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.util.Callback; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.watcher.client.WatcherClient; import org.elasticsearch.watcher.support.xcontent.ObjectPath; import org.elasticsearch.watcher.support.xcontent.XContentSource; @@ -32,8 +33,15 @@ import static org.hamcrest.Matchers.notNullValue; /** */ -@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") public class ExecutionVarsIntegrationTests extends AbstractWatcherIntegrationTestCase { + + @Override + protected List> pluginTypes() { + List> types = super.pluginTypes(); + types.add(GroovyPlugin.class); + return types; + } + @Override protected boolean timeWarped() { return true; diff --git a/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/GroovyManualExecutionTests.java b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/GroovyManualExecutionTests.java new file mode 100644 index 00000000000..3f54855c345 --- /dev/null +++ b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/GroovyManualExecutionTests.java @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.messy.tests; + +import static org.elasticsearch.watcher.actions.ActionBuilders.loggingAction; +import static org.elasticsearch.watcher.client.WatchSourceBuilders.watchBuilder; +import static org.elasticsearch.watcher.input.InputBuilders.simpleInput; +import static org.elasticsearch.watcher.trigger.TriggerBuilders.schedule; +import static org.elasticsearch.watcher.trigger.schedule.Schedules.cron; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; + +import java.util.ArrayList; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.watcher.client.WatchSourceBuilder; +import org.elasticsearch.watcher.condition.script.ScriptCondition; +import org.elasticsearch.watcher.execution.ManualExecutionContext; +import org.elasticsearch.watcher.execution.ManualExecutionTests.ExecutionRunner; +import org.elasticsearch.watcher.history.WatchRecord; +import org.elasticsearch.watcher.support.Script; +import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; +import org.elasticsearch.watcher.transport.actions.delete.DeleteWatchResponse; +import org.elasticsearch.watcher.transport.actions.get.GetWatchRequest; +import org.elasticsearch.watcher.transport.actions.put.PutWatchRequest; +import org.elasticsearch.watcher.transport.actions.put.PutWatchResponse; +import org.elasticsearch.watcher.trigger.manual.ManualTriggerEvent; +import org.elasticsearch.watcher.trigger.schedule.ScheduleTriggerEvent; +import org.elasticsearch.watcher.watch.Watch; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +/** + * Two groovy-using methods from ManualExecutionTests. + * They appear to be using groovy as a way to sleep. + */ +public class GroovyManualExecutionTests extends AbstractWatcherIntegrationTestCase { + + @Override + protected List> pluginTypes() { + List> types = super.pluginTypes(); + types.add(GroovyPlugin.class); + return types; + } + + @Override + protected boolean enableShield() { + return false; + } + + public void testWatchExecutionDuration() throws Exception { + WatchSourceBuilder watchBuilder = watchBuilder() + .trigger(schedule(cron("0 0 0 1 * ? 2099"))) + .input(simpleInput("foo", "bar")) + .condition(new ScriptCondition((new Script.Builder.Inline("sleep 100; return true")).build())) + .addAction("log", loggingAction("foobar")); + + Watch watch = watchParser().parse("_id", false, watchBuilder.buildAsBytes(XContentType.JSON)); + ManualExecutionContext.Builder ctxBuilder = ManualExecutionContext.builder(watch, false, new ManualTriggerEvent("_id", new ScheduleTriggerEvent(new DateTime(DateTimeZone.UTC), new DateTime(DateTimeZone.UTC))), new TimeValue(1, TimeUnit.HOURS)); + WatchRecord record = executionService().execute(ctxBuilder.build()); + assertThat(record.result().executionDurationMs(), greaterThanOrEqualTo(100L)); + } + + public void testForceDeletionOfLongRunningWatch() throws Exception { + WatchSourceBuilder watchBuilder = watchBuilder() + .trigger(schedule(cron("0 0 0 1 * ? 2099"))) + .input(simpleInput("foo", "bar")) + .condition(new ScriptCondition((new Script.Builder.Inline("sleep 10000; return true")).build())) + .defaultThrottlePeriod(new TimeValue(1, TimeUnit.HOURS)) + .addAction("log", loggingAction("foobar")); + + int numberOfThreads = scaledRandomIntBetween(1, 5); + PutWatchResponse putWatchResponse = watcherClient().putWatch(new PutWatchRequest("_id", watchBuilder)).actionGet(); + assertThat(putWatchResponse.getVersion(), greaterThan(0L)); + refresh(); + assertThat(watcherClient().getWatch(new GetWatchRequest("_id")).actionGet().isFound(), equalTo(true)); + + CountDownLatch startLatch = new CountDownLatch(1); + + List threads = new ArrayList<>(); + for (int i = 0; i < numberOfThreads; ++i) { + threads.add(new Thread(new ExecutionRunner(watchService(), executionService(), "_id", startLatch))); + } + + for (Thread thread : threads) { + thread.start(); + } + DeleteWatchResponse deleteWatchResponse = watcherClient().prepareDeleteWatch("_id").setForce(true).get(); + assertThat(deleteWatchResponse.isFound(), is(true)); + + deleteWatchResponse = watcherClient().prepareDeleteWatch("_id").get(); + assertThat(deleteWatchResponse.isFound(), is(false)); + + startLatch.countDown(); + + long startJoin = System.currentTimeMillis(); + for (Thread thread : threads) { + thread.join(); + } + long endJoin = System.currentTimeMillis(); + TimeValue tv = new TimeValue(10 * (numberOfThreads+1), TimeUnit.SECONDS); + assertThat("Shouldn't take longer than [" + tv.getSeconds() + "] seconds for all the threads to stop", (endJoin - startJoin), lessThan(tv.getMillis())); + } + +} diff --git a/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTransformMappingsTests.java b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/HistoryTemplateTransformMappingsTests.java similarity index 93% rename from watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTransformMappingsTests.java rename to qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/HistoryTemplateTransformMappingsTests.java index bac674d7821..ff9830ca489 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTransformMappingsTests.java +++ b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/HistoryTemplateTransformMappingsTests.java @@ -3,19 +3,21 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.watcher.history; +package org.elasticsearch.messy.tests; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.watcher.execution.ExecutionState; import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; import org.elasticsearch.watcher.transport.actions.put.PutWatchResponse; import java.io.IOException; +import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; @@ -33,8 +35,15 @@ import static org.hamcrest.Matchers.notNullValue; * This test makes sure that the http host and path fields in the watch_record action result are * not analyzed so they can be used in aggregations */ -@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") public class HistoryTemplateTransformMappingsTests extends AbstractWatcherIntegrationTestCase { + + @Override + protected List> pluginTypes() { + List> types = super.pluginTypes(); + types.add(GroovyPlugin.class); + return types; + } + @Override protected boolean timeWarped() { return true; // just to have better control over the triggers diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionIntegrationTests.java b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/IndexActionIntegrationTests.java similarity index 96% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionIntegrationTests.java rename to qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/IndexActionIntegrationTests.java index 4fedd5b0d8a..719ac19ff0e 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionIntegrationTests.java +++ b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/IndexActionIntegrationTests.java @@ -3,12 +3,13 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.watcher.actions.index; +package org.elasticsearch.messy.tests; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.sort.SortOrder; @@ -21,6 +22,8 @@ import org.elasticsearch.watcher.trigger.schedule.ScheduleTriggerEvent; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import java.util.List; + import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; @@ -39,8 +42,15 @@ import static org.hamcrest.Matchers.is; /** * */ -@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") public class IndexActionIntegrationTests extends AbstractWatcherIntegrationTestCase { + + @Override + protected List> pluginTypes() { + List> types = super.pluginTypes(); + types.add(GroovyPlugin.class); + return types; + } + public void testSimple() throws Exception { PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder() .trigger(schedule(cron("0/1 * * * * ? 2020"))) diff --git a/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/MessyTestUtils.java b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/MessyTestUtils.java new file mode 100644 index 00000000000..448f9a9074a --- /dev/null +++ b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/MessyTestUtils.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.messy.tests; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.script.ScriptContextRegistry; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.groovy.GroovyScriptEngineService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.watcher.support.init.proxy.ScriptServiceProxy; +import org.elasticsearch.watcher.support.text.xmustache.XMustacheScriptEngineService; +import org.junit.Ignore; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +@Ignore // not a test. +@SuppressForbidden(reason = "gradle is broken and tries to run me as a test") +public final class MessyTestUtils { + public static ScriptServiceProxy getScriptServiceProxy(ThreadPool tp) throws Exception { + Settings settings = Settings.settingsBuilder() + .put("script.inline", "on") + .put("script.indexed", "on") + .put("path.home", LuceneTestCase.createTempDir()) + .build(); + XMustacheScriptEngineService mustacheScriptEngineService = new XMustacheScriptEngineService(settings); + GroovyScriptEngineService groovyScriptEngineService = new GroovyScriptEngineService(settings); + Set engineServiceSet = new HashSet<>(); + engineServiceSet.add(mustacheScriptEngineService); + engineServiceSet.add(groovyScriptEngineService); + ScriptContextRegistry registry = new ScriptContextRegistry(Arrays.asList(ScriptServiceProxy.INSTANCE)); + + return ScriptServiceProxy.of(new ScriptService(settings, new Environment(settings), engineServiceSet, new ResourceWatcherService(settings, tp), registry)); + } +} diff --git a/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionSearchTests.java b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionSearchTests.java similarity index 88% rename from watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionSearchTests.java rename to qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionSearchTests.java index 168796ec327..d5df78e57ea 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionSearchTests.java +++ b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionSearchTests.java @@ -3,12 +3,13 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.watcher.condition.script; +package org.elasticsearch.messy.tests; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.text.StringText; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; @@ -17,30 +18,39 @@ import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.InternalSearchHits; import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.condition.script.ExecutableScriptCondition; +import org.elasticsearch.watcher.condition.script.ScriptCondition; import org.elasticsearch.watcher.execution.WatchExecutionContext; import org.elasticsearch.watcher.support.Script; import org.elasticsearch.watcher.support.init.proxy.ScriptServiceProxy; import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; -import org.elasticsearch.watcher.test.WatcherTestUtils; import org.elasticsearch.watcher.watch.Payload; import org.junit.After; import org.junit.Before; +import java.util.List; + import static org.elasticsearch.watcher.test.WatcherTestUtils.mockExecutionContext; import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.when; /** */ -@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") public class ScriptConditionSearchTests extends AbstractWatcherIntegrationTestCase { private ThreadPool tp = null; private ScriptServiceProxy scriptService; + + @Override + protected List> pluginTypes() { + List> types = super.pluginTypes(); + types.add(GroovyPlugin.class); + return types; + } @Before public void init() throws Exception { tp = new ThreadPool(ThreadPool.Names.SAME); - scriptService = WatcherTestUtils.getScriptServiceProxy(tp); + scriptService = MessyTestUtils.getScriptServiceProxy(tp); } @After diff --git a/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionTests.java b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionTests.java similarity index 96% rename from watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionTests.java rename to qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionTests.java index f66a3de6ce2..6042f70eecf 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionTests.java +++ b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionTests.java @@ -3,10 +3,9 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.watcher.condition.script; +package org.elasticsearch.messy.tests; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; @@ -21,6 +20,9 @@ import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.condition.Condition; +import org.elasticsearch.watcher.condition.script.ExecutableScriptCondition; +import org.elasticsearch.watcher.condition.script.ScriptCondition; +import org.elasticsearch.watcher.condition.script.ScriptConditionFactory; import org.elasticsearch.watcher.execution.WatchExecutionContext; import org.elasticsearch.watcher.support.Script; import org.elasticsearch.watcher.support.init.proxy.ScriptServiceProxy; @@ -35,18 +37,18 @@ import java.io.IOException; import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.watcher.support.Exceptions.illegalArgument; -import static org.elasticsearch.watcher.test.WatcherTestUtils.getScriptServiceProxy; import static org.elasticsearch.watcher.test.WatcherTestUtils.mockExecutionContext; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.elasticsearch.messy.tests.MessyTestUtils.getScriptServiceProxy; + /** */ -@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") public class ScriptConditionTests extends ESTestCase { ThreadPool tp = null; - + @Before public void init() { tp = new ThreadPool(ThreadPool.Names.SAME); diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transform/TransformIntegrationTests.java b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/TransformIntegrationTests.java similarity index 96% rename from watcher/src/test/java/org/elasticsearch/watcher/transform/TransformIntegrationTests.java rename to qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/TransformIntegrationTests.java index f6be9b031da..018d5c52a4b 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/transform/TransformIntegrationTests.java +++ b/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/TransformIntegrationTests.java @@ -3,14 +3,15 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.watcher.transform; +package org.elasticsearch.messy.tests; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.watcher.support.Script; import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; import org.elasticsearch.watcher.test.WatcherTestUtils; @@ -21,6 +22,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -43,8 +45,15 @@ import static org.hamcrest.Matchers.is; /** */ -@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") public class TransformIntegrationTests extends AbstractWatcherIntegrationTestCase { + + @Override + protected List> pluginTypes() { + List> types = super.pluginTypes(); + types.add(GroovyPlugin.class); + return types; + } + @Override public Settings nodeSettings(int nodeOrdinal) { Path configDir = createTempDir(); diff --git a/watcher/build.gradle b/watcher/build.gradle index 3a64e30bb84..bf34dd721bd 100644 --- a/watcher/build.gradle +++ b/watcher/build.gradle @@ -55,3 +55,16 @@ bundlePlugin { into 'bin' } } + +// TODO: don't publish test artifacts just to run messy tests, fix the tests! +// https://github.com/elastic/x-plugins/issues/724 +configurations { + testArtifacts.extendsFrom testRuntime +} +task testJar(type: Jar) { + classifier "test" + from sourceSets.test.output +} +artifacts { + testArtifacts testJar +} diff --git a/watcher/src/test/java/org/elasticsearch/watcher/execution/ManualExecutionTests.java b/watcher/src/test/java/org/elasticsearch/watcher/execution/ManualExecutionTests.java index 97114df413e..a745167271f 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/execution/ManualExecutionTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/execution/ManualExecutionTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.watcher.execution; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.unit.TimeValue; @@ -21,7 +20,6 @@ import org.elasticsearch.watcher.input.simple.SimpleInput; import org.elasticsearch.watcher.support.clock.SystemClock; import org.elasticsearch.watcher.support.xcontent.ObjectPath; import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; -import org.elasticsearch.watcher.transport.actions.delete.DeleteWatchResponse; import org.elasticsearch.watcher.transport.actions.execute.ExecuteWatchRequest; import org.elasticsearch.watcher.transport.actions.execute.ExecuteWatchRequestBuilder; import org.elasticsearch.watcher.transport.actions.execute.ExecuteWatchResponse; @@ -302,63 +300,7 @@ public class ManualExecutionTests extends AbstractWatcherIntegrationTestCase { assertThat(ObjectPath.eval("state", executeWatchResult), equalTo(ExecutionState.THROTTLED.toString())); } - @AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") - public void testWatchExecutionDuration() throws Exception { - WatchSourceBuilder watchBuilder = watchBuilder() - .trigger(schedule(cron("0 0 0 1 * ? 2099"))) - .input(simpleInput("foo", "bar")) -// .condition(new ScriptCondition((new Script.Builder.Inline("sleep 100; return true")).build())) - .addAction("log", loggingAction("foobar")); - - Watch watch = watchParser().parse("_id", false, watchBuilder.buildAsBytes(XContentType.JSON)); - ManualExecutionContext.Builder ctxBuilder = ManualExecutionContext.builder(watch, false, new ManualTriggerEvent("_id", new ScheduleTriggerEvent(new DateTime(DateTimeZone.UTC), new DateTime(DateTimeZone.UTC))), new TimeValue(1, TimeUnit.HOURS)); - WatchRecord record = executionService().execute(ctxBuilder.build()); - assertThat(record.result().executionDurationMs(), greaterThanOrEqualTo(100L)); - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") - public void testForceDeletionOfLongRunningWatch() throws Exception { - WatchSourceBuilder watchBuilder = watchBuilder() - .trigger(schedule(cron("0 0 0 1 * ? 2099"))) - .input(simpleInput("foo", "bar")) -// .condition(new ScriptCondition((new Script.Builder.Inline("sleep 10000; return true")).build())) - .defaultThrottlePeriod(new TimeValue(1, TimeUnit.HOURS)) - .addAction("log", loggingAction("foobar")); - - int numberOfThreads = scaledRandomIntBetween(1, 5); - PutWatchResponse putWatchResponse = watcherClient().putWatch(new PutWatchRequest("_id", watchBuilder)).actionGet(); - assertThat(putWatchResponse.getVersion(), greaterThan(0L)); - refresh(); - assertThat(watcherClient().getWatch(new GetWatchRequest("_id")).actionGet().isFound(), equalTo(true)); - - CountDownLatch startLatch = new CountDownLatch(1); - - List threads = new ArrayList<>(); - for (int i = 0; i < numberOfThreads; ++i) { - threads.add(new Thread(new ExecutionRunner(watchService(), executionService(), "_id", startLatch))); - } - - for (Thread thread : threads) { - thread.start(); - } - DeleteWatchResponse deleteWatchResponse = watcherClient().prepareDeleteWatch("_id").setForce(true).get(); - assertThat(deleteWatchResponse.isFound(), is(true)); - - deleteWatchResponse = watcherClient().prepareDeleteWatch("_id").get(); - assertThat(deleteWatchResponse.isFound(), is(false)); - - startLatch.countDown(); - - long startJoin = System.currentTimeMillis(); - for (Thread thread : threads) { - thread.join(); - } - long endJoin = System.currentTimeMillis(); - TimeValue tv = new TimeValue(10 * (numberOfThreads+1), TimeUnit.SECONDS); - assertThat("Shouldn't take longer than [" + tv.getSeconds() + "] seconds for all the threads to stop", (endJoin - startJoin), lessThan(tv.getMillis())); - } - - private static class ExecutionRunner implements Runnable { + public static class ExecutionRunner implements Runnable { final WatcherService watcherService; final ExecutionService executionService; @@ -366,7 +308,7 @@ public class ManualExecutionTests extends AbstractWatcherIntegrationTestCase { final CountDownLatch startLatch; final ManualExecutionContext.Builder ctxBuilder; - private ExecutionRunner(WatcherService watcherService, ExecutionService executionService, String watchId, CountDownLatch startLatch) { + public ExecutionRunner(WatcherService watcherService, ExecutionService executionService, String watchId, CountDownLatch startLatch) { this.watcherService = watcherService; this.executionService = executionService; this.watchId = watchId; From 59a10e6309f21b47451c4442392b94cf1ca3a2fc Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 25 Nov 2015 00:13:59 -0800 Subject: [PATCH 63/95] Build: Simplify ssl test to not use ant This change ports the tasks from the ssl ant build file into gradle tasks. Original commit: elastic/x-pack-elasticsearch@af881960505bcb19f1020f54d042acf0072eb887 --- qa/smoke-test-plugins-ssl/build.gradle | 187 ++++++++++++++++++------ qa/smoke-test-plugins-ssl/ssl-setup.xml | 170 --------------------- 2 files changed, 142 insertions(+), 215 deletions(-) delete mode 100644 qa/smoke-test-plugins-ssl/ssl-setup.xml diff --git a/qa/smoke-test-plugins-ssl/build.gradle b/qa/smoke-test-plugins-ssl/build.gradle index 1d4728c4be3..58afb5e9dfc 100644 --- a/qa/smoke-test-plugins-ssl/build.gradle +++ b/qa/smoke-test-plugins-ssl/build.gradle @@ -1,34 +1,154 @@ +import org.elasticsearch.gradle.LoggedExec + apply plugin: 'elasticsearch.rest-test' dependencies { testCompile project(path: ':x-plugins:shield', configuration: 'runtime') } -// ssl setup, it reuses the ssl-setup.xml from ant, for now. - -// location of target keystore +// location of keystore and files to generate it +File ca = new File(project.buildDir, 'ca') +File caConfig = new File(ca, 'conf/caconfig.cnf') +File cert = new File(project.buildDir, 'cert/test-node.csr') +File signedCert = new File(project.buildDir, 'cert/test-node-signed.csr') File keystore = new File(project.buildDir, 'keystore/test-node.jks') -// we touch keystore because otherwise it fails, extraConfigFile does not exist -// this tricks some broken compile-time check into just moving along: we nuke this stuff before we actually generate -keystore.parentFile.mkdirs() -keystore.createNewFile() +String caConfigData = """ +[ ca ] +default_ca = CA_default +[ CA_default ] +copy_extensions = copy +serial = ${ca}/serial +database = ${ca}/index.txt +new_certs_dir = ${ca}/certs +certificate = ${ca}/certs/cacert.pem +private_key = ${ca}/private/cakey.pem +default_days = 712 +default_md = sha256 +preserve = no +email_in_dn = no +x509_extensions = v3_ca +name_opt = ca_default +cert_opt = ca_default +policy = policy_anything +[ policy_anything ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional +[ req ] +default_bits = 2048 # Size of keys +default_keyfile = key.pem # name of generated keys +default_md = sha256 # message digest algorithm +string_mask = nombstr # permitted characters +distinguished_name = req_distinguished_name +req_extensions = v3_req +[ req_distinguished_name ] +# Variable name Prompt string +#------------------------- ---------------------------------- +0.organizationName = Organization Name (company) +organizationalUnitName = Organizational Unit Name (department, division) +emailAddress = Email Address +emailAddress_max = 40 +localityName = Locality Name (city, district) +stateOrProvinceName = State or Province Name (full name) +countryName = Country Name (2 letter code) +countryName_min = 2 +countryName_max = 2 +commonName = Common Name (hostname, IP, or your name) +commonName_max = 64 +# Default values for the above, for consistency and less typing. +# Variable name Value +#------------------------ ------------------------------ +0.organizationName_default = Elasticsearch Test Org +localityName_default = Amsterdam +stateOrProvinceName_default = Amsterdam +countryName_default = NL +emailAddress_default = cacerttest@YOUR.COMPANY.TLD +[ v3_ca ] +basicConstraints = CA:TRUE +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer:always +[ v3_req ] +basicConstraints = CA:FALSE +subjectKeyIdentifier = hash +""" + +// generate the keystore +task createKey(type: LoggedExec) { + doFirst { + project.delete(keystore.parentFile) + keystore.parentFile.mkdirs() + } + executable = 'keytool' + standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8')) + args '-genkey', + '-alias', 'test-node', + '-keystore', keystore, + '-keyalg', 'RSA', + '-keysize', '2048', + '-validity', '712', + '-ext', 'san=dns:localhost,ip:127.0.0.1', + '-storepass', 'keypass' +} + +task createCertificate(type: LoggedExec, dependsOn: createKey) { + doFirst { + project.delete(cert.parentFile) + cert.parentFile.mkdirs() + } + executable = 'keytool' + standardInput = new ByteArrayInputStream('keypass\n'.getBytes('UTF-8')) + args '-certreq', + '-alias', 'test-node', + '-keystore', keystore, + '-file', cert, + '-keyalg', 'RSA', + '-ext', 'san=dns:localhost,ip:127.0.0.1' +} + +task createCertificateAuthority(type: LoggedExec) { + doFirst { + project.delete(ca) + ca.mkdirs() + for (String dir : ['private', 'certs', 'conf']) { + new File(ca, dir).mkdirs() + } + caConfig.setText(caConfigData, 'UTF-8') + new File(ca, 'serial').setText('01', 'UTF-8') + new File(ca, 'index.txt').setText('', 'UTF-8') + } + executable = 'openssl' + args 'req', '-new', '-x509', '-extensions', 'v3_ca', + '-keyout', new File(ca, 'private/cakey.pem'), + '-out', new File(ca, 'certs/cacert.pem'), + '-days', '1460', + '-config', caConfig, + '-subj', '/OU=XPlugins QA', + '-passout', 'pass:capass' +} + +task signCertificate(type: LoggedExec, dependsOn: [createCertificate, createCertificateAuthority]) { + executable = 'openssl' + standardInput = new ByteArrayInputStream('y\ny\n'.getBytes('UTF-8')) + args 'ca', '-in', cert, '-notext', '-out', signedCert, '-config', caConfig, + '-extensions', 'v3_req', '-passin', 'pass:capass' +} + +task importCertificate(type: LoggedExec, dependsOn: signCertificate) { + executable = 'keytool' + standardInput = new ByteArrayInputStream('keypass\nyes\n'.getBytes('UTF-8')) + args '-importcert', '-keystore', keystore, '-file', signedCert, '-trustcacerts' +} // add keystore to test classpath: it expects it there sourceSets.test.resources.srcDir(keystore.parentFile) +processTestResources.dependsOn(importCertificate) -configurations { - antcontrib { - description = 'ant-contrib' - transitive = false - } -} - -dependencies { - antcontrib "ant-contrib:ant-contrib:1.0b3" -} - -// this loop must be outside of a configuration closure, otherwise it may get executed multiple times +// add ES plugins, this loop must be outside of a configuration closure, otherwise it may get executed multiple times for (Project subproj : project.rootProject.subprojects) { if (subproj.path.startsWith(':plugins:')) { // need to get a non-decorated project object, so must re-lookup the project by path @@ -36,38 +156,15 @@ for (Project subproj : project.rootProject.subprojects) { } } -// we should be able to taskdef, but gradle has *the worst* classloader management -// so just do a hack, jam ant-contrib directly into gradle's ant's classloader -ClassLoader antClassLoader = org.apache.tools.ant.Project.class.classLoader -configurations.antcontrib.each { File f -> - antClassLoader.addURL(f.toURI().toURL()) -} - -// suck in ssl-setup.xml, defining matching tasks in gradle -ant.property(name: 'integ.scratch', location: project.buildDir) -ant.property(name: 'keystore.path', keystore) -ant.importBuild 'ssl-setup.xml' - -// clean all intermediate/keystore files before regenerating it -task cleanKeystore(type: Delete) { - delete new File(project.buildDir, 'keystore'), - new File(project.buildDir, 'cert'), - new File(project.buildDir, 'ca') -} - -// wipe and regenerate keystore so its available as a test dep -processTestResources.dependsOn('cleanKeystore') -processTestResources.dependsOn('generate-keystore') - integTest { cluster { // TODO: use some variable here for port number systemProperty 'es.marvel.agent.exporter.es.hosts', 'https://marvel_export:changeme@localhost:9400' - systemProperty 'es.marvel.agent.exporter.es.ssl.truststore.path', 'test-node.jks' + systemProperty 'es.marvel.agent.exporter.es.ssl.truststore.path', keystore.name systemProperty 'es.marvel.agent.exporter.es.ssl.truststore.password', 'keypass' systemProperty 'es.shield.transport.ssl', 'true' systemProperty 'es.shield.http.ssl', 'true' - systemProperty 'es.shield.ssl.keystore.path', 'test-node.jks' + systemProperty 'es.shield.ssl.keystore.path', keystore.name systemProperty 'es.shield.ssl.keystore.password', 'keypass' plugin 'licence', project(':x-plugins:license:plugin') plugin 'shield', project(':x-plugins:shield') @@ -75,7 +172,7 @@ integTest { plugin 'marvel-agent', project(':x-plugins:marvel') // copy keystore into config/ - extraConfigFile 'test-node.jks', keystore + extraConfigFile keystore.name, keystore setupCommand 'setupTestUser', 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' setupCommand 'setupMarvelUser', diff --git a/qa/smoke-test-plugins-ssl/ssl-setup.xml b/qa/smoke-test-plugins-ssl/ssl-setup.xml deleted file mode 100644 index b30fbb8023a..00000000000 --- a/qa/smoke-test-plugins-ssl/ssl-setup.xml +++ /dev/null @@ -1,170 +0,0 @@ - - - - - - - - - - 01 - - [ ca ] -default_ca = CA_default -[ CA_default ] -copy_extensions = copy -dir = ${integ.scratch}/ca -serial = $dir/serial -database = $dir/index.txt -new_certs_dir = $dir/certs -certificate = $dir/certs/cacert.pem -private_key = $dir/private/cakey.pem -default_days = 712 -default_md = sha256 -preserve = no -email_in_dn = no -x509_extensions = v3_ca -name_opt = ca_default -cert_opt = ca_default -policy = policy_anything -[ policy_anything ] -countryName = optional -stateOrProvinceName = optional -localityName = optional -organizationName = optional -organizationalUnitName = optional -commonName = supplied -emailAddress = optional -[ req ] -default_bits = 2048 # Size of keys -default_keyfile = key.pem # name of generated keys -default_md = sha256 # message digest algorithm -string_mask = nombstr # permitted characters -distinguished_name = req_distinguished_name -req_extensions = v3_req -[ req_distinguished_name ] -# Variable name Prompt string -#------------------------- ---------------------------------- -0.organizationName = Organization Name (company) -organizationalUnitName = Organizational Unit Name (department, division) -emailAddress = Email Address -emailAddress_max = 40 -localityName = Locality Name (city, district) -stateOrProvinceName = State or Province Name (full name) -countryName = Country Name (2 letter code) -countryName_min = 2 -countryName_max = 2 -commonName = Common Name (hostname, IP, or your name) -commonName_max = 64 -# Default values for the above, for consistency and less typing. -# Variable name Value -#------------------------ ------------------------------ -0.organizationName_default = Elasticsearch Test Org -localityName_default = Amsterdam -stateOrProvinceName_default = Amsterdam -countryName_default = NL -emailAddress_default = cacerttest@YOUR.COMPANY.TLD -[ v3_ca ] -basicConstraints = CA:TRUE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always,issuer:always -[ v3_req ] -basicConstraints = CA:FALSE -subjectKeyIdentifier = hash - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From b9b39efac8c484558139a4b33e9572812ae535b2 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Fri, 27 Nov 2015 10:23:03 +0100 Subject: [PATCH 64/95] Disable SSL integration tests temporarily We disable SSL integration tests as a workaround for https://github.com/elastic/infra/issues/628 to ensure other problems can still be caught. Original commit: elastic/x-pack-elasticsearch@47bf56faecb6060860b6fe89790fc8f487afa9e7 --- qa/smoke-test-plugins-ssl/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/smoke-test-plugins-ssl/build.gradle b/qa/smoke-test-plugins-ssl/build.gradle index 58afb5e9dfc..f833c9e84e6 100644 --- a/qa/smoke-test-plugins-ssl/build.gradle +++ b/qa/smoke-test-plugins-ssl/build.gradle @@ -156,6 +156,7 @@ for (Project subproj : project.rootProject.subprojects) { } } +/* integTest { cluster { // TODO: use some variable here for port number @@ -183,3 +184,4 @@ integTest { } } } +*/ \ No newline at end of file From 75d66dfa955dea615b17e6d962c6cbdc9aff02b0 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Fri, 27 Nov 2015 14:40:29 +0100 Subject: [PATCH 65/95] Add SettingsFilter to SettingsModule in tests (change in core) Original commit: elastic/x-pack-elasticsearch@004edbcf767b99288a311b59125fbc86d840b127 --- .../shield/audit/AuditTrailModuleTests.java | 9 +++++---- .../shield/authc/AuthenticationModuleTests.java | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java b/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java index 9e2e3065323..b82b8ab6f71 100644 --- a/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.inject.Guice; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.indices.breaker.CircuitBreakerModule; import org.elasticsearch.shield.audit.logfile.LoggingAuditTrail; @@ -30,7 +31,7 @@ public class AuditTrailModuleTests extends ESTestCase { .put("client.type", "node") .put("shield.audit.enabled", false) .build(); - Injector injector = Guice.createInjector(new SettingsModule(settings), new AuditTrailModule(settings)); + Injector injector = Guice.createInjector(new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings)); AuditTrail auditTrail = injector.getInstance(AuditTrail.class); assertThat(auditTrail, is(AuditTrail.NOOP)); } @@ -38,7 +39,7 @@ public class AuditTrailModuleTests extends ESTestCase { public void testDisabledByDefault() throws Exception { Settings settings = Settings.builder() .put("client.type", "node").build(); - Injector injector = Guice.createInjector(new SettingsModule(settings), new AuditTrailModule(settings)); + Injector injector = Guice.createInjector(new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings)); AuditTrail auditTrail = injector.getInstance(AuditTrail.class); assertThat(auditTrail, is(AuditTrail.NOOP)); } @@ -50,7 +51,7 @@ public class AuditTrailModuleTests extends ESTestCase { .build(); ThreadPool pool = new ThreadPool("testLogFile"); try { - Injector injector = Guice.createInjector(new SettingsModule(settings), new AuditTrailModule(settings), new TransportModule(settings), new CircuitBreakerModule(settings), new ThreadPoolModule(pool), new Version.Module(Version.CURRENT)); + Injector injector = Guice.createInjector(new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings), new TransportModule(settings), new CircuitBreakerModule(settings), new ThreadPoolModule(pool), new Version.Module(Version.CURRENT)); AuditTrail auditTrail = injector.getInstance(AuditTrail.class); assertThat(auditTrail, instanceOf(AuditTrailService.class)); AuditTrailService service = (AuditTrailService) auditTrail; @@ -69,7 +70,7 @@ public class AuditTrailModuleTests extends ESTestCase { .put("client.type", "node") .build(); try { - Guice.createInjector(new SettingsModule(settings), new AuditTrailModule(settings)); + Guice.createInjector(new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings)); fail("Expect initialization to fail when an unknown audit trail output is configured"); } catch (Throwable t) { // expected diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/AuthenticationModuleTests.java b/shield/src/test/java/org/elasticsearch/shield/authc/AuthenticationModuleTests.java index 40eea778378..e9d74f77d2a 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authc/AuthenticationModuleTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authc/AuthenticationModuleTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.common.inject.Guice; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; @@ -78,7 +79,7 @@ public class AuthenticationModuleTests extends ESTestCase { Environment env = new Environment(settings); ThreadPool pool = new ThreadPool(settings); try { - Injector injector = Guice.createInjector(module, new SettingsModule(settings), new AuditTrailModule(settings), new CryptoModule(settings), new EnvironmentModule(env), new ThreadPoolModule(pool)); + Injector injector = Guice.createInjector(module, new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings), new CryptoModule(settings), new EnvironmentModule(env), new ThreadPoolModule(pool)); Realms realms = injector.getInstance(Realms.class); Realm.Factory factory = realms.realmFactory("custom"); assertThat(factory, notNullValue()); @@ -106,7 +107,7 @@ public class AuthenticationModuleTests extends ESTestCase { ThreadPool pool = new ThreadPool(settings); try { - Injector injector = Guice.createInjector(module, new SettingsModule(settings), new AuditTrailModule(settings), new CryptoModule(settings), new EnvironmentModule(env), new ThreadPoolModule(pool)); + Injector injector = Guice.createInjector(module, new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings), new CryptoModule(settings), new EnvironmentModule(env), new ThreadPoolModule(pool)); AuthenticationFailureHandler failureHandler = injector.getInstance(AuthenticationFailureHandler.class); assertThat(failureHandler, notNullValue()); assertThat(failureHandler, instanceOf(DefaultAuthenticationFailureHandler.class)); @@ -130,7 +131,7 @@ public class AuthenticationModuleTests extends ESTestCase { ThreadPool pool = new ThreadPool(settings); try { - Injector injector = Guice.createInjector(module, new SettingsModule(settings), new AuditTrailModule(settings), new CryptoModule(settings), new EnvironmentModule(env), new ThreadPoolModule(pool)); + Injector injector = Guice.createInjector(module, new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings), new CryptoModule(settings), new EnvironmentModule(env), new ThreadPoolModule(pool)); AuthenticationFailureHandler failureHandler = injector.getInstance(AuthenticationFailureHandler.class); assertThat(failureHandler, notNullValue()); assertThat(failureHandler, instanceOf(NoOpFailureHandler.class)); From f1b32117ac0730f90135c335a885d7b07835e071 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Fri, 27 Nov 2015 15:52:51 +0100 Subject: [PATCH 66/95] Add dependency on NetworkModule in AuditTrailModuleTests Original commit: elastic/x-pack-elasticsearch@d5c0429d33f031ce9766fc43d1c1f0045bb7eb8f --- .../shield/audit/AuditTrailModuleTests.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java b/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java index b82b8ab6f71..207cc8b4aac 100644 --- a/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java @@ -8,6 +8,8 @@ package org.elasticsearch.shield.audit; import org.elasticsearch.Version; import org.elasticsearch.common.inject.Guice; import org.elasticsearch.common.inject.Injector; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; @@ -51,7 +53,15 @@ public class AuditTrailModuleTests extends ESTestCase { .build(); ThreadPool pool = new ThreadPool("testLogFile"); try { - Injector injector = Guice.createInjector(new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings), new TransportModule(settings), new CircuitBreakerModule(settings), new ThreadPoolModule(pool), new Version.Module(Version.CURRENT)); + Injector injector = Guice.createInjector( + new SettingsModule(settings, new SettingsFilter(settings)), + new AuditTrailModule(settings), + new TransportModule(settings), + new CircuitBreakerModule(settings), + new ThreadPoolModule(pool), + new Version.Module(Version.CURRENT), + new NetworkModule(new NetworkService(settings)) + ); AuditTrail auditTrail = injector.getInstance(AuditTrail.class); assertThat(auditTrail, instanceOf(AuditTrailService.class)); AuditTrailService service = (AuditTrailService) auditTrail; From ad697c077eb7a654bfc19ab8146d953cc5d28ac3 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Fri, 27 Nov 2015 17:25:51 +0100 Subject: [PATCH 67/95] Reenable SSL smoke tests Original commit: elastic/x-pack-elasticsearch@1969c6d0203185a598fe940a94fc9c90f93cc9da --- qa/smoke-test-plugins-ssl/build.gradle | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qa/smoke-test-plugins-ssl/build.gradle b/qa/smoke-test-plugins-ssl/build.gradle index f833c9e84e6..7cc40b4b084 100644 --- a/qa/smoke-test-plugins-ssl/build.gradle +++ b/qa/smoke-test-plugins-ssl/build.gradle @@ -13,6 +13,11 @@ File cert = new File(project.buildDir, 'cert/test-node.csr') File signedCert = new File(project.buildDir, 'cert/test-node-signed.csr') File keystore = new File(project.buildDir, 'keystore/test-node.jks') +// we touch keystore because otherwise it fails, extraConfigFile does not exist +// this tricks some broken compile-time check into just moving along: we nuke this stuff before we actually generate +keystore.parentFile.mkdirs() +keystore.createNewFile() + String caConfigData = """ [ ca ] default_ca = CA_default @@ -156,7 +161,6 @@ for (Project subproj : project.rootProject.subprojects) { } } -/* integTest { cluster { // TODO: use some variable here for port number @@ -183,5 +187,4 @@ integTest { return true } } -} -*/ \ No newline at end of file +} \ No newline at end of file From a67aebc9fcb7b5b290f290cbcd6eb3d7731afdfd Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Sat, 28 Nov 2015 18:07:37 -0800 Subject: [PATCH 68/95] Build: Remove hack to touch keystore before it is created This will be fixed in ES with elastic/elasticsearchelastic/elasticsearch#15089 Original commit: elastic/x-pack-elasticsearch@55b42a7ad43d8bb680bb928db4ac4b004d7b42cc --- qa/smoke-test-plugins-ssl/build.gradle | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/qa/smoke-test-plugins-ssl/build.gradle b/qa/smoke-test-plugins-ssl/build.gradle index 7cc40b4b084..c16ad8b2e7d 100644 --- a/qa/smoke-test-plugins-ssl/build.gradle +++ b/qa/smoke-test-plugins-ssl/build.gradle @@ -13,11 +13,6 @@ File cert = new File(project.buildDir, 'cert/test-node.csr') File signedCert = new File(project.buildDir, 'cert/test-node-signed.csr') File keystore = new File(project.buildDir, 'keystore/test-node.jks') -// we touch keystore because otherwise it fails, extraConfigFile does not exist -// this tricks some broken compile-time check into just moving along: we nuke this stuff before we actually generate -keystore.parentFile.mkdirs() -keystore.createNewFile() - String caConfigData = """ [ ca ] default_ca = CA_default @@ -187,4 +182,4 @@ integTest { return true } } -} \ No newline at end of file +} From 171179d91fc6bfdccc02545a24adeab05171523c Mon Sep 17 00:00:00 2001 From: javanna Date: Mon, 30 Nov 2015 12:12:31 +0100 Subject: [PATCH 69/95] [TEST] non stored fields are not returned anymore via fields Relates to https://github.com/elastic/elasticsearch/issues/14489 Original commit: elastic/x-pack-elasticsearch@2897dc5df7f1443237c2940057bea375b82425fd --- .../java/org/elasticsearch/shield/audit/IndexAuditIT.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qa/shield-audit-tests/src/test/java/org/elasticsearch/shield/audit/IndexAuditIT.java b/qa/shield-audit-tests/src/test/java/org/elasticsearch/shield/audit/IndexAuditIT.java index 1c6b3bdb3a3..e899db8d3e1 100644 --- a/qa/shield-audit-tests/src/test/java/org/elasticsearch/shield/audit/IndexAuditIT.java +++ b/qa/shield-audit-tests/src/test/java/org/elasticsearch/shield/audit/IndexAuditIT.java @@ -42,10 +42,9 @@ public class IndexAuditIT extends ESIntegTestCase { assertThat(found, is(true)); - SearchResponse searchResponse = client().prepareSearch(".shield_audit_log*").setQuery(QueryBuilders.matchQuery("principal", USER)).addField("principal").get(); + SearchResponse searchResponse = client().prepareSearch(".shield_audit_log*").setQuery(QueryBuilders.matchQuery("principal", USER)).get(); assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); - assertThat((String) searchResponse.getHits().getAt(0).field("principal").getValue(), is(USER)); - + assertThat((String) searchResponse.getHits().getAt(0).sourceAsMap().get("principal"), is(USER)); } @Override From 8a22ba0a0871f851537013dbaf5e4d4324e3b1c6 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 30 Nov 2015 08:42:25 -0500 Subject: [PATCH 70/95] smoke-test-plugins-ssl shoudl check if openssl is available Today some jenkins servers dont have it (e.g. windows), and it constantly fails... Original commit: elastic/x-pack-elasticsearch@6b561c73e0b8ef320dcf136b84c0b97dac5f913d --- qa/smoke-test-plugins-ssl/build.gradle | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qa/smoke-test-plugins-ssl/build.gradle b/qa/smoke-test-plugins-ssl/build.gradle index c16ad8b2e7d..4f05bcc556d 100644 --- a/qa/smoke-test-plugins-ssl/build.gradle +++ b/qa/smoke-test-plugins-ssl/build.gradle @@ -157,6 +157,14 @@ for (Project subproj : project.rootProject.subprojects) { } integTest { + // in some environments, openssl might not be available + try { + int ret = Runtime.getRuntime().exec("openssl version").waitFor(); + enabled = (ret == 0); + } catch (IOException unavailable) { + enabled = false; + } + cluster { // TODO: use some variable here for port number systemProperty 'es.marvel.agent.exporter.es.hosts', 'https://marvel_export:changeme@localhost:9400' From 9df905ff19a401bb4979dac5a3576ec9f577b5f5 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 30 Nov 2015 09:28:16 -0500 Subject: [PATCH 71/95] Simplify SSL test to not use openssl. I think the intent here is to just test that our SSL layers work, not invoke a long chain of keytool + openssl commands. This simplifies the build and will work on windows. Original commit: elastic/x-pack-elasticsearch@af07d0d4f7c1ef26d70dd96265dbad6fe761b183 --- qa/smoke-test-plugins-ssl/build.gradle | 127 +------------------------ 1 file changed, 1 insertion(+), 126 deletions(-) diff --git a/qa/smoke-test-plugins-ssl/build.gradle b/qa/smoke-test-plugins-ssl/build.gradle index 4f05bcc556d..30f361cd459 100644 --- a/qa/smoke-test-plugins-ssl/build.gradle +++ b/qa/smoke-test-plugins-ssl/build.gradle @@ -7,76 +7,8 @@ dependencies { } // location of keystore and files to generate it -File ca = new File(project.buildDir, 'ca') -File caConfig = new File(ca, 'conf/caconfig.cnf') -File cert = new File(project.buildDir, 'cert/test-node.csr') -File signedCert = new File(project.buildDir, 'cert/test-node-signed.csr') File keystore = new File(project.buildDir, 'keystore/test-node.jks') -String caConfigData = """ -[ ca ] -default_ca = CA_default -[ CA_default ] -copy_extensions = copy -serial = ${ca}/serial -database = ${ca}/index.txt -new_certs_dir = ${ca}/certs -certificate = ${ca}/certs/cacert.pem -private_key = ${ca}/private/cakey.pem -default_days = 712 -default_md = sha256 -preserve = no -email_in_dn = no -x509_extensions = v3_ca -name_opt = ca_default -cert_opt = ca_default -policy = policy_anything -[ policy_anything ] -countryName = optional -stateOrProvinceName = optional -localityName = optional -organizationName = optional -organizationalUnitName = optional -commonName = supplied -emailAddress = optional -[ req ] -default_bits = 2048 # Size of keys -default_keyfile = key.pem # name of generated keys -default_md = sha256 # message digest algorithm -string_mask = nombstr # permitted characters -distinguished_name = req_distinguished_name -req_extensions = v3_req -[ req_distinguished_name ] -# Variable name Prompt string -#------------------------- ---------------------------------- -0.organizationName = Organization Name (company) -organizationalUnitName = Organizational Unit Name (department, division) -emailAddress = Email Address -emailAddress_max = 40 -localityName = Locality Name (city, district) -stateOrProvinceName = State or Province Name (full name) -countryName = Country Name (2 letter code) -countryName_min = 2 -countryName_max = 2 -commonName = Common Name (hostname, IP, or your name) -commonName_max = 64 -# Default values for the above, for consistency and less typing. -# Variable name Value -#------------------------ ------------------------------ -0.organizationName_default = Elasticsearch Test Org -localityName_default = Amsterdam -stateOrProvinceName_default = Amsterdam -countryName_default = NL -emailAddress_default = cacerttest@YOUR.COMPANY.TLD -[ v3_ca ] -basicConstraints = CA:TRUE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always,issuer:always -[ v3_req ] -basicConstraints = CA:FALSE -subjectKeyIdentifier = hash -""" - // generate the keystore task createKey(type: LoggedExec) { doFirst { @@ -95,58 +27,9 @@ task createKey(type: LoggedExec) { '-storepass', 'keypass' } -task createCertificate(type: LoggedExec, dependsOn: createKey) { - doFirst { - project.delete(cert.parentFile) - cert.parentFile.mkdirs() - } - executable = 'keytool' - standardInput = new ByteArrayInputStream('keypass\n'.getBytes('UTF-8')) - args '-certreq', - '-alias', 'test-node', - '-keystore', keystore, - '-file', cert, - '-keyalg', 'RSA', - '-ext', 'san=dns:localhost,ip:127.0.0.1' -} - -task createCertificateAuthority(type: LoggedExec) { - doFirst { - project.delete(ca) - ca.mkdirs() - for (String dir : ['private', 'certs', 'conf']) { - new File(ca, dir).mkdirs() - } - caConfig.setText(caConfigData, 'UTF-8') - new File(ca, 'serial').setText('01', 'UTF-8') - new File(ca, 'index.txt').setText('', 'UTF-8') - } - executable = 'openssl' - args 'req', '-new', '-x509', '-extensions', 'v3_ca', - '-keyout', new File(ca, 'private/cakey.pem'), - '-out', new File(ca, 'certs/cacert.pem'), - '-days', '1460', - '-config', caConfig, - '-subj', '/OU=XPlugins QA', - '-passout', 'pass:capass' -} - -task signCertificate(type: LoggedExec, dependsOn: [createCertificate, createCertificateAuthority]) { - executable = 'openssl' - standardInput = new ByteArrayInputStream('y\ny\n'.getBytes('UTF-8')) - args 'ca', '-in', cert, '-notext', '-out', signedCert, '-config', caConfig, - '-extensions', 'v3_req', '-passin', 'pass:capass' -} - -task importCertificate(type: LoggedExec, dependsOn: signCertificate) { - executable = 'keytool' - standardInput = new ByteArrayInputStream('keypass\nyes\n'.getBytes('UTF-8')) - args '-importcert', '-keystore', keystore, '-file', signedCert, '-trustcacerts' -} - // add keystore to test classpath: it expects it there sourceSets.test.resources.srcDir(keystore.parentFile) -processTestResources.dependsOn(importCertificate) +processTestResources.dependsOn(createKey) // add ES plugins, this loop must be outside of a configuration closure, otherwise it may get executed multiple times for (Project subproj : project.rootProject.subprojects) { @@ -157,14 +40,6 @@ for (Project subproj : project.rootProject.subprojects) { } integTest { - // in some environments, openssl might not be available - try { - int ret = Runtime.getRuntime().exec("openssl version").waitFor(); - enabled = (ret == 0); - } catch (IOException unavailable) { - enabled = false; - } - cluster { // TODO: use some variable here for port number systemProperty 'es.marvel.agent.exporter.es.hosts', 'https://marvel_export:changeme@localhost:9400' From acff7a8f4c529a0e18ec54389478aa48b5e54730 Mon Sep 17 00:00:00 2001 From: Court Ewing Date: Mon, 30 Nov 2015 10:32:09 -0500 Subject: [PATCH 72/95] Add field_stats to kibana user role in shield Kibana 4.3 now uses the field_stats api to pre-flight all search requests to any configured index patterns, so shield needs to allow it. Original commit: elastic/x-pack-elasticsearch@793ad01424d1a97cefa1e77ffc1f6e6c5c6c615b --- shield/config/shield/roles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shield/config/shield/roles.yml b/shield/config/shield/roles.yml index 5ae98201e3b..b08330e4d9f 100644 --- a/shield/config/shield/roles.yml +++ b/shield/config/shield/roles.yml @@ -32,7 +32,7 @@ kibana4: - cluster:monitor/health indices: '*': - privileges: indices:admin/mappings/fields/get, indices:admin/validate/query, indices:data/read/search, indices:data/read/msearch, indices:admin/get + privileges: indices:admin/mappings/fields/get, indices:admin/validate/query, indices:data/read/search, indices:data/read/msearch, indices:data/read/field_stats, indices:admin/get '.kibana': privileges: indices:admin/exists, indices:admin/mapping/put, indices:admin/mappings/fields/get, indices:admin/refresh, indices:admin/validate/query, indices:data/read/get, indices:data/read/mget, indices:data/read/search, indices:data/write/delete, indices:data/write/index, indices:data/write/update From 445e81bb7040e33cad8068b037f95f921cf41d3d Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 1 Dec 2015 16:03:42 +0100 Subject: [PATCH 73/95] Marvel: Mute MultiNodesStatsTests Original commit: elastic/x-pack-elasticsearch@f0adbf3e2035af06490fede53c7f3e20188e1500 --- .../marvel/agent/renderer/node/MultiNodesStatsTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java index 5d6ebfe54b5..0b3d3674232 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.marvel.agent.renderer.node; +import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; @@ -20,6 +21,7 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; import static org.hamcrest.Matchers.greaterThan; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/960") @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class MultiNodesStatsTests extends MarvelIntegTestCase { From 828097b28108b35bfcfb69d58dec2c75624f130d Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 1 Dec 2015 14:11:40 -0500 Subject: [PATCH 74/95] fix watcher to compile with full jdk for now Original commit: elastic/x-pack-elasticsearch@09897a26eb281c89c1369e3e60cf181b6b4c15c0 --- watcher/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/watcher/build.gradle b/watcher/build.gradle index bf34dd721bd..a6d0b10b024 100644 --- a/watcher/build.gradle +++ b/watcher/build.gradle @@ -12,6 +12,9 @@ ext.versions = [ okhttp: '2.3.0' ] +// TODO: fix this! https://github.com/elastic/x-plugins/issues/1066 +ext.compactProfile = 'full' + dependencies { provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') provided project(path: ':x-plugins:shield', configuration: 'runtime') From afb5cc0d08ff7902fd6b9f8adb394b63a0009a2c Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 25 Nov 2015 19:29:35 +0100 Subject: [PATCH 75/95] Stop using onIndexService. Now that mappers are registered at the index level, Shield is the last user of Plugin.onIndexService. Yet this can be implemented on the IndexModule by registering a listener on index creation. Once this change is merged, we can remove Plugin.onIndexService entirely. Original commit: elastic/x-pack-elasticsearch@fba68099498818f1b0fda6a77cda6b85721b9fcb --- .../org/elasticsearch/shield/ShieldPlugin.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java index 095b971ccd1..2f5ac7334a2 100644 --- a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java +++ b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java @@ -18,6 +18,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.http.HttpServerModule; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestModule; import org.elasticsearch.shield.action.ShieldActionFilter; @@ -51,7 +52,6 @@ import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport; import org.elasticsearch.shield.transport.netty.ShieldNettyTransport; import org.elasticsearch.transport.TransportModule; -import java.io.Closeable; import java.nio.file.Path; import java.util.*; import java.security.AccessController; @@ -179,13 +179,6 @@ public class ShieldPlugin extends Plugin { clusterDynamicSettingsModule.registerClusterDynamicSetting(IPFilter.IP_FILTER_ENABLED_HTTP_SETTING, Validator.EMPTY); } - @Override - public void onIndexService(IndexService indexService) { - if (enabled && clientMode == false) { - failIfShieldQueryCacheIsNotActive(indexService.getIndexSettings().getSettings(), false); - } - } - @Override public void onIndexModule(IndexModule module) { if (enabled == false) { @@ -199,6 +192,12 @@ public class ShieldPlugin extends Plugin { } if (clientMode == false) { module.registerQueryCache(ShieldPlugin.OPT_OUT_QUERY_CACHE, OptOutQueryCache::new); + module.addIndexEventListener(new IndexEventListener() { + @Override + public void afterIndexCreated(IndexService indexService) { + failIfShieldQueryCacheIsNotActive(indexService.getIndexSettings().getSettings(), false); + } + }); } } From b43f81a15c34c396cc7881bef87a52ebd3a7b321 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 2 Dec 2015 14:56:34 +0100 Subject: [PATCH 76/95] Use IndexModule.getSettings() to get the settings. Original commit: elastic/x-pack-elasticsearch@770f859b6cadf4c93d0069567bc5e80ee48ce1dd --- .../main/java/org/elasticsearch/shield/ShieldPlugin.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java index 2f5ac7334a2..fc7a9a2e0a4 100644 --- a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java +++ b/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java @@ -192,12 +192,7 @@ public class ShieldPlugin extends Plugin { } if (clientMode == false) { module.registerQueryCache(ShieldPlugin.OPT_OUT_QUERY_CACHE, OptOutQueryCache::new); - module.addIndexEventListener(new IndexEventListener() { - @Override - public void afterIndexCreated(IndexService indexService) { - failIfShieldQueryCacheIsNotActive(indexService.getIndexSettings().getSettings(), false); - } - }); + failIfShieldQueryCacheIsNotActive(module.getSettings(), false); } } From 8ea7a62b862bb924d9349de9119aba8618f5259e Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 2 Dec 2015 10:44:23 -0500 Subject: [PATCH 77/95] Remove and forbid use of the type-unsafe empty Collections fields This commit removes and now forbids all uses of the type-unsafe empty Collections fields Collections#EMPTY_LIST, Collections#EMPTY_MAP, and Collections#EMPTY_SET. The type-safe methods Collections#emptyList, Collections#emptyMap, and Collections#emptySet should be used instead. Relates elastic/elasticsearchelastic/elasticsearch#15187 Original commit: elastic/x-pack-elasticsearch@99f6fdd3a67f0b577014004e10d0bb620bfef7df --- .../org/elasticsearch/shield/transport/filter/IPFilter.java | 4 ++-- .../elasticsearch/watcher/execution/TriggeredWatchStore.java | 2 +- .../main/java/org/elasticsearch/watcher/support/Script.java | 2 +- .../org/elasticsearch/watcher/support/text/TextTemplate.java | 2 +- .../src/main/java/org/elasticsearch/watcher/watch/Watch.java | 2 +- .../actions/slack/service/message/SlackMessageTests.java | 2 +- .../test/java/org/elasticsearch/watcher/watch/WatchTests.java | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/filter/IPFilter.java b/shield/src/main/java/org/elasticsearch/shield/transport/filter/IPFilter.java index c565fa98612..a453f993792 100644 --- a/shield/src/main/java/org/elasticsearch/shield/transport/filter/IPFilter.java +++ b/shield/src/main/java/org/elasticsearch/shield/transport/filter/IPFilter.java @@ -76,7 +76,7 @@ public class IPFilter extends AbstractLifecycleComponent { private final Transport transport; private final ShieldLicenseState licenseState; private final boolean alwaysAllowBoundAddresses; - private Map rules = Collections.EMPTY_MAP; + private Map rules = Collections.emptyMap(); private HttpServerTransport httpServerTransport = null; @Inject @@ -156,7 +156,7 @@ public class IPFilter extends AbstractLifecycleComponent { boolean isHttpFilterEnabled = settings.getAsBoolean(IP_FILTER_ENABLED_HTTP_SETTING, isIpFilterEnabled); if (!isIpFilterEnabled && !isHttpFilterEnabled) { - return Collections.EMPTY_MAP; + return Collections.emptyMap(); } Map profileRules = new HashMap<>(); diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java b/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java index 7677eb0a253..d7a4d8cd9d6 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java @@ -132,7 +132,7 @@ public class TriggeredWatchStore extends AbstractComponent { public void putAll(final List triggeredWatches, final ActionListener> listener) throws Exception { if (triggeredWatches.isEmpty()) { - listener.onResponse(Collections.EMPTY_LIST); + listener.onResponse(Collections.emptyList()); return; } diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/Script.java b/watcher/src/main/java/org/elasticsearch/watcher/support/Script.java index 59a54e83c3a..c102414f10c 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/support/Script.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/support/Script.java @@ -56,7 +56,7 @@ public class Script implements ToXContent { } public Map params() { - return params != null ? params : Collections.EMPTY_MAP; + return params != null ? params : Collections.emptyMap(); } @Override diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplate.java b/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplate.java index 02b0d5993c1..e4f24d66ff7 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplate.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplate.java @@ -51,7 +51,7 @@ public class TextTemplate implements ToXContent { } public Map getParams() { - return params != null ? params : Collections.EMPTY_MAP; + return params != null ? params : Collections.emptyMap(); } @Override diff --git a/watcher/src/main/java/org/elasticsearch/watcher/watch/Watch.java b/watcher/src/main/java/org/elasticsearch/watcher/watch/Watch.java index 211ed3e56b7..7b7264ef09b 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/watch/Watch.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/watch/Watch.java @@ -230,7 +230,7 @@ public class Watch implements TriggerEngine.Job, ToXContent { this.secretService = secretService; this.defaultInput = new ExecutableNoneInput(logger); this.defaultCondition = new ExecutableAlwaysCondition(logger); - this.defaultActions = new ExecutableActions(Collections.EMPTY_LIST); + this.defaultActions = new ExecutableActions(Collections.emptyList()); this.clock = clock; } diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageTests.java b/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageTests.java index 486a4a440c6..1a4d2736f9e 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageTests.java @@ -450,7 +450,7 @@ public class SlackMessageTests extends ESTestCase { SlackMessage.Template template = templateBuilder.build(); - SlackMessage message = template.render("_w1", "_a1", engine, Collections.EMPTY_MAP, defaults); + SlackMessage message = template.render("_w1", "_a1", engine, Collections.emptyMap(), defaults); assertThat(message, notNullValue()); if (template.from != null) { assertThat(message.from, is(template.from.getTemplate())); diff --git a/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchTests.java b/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchTests.java index 8e6ab9e37dc..5c1e4c58c0b 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchTests.java @@ -260,7 +260,7 @@ public class WatchTests extends ESTestCase { ConditionRegistry conditionRegistry = registry(new ExecutableAlwaysCondition(logger)); InputRegistry inputRegistry = registry(new ExecutableNoneInput(logger)); TransformRegistry transformRegistry = transformRegistry(); - ExecutableActions actions = new ExecutableActions(Collections.EMPTY_LIST); + ExecutableActions actions = new ExecutableActions(Collections.emptyList()); ActionRegistry actionRegistry = registry(actions, transformRegistry); XContentBuilder builder = XContentFactory.jsonBuilder(); From 83c2f1b3c441deff7aee5de28663fb3ceb78ba46 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 2 Dec 2015 11:05:16 -0800 Subject: [PATCH 78/95] Build: Disable licenses checks for all xplugins Original commit: elastic/x-pack-elasticsearch@f6a29344e953e5ab9c20ec56018cf0290b221ac7 --- marvel/build.gradle | 2 ++ shield/build.gradle | 2 ++ watcher/build.gradle | 2 ++ 3 files changed, 6 insertions(+) diff --git a/marvel/build.gradle b/marvel/build.gradle index 7eda0dacc35..a58367779e3 100644 --- a/marvel/build.gradle +++ b/marvel/build.gradle @@ -49,6 +49,8 @@ integTest { } } +dependencyLicenses.enabled = false + bundlePlugin { from(projectDir) { include 'LICENSE.txt' diff --git a/shield/build.gradle b/shield/build.gradle index 0d2c5999458..38c142f9ba4 100644 --- a/shield/build.gradle +++ b/shield/build.gradle @@ -27,6 +27,8 @@ compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,- // no integ tests... integTest.enabled = false +dependencyLicenses.enabled = false + // TODO: standardize packaging config for plugins bundlePlugin { from(projectDir) { diff --git a/watcher/build.gradle b/watcher/build.gradle index a6d0b10b024..7d2c5d7e4c3 100644 --- a/watcher/build.gradle +++ b/watcher/build.gradle @@ -59,6 +59,8 @@ bundlePlugin { } } +dependencyLicenses.enabled = false + // TODO: don't publish test artifacts just to run messy tests, fix the tests! // https://github.com/elastic/x-plugins/issues/724 configurations { From e5b0e7f5cb8f3a6bb17614aafdb8d1b086816fa3 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 25 Nov 2015 10:18:08 -0500 Subject: [PATCH 79/95] reorganize directory layout See elastic/elasticsearch#1022 Original commit: elastic/x-pack-elasticsearch@3ee8761312524e817bd1d10f865fd17405cd0ef3 --- {qa => elasticsearch}/build.gradle | 0 elasticsearch/license/README.md | 36 + elasticsearch/license/base/build.gradle | 13 + .../license/core/CryptUtils.java | 245 +++++ .../elasticsearch/license/core/DateUtils.java | 44 + .../elasticsearch/license/core/License.java | 693 ++++++++++++++ .../license/core/LicenseVerifier.java | 69 ++ .../core/LicenseSerializationTests.java | 99 ++ .../elasticsearch/license/core/TestUtils.java | 197 ++++ .../base/src/test/resources/log4j.properties | 11 + .../base/src/test/resources/private.key | Bin 0 -> 1232 bytes .../base/src/test/resources/public.key | 3 + elasticsearch/license/build.gradle | 7 + elasticsearch/license/found-plugin/.gitignore | 1 + .../license/found-plugin/build.gradle | 19 + .../license/plugin/LicenseModule.java | 23 + .../license/plugin/LicensePlugin.java | 49 + .../plugin/core/FoundLicensesService.java | 57 ++ .../license/licensor/bin/key-pair-generator | 58 ++ .../license/licensor/bin/license-generator | 56 ++ .../license/licensor/bin/verify-license | 57 ++ elasticsearch/license/licensor/build.gradle | 9 + .../licensor/dev-tools/integration-tests.xml | 122 +++ .../license/licensor/sample/license_spec.json | 1 + .../license/licensor/LicenseSigner.java | 78 ++ .../licensor/tools/KeyPairGeneratorTool.java | 108 +++ .../licensor/tools/LicenseGeneratorTool.java | 125 +++ .../tools/LicenseVerificationTool.java | 117 +++ .../tools/licensor-key-pair-generator.help | 22 + .../tools/licensor-license-generator.help | 26 + .../tools/licensor-verify-license.help | 28 + .../licensor/LicenseVerificationTests.java | 83 ++ .../license/licensor/TestUtils.java | 214 +++++ .../tools/KeyPairGenerationToolTests.java | 95 ++ .../tools/LicenseGenerationToolTests.java | 140 +++ .../tools/LicenseVerificationToolTests.java | 154 ++++ .../src/test/resources/log4j.properties | 11 + .../licensor/src/test/resources/private.key | Bin 0 -> 1232 bytes .../licensor/src/test/resources/public.key | 3 + elasticsearch/license/plugin-api/.gitignore | 1 + elasticsearch/license/plugin-api/build.gradle | 15 + .../core/AbstractLicenseeComponent.java | 74 ++ .../license/plugin/core/LicenseState.java | 42 + .../license/plugin/core/LicenseUtils.java | 35 + .../license/plugin/core/Licensee.java | 87 ++ .../license/plugin/core/LicenseeRegistry.java | 14 + .../plugin/core/LicensesManagerService.java | 23 + .../plugin/core/LicenseUtilsTests.java | 34 + elasticsearch/license/plugin/.gitignore | 1 + .../license/plugin}/LICENSE.txt | 0 .../license/plugin}/NOTICE.txt | 0 elasticsearch/license/plugin/build.gradle | 26 + .../license/plugin/keys/dev/public.key | 3 + .../license/plugin/keys/prod/public.key | 3 + .../license/plugin/LicenseModule.java | 24 + .../license/plugin/LicensePlugin.java | 93 ++ .../action/delete/DeleteLicenseAction.java | 29 + .../action/delete/DeleteLicenseRequest.java | 36 + .../delete/DeleteLicenseRequestBuilder.java | 23 + .../action/delete/DeleteLicenseResponse.java | 35 + .../delete/TransportDeleteLicenseAction.java | 65 ++ .../plugin/action/get/GetLicenseAction.java | 29 + .../plugin/action/get/GetLicenseRequest.java | 21 + .../action/get/GetLicenseRequestBuilder.java | 21 + .../plugin/action/get/GetLicenseResponse.java | 49 + .../action/get/TransportGetLicenseAction.java | 53 ++ .../plugin/action/put/PutLicenseAction.java | 29 + .../plugin/action/put/PutLicenseRequest.java | 77 ++ .../action/put/PutLicenseRequestBuilder.java | 46 + .../plugin/action/put/PutLicenseResponse.java | 129 +++ .../action/put/TransportPutLicenseAction.java | 68 ++ .../license/plugin/core/LicensesMetaData.java | 237 +++++ .../license/plugin/core/LicensesService.java | 845 ++++++++++++++++++ .../license/plugin/core/LicensesStatus.java | 34 + .../license/plugin/core/TrialLicense.java | 63 ++ .../plugin/rest/RestDeleteLicenseAction.java | 35 + .../plugin/rest/RestGetLicenseAction.java | 68 ++ .../plugin/rest/RestPutLicenseAction.java | 46 + ...nsesConsumerPluginIntegrationTestCase.java | 151 ++++ .../AbstractLicensesIntegrationTestCase.java | 145 +++ ...esEagerConsumerPluginIntegrationTests.java | 19 + ...sesLazyConsumerPluginIntegrationTests.java | 19 + .../LicensesPluginIntegrationTests.java | 161 ++++ .../LicensesPluginsIntegrationTests.java | 137 +++ .../plugin/LicensesServiceClusterTests.java | 232 +++++ .../plugin/LicensesServiceNodeTests.java | 56 ++ .../plugin/LicensesTransportTests.java | 149 +++ .../plugin/PutLicenseResponseTests.java | 63 ++ .../license/plugin/TestUtils.java | 217 +++++ .../license/plugin/TrialLicenseTests.java | 112 +++ ...agerLicenseRegistrationConsumerPlugin.java | 40 + ...EagerLicenseRegistrationPluginService.java | 27 + ...LazyLicenseRegistrationConsumerPlugin.java | 40 + .../LazyLicenseRegistrationPluginService.java | 28 + .../consumer/TestConsumerPluginBase.java | 55 ++ .../consumer/TestPluginServiceBase.java | 104 +++ .../core/LicensesAcknowledgementTests.java | 161 ++++ .../core/LicensesExpirationCallbackTests.java | 160 ++++ .../core/LicensesExpiryNotificationTests.java | 213 +++++ .../core/LicensesManagerServiceTests.java | 138 +++ .../LicensesMetaDataSerializationTests.java | 196 ++++ .../license/plugin/rest/LicensesRestIT.java | 27 + .../src/test/resources/log4j.properties | 11 + .../plugin/src/test/resources/private.key | Bin 0 -> 1232 bytes .../rest-api-spec/api/license.get.json | 19 + .../rest-api-spec/api/license.post.json | 21 + .../rest-api-spec/test/license/10_basic.yaml | 15 + .../test/license/20_put_license.yaml | 57 ++ elasticsearch/marvel/LICENSE.txt | 439 +++++++++ elasticsearch/marvel/NOTICE.txt | 6 + {marvel => elasticsearch/marvel}/build.gradle | 6 +- .../elasticsearch/marvel/MarvelPlugin.java | 0 .../marvel/agent/AgentService.java | 0 .../agent/collector/AbstractCollector.java | 0 .../marvel/agent/collector/Collector.java | 0 .../agent/collector/CollectorModule.java | 0 .../cluster/ClusterInfoCollector.java | 0 .../cluster/ClusterInfoMarvelDoc.java | 0 .../cluster/ClusterStateCollector.java | 0 .../cluster/ClusterStateMarvelDoc.java | 0 .../cluster/ClusterStateNodeMarvelDoc.java | 0 .../cluster/ClusterStatsCollector.java | 0 .../cluster/ClusterStatsMarvelDoc.java | 0 .../cluster/DiscoveryNodeMarvelDoc.java | 0 .../indices/IndexRecoveryCollector.java | 0 .../indices/IndexRecoveryMarvelDoc.java | 0 .../indices/IndexStatsCollector.java | 0 .../indices/IndexStatsMarvelDoc.java | 0 .../indices/IndicesStatsCollector.java | 0 .../indices/IndicesStatsMarvelDoc.java | 0 .../collector/node/NodeStatsCollector.java | 0 .../collector/node/NodeStatsMarvelDoc.java | 0 .../collector/shards/ShardMarvelDoc.java | 0 .../collector/shards/ShardsCollector.java | 0 .../marvel/agent/exporter/ExportBulk.java | 0 .../marvel/agent/exporter/Exporter.java | 0 .../marvel/agent/exporter/ExporterModule.java | 0 .../marvel/agent/exporter/Exporters.java | 0 .../agent/exporter/IndexNameResolver.java | 0 .../marvel/agent/exporter/MarvelDoc.java | 0 .../agent/exporter/MarvelTemplateUtils.java | 0 .../agent/exporter/http/HttpExporter.java | 0 .../exporter/http/HttpExporterUtils.java | 0 .../agent/exporter/local/LocalBulk.java | 0 .../agent/exporter/local/LocalExporter.java | 0 .../agent/renderer/AbstractRenderer.java | 0 .../marvel/agent/renderer/Renderer.java | 0 .../marvel/agent/renderer/RendererModule.java | 0 .../agent/renderer/RendererRegistry.java | 0 .../renderer/cluster/ClusterInfoRenderer.java | 0 .../cluster/ClusterStateNodeRenderer.java | 0 .../cluster/ClusterStateRenderer.java | 0 .../cluster/ClusterStatsRenderer.java | 0 .../cluster/DiscoveryNodeRenderer.java | 0 .../indices/IndexRecoveryRenderer.java | 0 .../renderer/indices/IndexStatsRenderer.java | 0 .../indices/IndicesStatsRenderer.java | 0 .../renderer/node/NodeStatsRenderer.java | 0 .../agent/renderer/shards/ShardsRenderer.java | 0 .../marvel/agent/settings/MarvelModule.java | 0 .../marvel/agent/settings/MarvelSetting.java | 0 .../marvel/agent/settings/MarvelSettings.java | 0 .../marvel/license/LicenseModule.java | 0 .../marvel/license/MarvelLicensee.java | 0 .../shield/MarvelInternalUserHolder.java | 0 .../marvel/shield/MarvelSettingsFilter.java | 0 .../shield/MarvelShieldIntegration.java | 0 .../marvel/shield/MarvelShieldModule.java | 0 .../marvel/shield/SecuredClient.java | 0 .../marvel/support/VersionUtils.java | 0 .../main/resources/marvel_index_template.json | 0 .../org/elasticsearch/marvel/MarvelF.java | 0 .../marvel/MarvelPluginClientTests.java | 0 .../marvel/MarvelPluginTests.java | 0 .../collector/AbstractCollectorTestCase.java | 0 .../cluster/ClusterInfoCollectorTests.java | 0 .../cluster/ClusterStateCollectorTests.java | 0 .../cluster/ClusterStatsCollectorTests.java | 0 .../indices/IndexRecoveryCollectorTests.java | 0 .../indices/IndexStatsCollectorTests.java | 0 .../indices/IndicesStatsCollectorTests.java | 0 .../node/NodeStatsCollectorTests.java | 0 .../shards/ShardsCollectorTests.java | 0 .../marvel/agent/exporter/ExportersTests.java | 0 .../exporter/http/HttpExporterTests.java | 0 .../exporter/http/HttpExporterUtilsTests.java | 0 .../exporter/local/LocalExporterTests.java | 0 .../agent/renderer/RendererTestUtils.java | 0 .../renderer/cluster/ClusterInfoTests.java | 0 .../cluster/ClusterStateRendererTests.java | 0 .../renderer/cluster/ClusterStateTests.java | 0 .../cluster/ClusterStatsRendererTests.java | 0 .../renderer/cluster/ClusterStatsTests.java | 0 .../indices/IndexRecoveryRendererTests.java | 0 .../renderer/indices/IndexRecoveryTests.java | 0 .../indices/IndexStatsRendererTests.java | 0 .../renderer/indices/IndexStatsTests.java | 0 .../indices/IndicesStatsRendererTests.java | 0 .../renderer/indices/IndicesStatsTests.java | 0 .../renderer/node/MultiNodesStatsTests.java | 0 .../renderer/node/NodeStatsRendererTests.java | 0 .../agent/renderer/node/NodeStatsTests.java | 0 .../renderer/shards/ShardsRendererTests.java | 0 .../agent/renderer/shards/ShardsTests.java | 0 .../agent/settings/MarvelSettingTests.java | 0 .../agent/settings/MarvelSettingsTests.java | 0 .../license/LicenseIntegrationTests.java | 0 .../marvel/rest/MarvelRestIT.java | 0 .../marvel/support/VersionUtilsTests.java | 0 .../marvel/test/MarvelIntegTestCase.java | 0 .../src/test/resources/log4j.properties | 0 .../rest-api-spec/test/marvel/10_basic.yaml | 0 .../test/resources/samples/cluster_state.json | 0 .../test/resources/samples/cluster_stats.json | 0 .../resources/samples/index_recovery.json | 0 .../test/resources/samples/index_stats.json | 0 .../test/resources/samples/indices_stats.json | 0 .../test/resources/samples/node_stats.json | 0 .../src/test/resources/samples/shards.json | 0 .../qa/build.gradle | 0 .../build.gradle | 0 .../tests/ExecutionVarsIntegrationTests.java | 0 .../tests/GroovyManualExecutionTests.java | 0 ...HistoryTemplateTransformMappingsTests.java | 0 .../tests/IndexActionIntegrationTests.java | 0 .../messy/tests/MessyTestUtils.java | 0 .../tests/ScriptConditionSearchTests.java | 0 .../messy/tests/ScriptConditionTests.java | 0 .../tests/TransformIntegrationTests.java | 0 .../qa}/shield-audit-tests/build.gradle | 6 +- .../shield/audit/IndexAuditIT.java | 0 .../qa}/shield-client-tests/build.gradle | 6 +- .../shield/qa/ShieldTransportClientIT.java | 0 .../qa}/shield-core-rest-tests/build.gradle | 6 +- .../java/org/elasticsearch/shield/RestIT.java | 0 .../qa}/shield-example-realm/build.gradle | 8 +- .../example/ExampleRealmPlugin.java | 0 .../CustomAuthenticationFailureHandler.java | 0 .../example/realm/CustomRealm.java | 0 .../example/realm/CustomRealmFactory.java | 0 .../example/realm/CustomRealmIT.java | 0 .../example/realm/CustomRealmTests.java | 0 .../integration-tests.xml | 0 .../test/tribe_node/10_basic.yaml | 0 .../shield-tribe-node-tests/shield-roles.yml | 0 .../org/elasticsearch/ant/HttpCondition.java | 0 .../java/org/elasticsearch/ant/HttpTask.java | 0 .../java/org/elasticsearch/shield/RestIT.java | 0 .../shield/TribeRestTestCase.java | 0 .../integration-tests.xml | 0 .../smoketest/MarvelClusterInfoIT.java | 0 .../smoketest/WatcherWithShieldIT.java | 0 .../WatcherWithShieldInsufficientRoleIT.java | 0 .../watcher-with-shield-roles.yml | 0 .../qa}/smoke-test-plugins-ssl/build.gradle | 10 +- .../smoketest/SmokeTestPluginsSslIT.java | 0 .../test/smoke_test_plugins_ssl/10_basic.yaml | 0 .../qa}/smoke-test-plugins/build.gradle | 10 +- .../smoketest/SmokeTestPluginsIT.java | 0 .../test/smoke_test_plugins/10_basic.yaml | 0 .../build.gradle | 6 +- .../smoketest/WatcherRestTestCase.java | 0 .../smoketest/WatcherWithGroovyIT.java | 0 .../test/watcher_groovy/10_basic.yaml | 0 .../test/watcher_groovy/20_minimal_body.yaml | 0 .../test/watcher_groovy/30_inline_watch.yaml | 0 .../build.gradle | 12 +- .../smoke-test-watcher-with-shield/roles.yml | 0 .../smoketest/WatcherWithShieldIT.java | 0 .../WatcherWithShieldInsufficientRoleIT.java | 0 {shield => elasticsearch/shield}/LICENSE.txt | 0 {shield => elasticsearch/shield}/NOTICE.txt | 0 .../shield}/README.asciidoc | 0 .../shield}/TESTING.asciidoc | 0 .../shield}/bin/shield/.in.bat | 0 .../shield}/bin/shield/esusers | 0 .../shield}/bin/shield/esusers.bat | 0 .../shield}/bin/shield/syskeygen | 0 .../shield}/bin/shield/syskeygen.bat | 0 {shield => elasticsearch/shield}/build.gradle | 4 +- .../shield}/config/shield/logging.yml | 0 .../shield/config/shield/role_mapping.yml | 0 .../shield}/config/shield/roles.yml | 0 .../shield/config/shield/users | 0 .../shield/config/shield/users_roles | 0 .../esvm/.esvm-shield-config/role_mapping.yml | 0 .../esvm/.esvm-shield-config/roles.yml | 0 .../esvm/.esvm-shield-config/system_key | 0 .../dev-tools/esvm/.esvm-shield-config/users | 0 .../esvm/.esvm-shield-config/users_roles | 0 .../shield}/dev-tools/esvm/.esvmrc | 0 .../shield}/dev-tools/esvm/readme.txt | 0 .../shield}/dev-tools/randomization.yml | 0 .../org/elasticsearch/shield/ShieldBuild.java | 0 .../shield/ShieldDisabledModule.java | 0 .../shield/ShieldLifecycleService.java | 0 .../elasticsearch/shield/ShieldModule.java | 0 .../elasticsearch/shield/ShieldPlugin.java | 0 .../shield/ShieldSettingsFilter.java | 0 .../java/org/elasticsearch/shield/User.java | 0 .../shield/action/ShieldActionFilter.java | 0 .../shield/action/ShieldActionMapper.java | 0 .../shield/action/ShieldActionModule.java | 0 .../authc/cache/ClearRealmCacheAction.java | 0 .../authc/cache/ClearRealmCacheRequest.java | 0 .../cache/ClearRealmCacheRequestBuilder.java | 0 .../authc/cache/ClearRealmCacheResponse.java | 0 .../cache/TransportClearRealmCacheAction.java | 0 .../interceptor/BulkRequestInterceptor.java | 0 .../FieldSecurityRequestInterceptor.java | 0 .../RealtimeRequestInterceptor.java | 0 .../interceptor/RequestInterceptor.java | 0 .../interceptor/SearchRequestInterceptor.java | 0 .../interceptor/UpdateRequestInterceptor.java | 0 .../shield/audit/AuditTrail.java | 0 .../shield/audit/AuditTrailModule.java | 0 .../shield/audit/AuditTrailService.java | 0 .../elasticsearch/shield/audit/AuditUtil.java | 0 .../shield/audit/index/IndexAuditLevel.java | 0 .../shield/audit/index/IndexAuditTrail.java | 0 .../audit/index/IndexAuditUserHolder.java | 0 .../shield/audit/index/IndexNameResolver.java | 0 .../audit/logfile/LoggingAuditTrail.java | 0 .../shield/authc/AnonymousService.java | 0 .../authc/AuthenticationFailureHandler.java | 0 .../shield/authc/AuthenticationModule.java | 0 .../shield/authc/AuthenticationService.java | 0 .../shield/authc/AuthenticationToken.java | 0 .../DefaultAuthenticationFailureHandler.java | 0 .../authc/InternalAuthenticationService.java | 0 .../org/elasticsearch/shield/authc/Realm.java | 0 .../shield/authc/RealmConfig.java | 0 .../elasticsearch/shield/authc/Realms.java | 0 .../ActiveDirectoryGroupsResolver.java | 0 .../activedirectory/ActiveDirectoryRealm.java | 0 .../ActiveDirectorySessionFactory.java | 0 .../shield/authc/esusers/ESUsersRealm.java | 0 .../authc/esusers/FileUserPasswdStore.java | 0 .../authc/esusers/FileUserRolesStore.java | 0 .../authc/esusers/tool/ESUsersTool.java | 0 .../shield/authc/ldap/LdapRealm.java | 0 .../shield/authc/ldap/LdapSessionFactory.java | 0 .../ldap/LdapUserSearchSessionFactory.java | 0 .../authc/ldap/SearchGroupsResolver.java | 0 .../ldap/UserAttributeGroupsResolver.java | 0 .../authc/ldap/support/AbstractLdapRealm.java | 0 .../authc/ldap/support/LdapSearchScope.java | 0 .../authc/ldap/support/LdapSession.java | 0 .../shield/authc/ldap/support/LdapUtils.java | 0 .../authc/ldap/support/SessionFactory.java | 0 .../shield/authc/pki/PkiRealm.java | 0 .../authc/pki/X509AuthenticationToken.java | 0 .../shield/authc/support/BCrypt.java | 0 .../shield/authc/support/CachingRealm.java | 0 .../support/CachingUsernamePasswordRealm.java | 0 .../shield/authc/support/CharArrays.java | 0 .../shield/authc/support/DnRoleMapper.java | 0 .../shield/authc/support/Hasher.java | 0 .../shield/authc/support/RefreshListener.java | 0 .../shield/authc/support/SecuredString.java | 0 .../authc/support/UsernamePasswordRealm.java | 0 .../authc/support/UsernamePasswordToken.java | 0 .../shield/authz/AuthorizationModule.java | 0 .../shield/authz/AuthorizationService.java | 0 .../authz/InternalAuthorizationService.java | 0 .../shield/authz/Permission.java | 0 .../elasticsearch/shield/authz/Privilege.java | 0 .../shield/authz/SystemRole.java | 0 .../accesscontrol/DocumentSubsetReader.java | 0 .../accesscontrol/FieldSubsetReader.java | 0 .../accesscontrol/IndicesAccessControl.java | 0 .../authz/accesscontrol/OptOutQueryCache.java | 0 .../authz/accesscontrol/RequestContext.java | 0 .../ShieldIndexSearcherWrapper.java | 0 .../DefaultIndicesAndAliasesResolver.java | 0 .../IndicesAndAliasesResolver.java | 0 .../shield/authz/store/FileRolesStore.java | 0 .../shield/authz/store/RolesStore.java | 0 .../shield/client/ShieldAuthcClient.java | 0 .../shield/client/ShieldClient.java | 0 .../shield/crypto/CryptoModule.java | 0 .../shield/crypto/CryptoService.java | 0 .../shield/crypto/InternalCryptoService.java | 0 .../shield/crypto/tool/SystemKeyTool.java | 0 .../shield/license/LicenseModule.java | 0 .../shield/license/ShieldLicenseState.java | 0 .../shield/license/ShieldLicensee.java | 0 .../shield/rest/RemoteHostHeader.java | 0 .../shield/rest/ShieldRestFilter.java | 0 .../shield/rest/ShieldRestModule.java | 0 .../rest/action/RestShieldInfoAction.java | 0 .../cache/RestClearRealmCacheAction.java | 0 .../shield/ssl/AbstractSSLService.java | 0 .../shield/ssl/ClientSSLService.java | 0 .../elasticsearch/shield/ssl/SSLModule.java | 0 .../shield/ssl/ServerSSLService.java | 0 .../shield/support/AbstractShieldModule.java | 0 .../shield/support/AutomatonPredicate.java | 0 .../shield/support/Automatons.java | 0 .../shield/support/Exceptions.java | 0 .../shield/support/NoOpLogger.java | 0 .../shield/support/ShieldFiles.java | 0 .../shield/support/Validation.java | 0 .../transport/ClientTransportFilter.java | 0 .../shield/transport/SSLClientAuth.java | 0 .../shield/transport/SSLExceptionHelper.java | 0 .../transport/ServerTransportFilter.java | 0 .../ShieldClientTransportService.java | 0 .../ShieldServerTransportService.java | 0 .../transport/ShieldTransportModule.java | 0 .../shield/transport/filter/IPFilter.java | 0 .../transport/filter/ShieldIpFilterRule.java | 0 .../netty/HandshakeWaitingHandler.java | 0 .../netty/IPFilterNettyUpstreamHandler.java | 0 .../netty/ShieldNettyHttpServerTransport.java | 0 .../transport/netty/ShieldNettyTransport.java | 0 .../plugin-metadata/plugin-security.policy | 0 .../authc/esusers/tool/esusers-list.help | 0 .../authc/esusers/tool/esusers-passwd.help | 0 .../authc/esusers/tool/esusers-roles.help | 0 .../authc/esusers/tool/esusers-useradd.help | 0 .../authc/esusers/tool/esusers-userdel.help | 0 .../shield/authc/esusers/tool/esusers.help | 0 .../shield/crypto/tool/syskey-generate.help | 0 .../main/resources/shield-build.properties | 0 .../src/main/resources/shield_audit_log.json | 0 .../elasticsearch/bench/HasherBenchmark.java | 0 .../http/netty/NettyHttpMockUtil.java | 0 .../AbstractPrivilegeTestCase.java | 0 .../integration/BulkUpdateTests.java | 0 .../integration/ClearRealmsCacheTests.java | 0 .../integration/ClusterPrivilegeTests.java | 0 .../DocumentAndFieldLevelSecurityTests.java | 0 .../DocumentLevelSecurityRandomTests.java | 0 .../DocumentLevelSecurityTests.java | 0 .../FieldLevelSecurityRandomTests.java | 0 .../integration/FieldLevelSecurityTests.java | 0 .../integration/IndexPrivilegeTests.java | 0 ...onsWithAliasesWildcardsAndRegexsTests.java | 0 .../integration/LicensingTests.java | 0 .../MultipleIndicesPermissionsTests.java | 0 .../PermissionPrecedenceTests.java | 0 .../integration/ScrollIdSigningTests.java | 0 .../SearchGetAndSuggestPermissionsTests.java | 0 .../integration/SettingsFilterTests.java | 0 .../ShieldCachePermissionTests.java | 0 .../integration/ShieldClearScrollTests.java | 0 .../ldap/AbstractAdLdapRealmTestCase.java | 0 .../integration/ldap/GroupMappingTests.java | 0 .../ldap/MultiGroupMappingTests.java | 0 .../ShieldPluginEnabledDisabledTests.java | 0 .../shield/ShieldPluginSettingsTests.java | 0 .../shield/ShieldPluginTests.java | 0 .../org/elasticsearch/shield/UserTests.java | 0 .../shield/VersionCompatibilityTests.java | 0 .../action/ShieldActionFilterTests.java | 0 .../action/ShieldActionMapperTests.java | 0 .../shield/audit/AuditTrailModuleTests.java | 0 .../shield/audit/AuditTrailServiceTests.java | 0 .../audit/index/IndexAuditLevelTests.java | 0 .../index/IndexAuditTrailEnabledTests.java | 0 .../audit/index/IndexAuditTrailTests.java | 0 .../IndexAuditTrailUpdateMappingTests.java | 0 .../RemoteIndexAuditTrailStartingTests.java | 0 .../shield/audit/logfile/CapturingLogger.java | 0 .../audit/logfile/LoggingAuditTrailTests.java | 0 .../authc/AnonymousUserHolderTests.java | 0 .../shield/authc/AnonymousUserTests.java | 0 .../authc/AuthenticationModuleTests.java | 0 .../InternalAuthenticationServiceTests.java | 0 .../shield/authc/RealmsTests.java | 0 .../shield/authc/RunAsIntegTests.java | 0 .../ActiveDirectoryGroupsResolverTests.java | 0 .../ActiveDirectoryRealmTests.java | 0 .../ActiveDirectorySessionFactoryTests.java | 0 .../authc/esusers/ESUsersRealmTests.java | 0 .../esusers/FileUserPasswdStoreTests.java | 0 .../esusers/FileUserRolesStoreTests.java | 0 .../authc/esusers/tool/ESUsersToolTests.java | 0 .../shield/authc/ldap/LdapRealmTests.java | 0 .../authc/ldap/LdapSessionFactoryTests.java | 0 .../LdapUserSearchSessionFactoryTests.java | 0 .../shield/authc/ldap/OpenLdapTests.java | 0 .../authc/ldap/SearchGroupsResolverTests.java | 0 .../UserAttributeGroupsResolverTests.java | 0 .../authc/ldap/support/LDAPServersTests.java | 0 .../authc/ldap/support/LdapTestCase.java | 0 .../ldap/support/SessionFactoryTests.java | 0 .../authc/pki/PkiAuthenticationTests.java | 0 .../authc/pki/PkiOptionalClientAuthTests.java | 0 .../shield/authc/pki/PkiRealmTests.java | 0 .../PkiWithoutClientAuthenticationTests.java | 0 .../shield/authc/pki/PkiWithoutSSLTests.java | 0 .../shield/authc/support/BCryptTests.java | 0 .../CachingUsernamePasswordRealmTests.java | 0 .../authc/support/DnRoleMapperTests.java | 0 .../shield/authc/support/HasherTests.java | 0 .../authc/support/SecuredStringTests.java | 0 .../support/UsernamePasswordTokenTests.java | 0 .../shield/authz/AnalyzeTests.java | 0 .../shield/authz/IndexAliasesTests.java | 0 .../InternalAuthorizationServiceTests.java | 0 .../shield/authz/PermissionTests.java | 0 .../shield/authz/PrivilegeTests.java | 0 .../shield/authz/SystemRoleTests.java | 0 .../DocumentSubsetReaderTests.java | 0 ...ldDataCacheWithFieldSubsetReaderTests.java | 0 .../accesscontrol/FieldSubsetReaderTests.java | 0 .../accesscontrol/IndicesPermissionTests.java | 0 ...dIndexSearcherWrapperIntegrationTests.java | 0 .../ShieldIndexSearcherWrapperUnitTests.java | 0 .../DefaultIndicesResolverTests.java | 0 ...cesAndAliasesResolverIntegrationTests.java | 0 .../authz/store/FileRolesStoreTests.java | 0 .../crypto/InternalCryptoServiceTests.java | 0 .../crypto/tool/SystemKeyToolTests.java | 0 .../license/ShieldLicenseStateTests.java | 0 .../shield/rest/ShieldRestFilterTests.java | 0 .../shield/ssl/ClientSSLServiceTests.java | 0 .../shield/ssl/SSLSettingsTests.java | 0 .../shield/ssl/ServerSSLServiceTests.java | 0 .../shield/support/AutomatonsTests.java | 0 .../shield/support/ShieldFilesTests.java | 0 .../shield/support/ValidationTests.java | 0 .../shield/test/ShieldAssertions.java | 0 .../shield/test/ShieldTestUtils.java | 0 .../transport/ClientTransportFilterTests.java | 0 ...ServerTransportFilterIntegrationTests.java | 0 .../transport/ServerTransportFilterTests.java | 0 .../transport/TransportFilterTests.java | 0 .../transport/filter/IPFilterTests.java | 0 .../filter/IpFilteringIntegrationTests.java | 0 .../filter/IpFilteringUpdateTests.java | 0 .../filter/ShieldIpFilterRuleTests.java | 0 .../netty/HandshakeWaitingHandlerTests.java | 0 .../IPFilterNettyUpstreamHandlerTests.java | 0 .../netty/IPHostnameVerificationTests.java | 0 .../ShieldNettyHttpServerTransportTests.java | 0 .../netty/ShieldNettyTransportTests.java | 0 .../netty/SslHostnameVerificationTests.java | 0 .../transport/ssl/SslClientAuthTests.java | 0 .../transport/ssl/SslIntegrationTests.java | 0 .../transport/ssl/SslMultiPortTests.java | 0 .../shield/tribe/TribeShieldLoadedTests.java | 0 .../test/ShieldIntegTestCase.java | 0 .../test/ShieldSettingsSource.java | 0 .../elasticsearch/test/ShieldTestsUtils.java | 0 .../transport/KnownActionsTests.java | 0 .../ShieldServerTransportServiceTests.java | 0 .../transport/netty/NettyMockUtil.java | 0 .../src/test/resources/log4j.properties | 0 .../src/test/resources/logstash-shield.conf | 0 .../authc/activedirectory/ad-schema.ldif | 0 .../shield/authc/activedirectory/ad.ldif | 0 .../authc/activedirectory/role_mapping.yml | 0 .../elasticsearch/shield/authc/esusers/users | 0 .../shield/authc/esusers/users_roles | 0 .../ldap/support/ldapWithGroupSearch.yaml | 0 .../ldap/support/ldapWithRoleMapping.yaml | 0 .../shield/authc/ldap/support/ldaptrust.jks | Bin .../shield/authc/ldap/support/seven-seas.ldif | 0 .../shield/authc/pki/role_mapping.yml | 0 .../shield/authc/support/role_mapping.yml | 0 .../shield/authz/store/default_roles.yml | 0 .../shield/authz/store/invalid_roles.yml | 0 .../shield/authz/store/reserved_roles.yml | 0 .../shield/authz/store/roles.yml | 0 .../org/elasticsearch/shield/plugin/roles.yml | 0 .../org/elasticsearch/shield/plugin/users | 0 .../elasticsearch/shield/plugin/users_roles | 0 .../ssl/certs/simple/README.asciidoc | 0 .../transport/ssl/certs/simple/activedir.crt | 0 .../transport/ssl/certs/simple/openldap.crt | Bin .../ssl/certs/simple/openssl_config.cnf | 0 .../simple/testclient-client-profile.cert | 0 .../simple/testclient-client-profile.jks | Bin .../simple/testclient-client-profile.p12 | Bin .../simple/testclient-client-profile.pem | 0 .../ssl/certs/simple/testclient.cert | 0 .../transport/ssl/certs/simple/testclient.jks | Bin .../transport/ssl/certs/simple/testclient.p12 | Bin .../transport/ssl/certs/simple/testclient.pem | 0 .../certs/simple/testnode-client-profile.cert | 0 .../certs/simple/testnode-client-profile.jks | Bin .../certs/simple/testnode-client-profile.p12 | Bin .../certs/simple/testnode-client-profile.pem | 0 .../simple/testnode-different-passwords.jks | Bin .../ssl/certs/simple/testnode-ip-only.cert | 0 .../ssl/certs/simple/testnode-ip-only.jks | Bin .../certs/simple/testnode-no-subjaltname.cert | 0 .../certs/simple/testnode-no-subjaltname.jks | Bin .../transport/ssl/certs/simple/testnode.cert | 0 .../transport/ssl/certs/simple/testnode.jks | Bin .../transport/ssl/certs/simple/testnode.p12 | Bin .../transport/ssl/certs/simple/testnode.pem | 0 .../certs/simple/truststore-testnode-only.jks | Bin .../org/elasticsearch/transport/actions | 0 .../org/elasticsearch/transport/handlers | 0 .../shield}/test-signatures.txt | 0 .../watcher}/LICENSE.txt | 0 {watcher => elasticsearch/watcher}/NOTICE.txt | 0 .../watcher}/README.asciidoc | 0 .../watcher}/bin/watcher/.in.bat | 0 .../watcher}/bin/watcher/croneval | 0 .../watcher}/bin/watcher/croneval.bat | 0 .../watcher}/build.gradle | 6 +- .../watcher}/dev-tools/randomization.yml | 0 .../elasticsearch/watcher/WatcherBuild.java | 0 .../watcher/WatcherLifeCycleService.java | 0 .../watcher/WatcherMetaData.java | 0 .../elasticsearch/watcher/WatcherModule.java | 0 .../elasticsearch/watcher/WatcherPlugin.java | 0 .../elasticsearch/watcher/WatcherService.java | 0 .../elasticsearch/watcher/WatcherState.java | 0 .../elasticsearch/watcher/actions/Action.java | 0 .../watcher/actions/ActionBuilders.java | 0 .../watcher/actions/ActionFactory.java | 0 .../watcher/actions/ActionRegistry.java | 0 .../watcher/actions/ActionStatus.java | 0 .../watcher/actions/ActionWrapper.java | 0 .../watcher/actions/ExecutableAction.java | 0 .../watcher/actions/ExecutableActions.java | 0 .../watcher/actions/WatcherActionModule.java | 0 .../watcher/actions/email/DataAttachment.java | 0 .../watcher/actions/email/EmailAction.java | 0 .../actions/email/EmailActionFactory.java | 0 .../actions/email/ExecutableEmailAction.java | 0 .../actions/email/service/Account.java | 0 .../actions/email/service/Accounts.java | 0 .../actions/email/service/Attachment.java | 0 .../actions/email/service/Authentication.java | 0 .../watcher/actions/email/service/Email.java | 0 .../actions/email/service/EmailService.java | 0 .../actions/email/service/EmailTemplate.java | 0 .../actions/email/service/HtmlSanitizer.java | 0 .../watcher/actions/email/service/Inline.java | 0 .../email/service/InternalEmailService.java | 0 .../actions/email/service/Profile.java | 0 .../email/service/support/BodyPartSource.java | 0 .../hipchat/ExecutableHipChatAction.java | 0 .../actions/hipchat/HipChatAction.java | 0 .../actions/hipchat/HipChatActionFactory.java | 0 .../hipchat/service/HipChatAccount.java | 0 .../hipchat/service/HipChatAccounts.java | 0 .../hipchat/service/HipChatMessage.java | 0 .../hipchat/service/HipChatServer.java | 0 .../hipchat/service/HipChatService.java | 0 .../hipchat/service/IntegrationAccount.java | 0 .../service/InternalHipChatService.java | 0 .../actions/hipchat/service/SentMessages.java | 0 .../actions/hipchat/service/UserAccount.java | 0 .../actions/hipchat/service/V1Account.java | 0 .../actions/index/ExecutableIndexAction.java | 0 .../watcher/actions/index/IndexAction.java | 0 .../actions/index/IndexActionFactory.java | 0 .../logging/ExecutableLoggingAction.java | 0 .../actions/logging/LoggingAction.java | 0 .../actions/logging/LoggingActionFactory.java | 0 .../watcher/actions/logging/LoggingLevel.java | 0 .../actions/slack/ExecutableSlackAction.java | 0 .../watcher/actions/slack/SlackAction.java | 0 .../actions/slack/SlackActionFactory.java | 0 .../slack/service/InternalSlackService.java | 0 .../actions/slack/service/SentMessages.java | 0 .../actions/slack/service/SlackAccount.java | 0 .../actions/slack/service/SlackAccounts.java | 0 .../actions/slack/service/SlackService.java | 0 .../slack/service/message/Attachment.java | 0 .../service/message/DynamicAttachments.java | 0 .../actions/slack/service/message/Field.java | 0 .../slack/service/message/MessageElement.java | 0 .../slack/service/message/SlackMessage.java | 0 .../service/message/SlackMessageDefaults.java | 0 .../actions/throttler/AckThrottler.java | 0 .../actions/throttler/ActionThrottler.java | 0 .../actions/throttler/PeriodThrottler.java | 0 .../watcher/actions/throttler/Throttler.java | 0 .../webhook/ExecutableWebhookAction.java | 0 .../actions/webhook/WebhookAction.java | 0 .../actions/webhook/WebhookActionFactory.java | 0 .../watcher/client/WatchSourceBuilder.java | 0 .../watcher/client/WatchSourceBuilders.java | 0 .../watcher/client/WatcherClient.java | 0 .../watcher/client/WatcherClientModule.java | 0 .../watcher/condition/Condition.java | 0 .../watcher/condition/ConditionBuilders.java | 0 .../watcher/condition/ConditionFactory.java | 0 .../watcher/condition/ConditionModule.java | 0 .../watcher/condition/ConditionRegistry.java | 0 .../condition/ExecutableCondition.java | 0 .../condition/always/AlwaysCondition.java | 0 .../always/AlwaysConditionFactory.java | 0 .../always/ExecutableAlwaysCondition.java | 0 .../AbstractExecutableCompareCondition.java | 0 .../condition/compare/CompareCondition.java | 0 .../compare/CompareConditionFactory.java | 0 .../compare/ExecutableCompareCondition.java | 0 .../condition/compare/LenientCompare.java | 0 .../compare/array/ArrayCompareCondition.java | 0 .../array/ArrayCompareConditionFactory.java | 0 .../ExecutableArrayCompareCondition.java | 0 .../never/ExecutableNeverCondition.java | 0 .../condition/never/NeverCondition.java | 0 .../never/NeverConditionFactory.java | 0 .../script/ExecutableScriptCondition.java | 0 .../condition/script/ScriptCondition.java | 0 .../script/ScriptConditionFactory.java | 0 .../execution/ActionExecutionMode.java | 0 .../execution/AsyncTriggerListener.java | 0 .../watcher/execution/CurrentExecutions.java | 0 .../watcher/execution/ExecutionModule.java | 0 .../watcher/execution/ExecutionPhase.java | 0 .../watcher/execution/ExecutionService.java | 0 .../watcher/execution/ExecutionState.java | 0 .../execution/InternalWatchExecutor.java | 0 .../execution/ManualExecutionContext.java | 0 .../watcher/execution/QueuedWatch.java | 0 .../execution/SyncTriggerListener.java | 0 .../execution/TriggeredExecutionContext.java | 0 .../watcher/execution/TriggeredWatch.java | 0 .../execution/TriggeredWatchStore.java | 0 .../execution/WatchExecutionContext.java | 0 .../execution/WatchExecutionResult.java | 0 .../execution/WatchExecutionSnapshot.java | 0 .../watcher/execution/WatchExecutor.java | 0 .../elasticsearch/watcher/execution/Wid.java | 0 .../watcher/history/HistoryModule.java | 0 .../watcher/history/HistoryStore.java | 0 .../watcher/history/WatchRecord.java | 0 .../watcher/input/ExecutableInput.java | 0 .../elasticsearch/watcher/input/Input.java | 0 .../watcher/input/InputBuilders.java | 0 .../watcher/input/InputFactory.java | 0 .../watcher/input/InputModule.java | 0 .../watcher/input/InputRegistry.java | 0 .../watcher/input/chain/ChainInput.java | 0 .../input/chain/ChainInputFactory.java | 0 .../input/chain/ExecutableChainInput.java | 0 .../input/http/ExecutableHttpInput.java | 0 .../watcher/input/http/HttpInput.java | 0 .../watcher/input/http/HttpInputFactory.java | 0 .../input/none/ExecutableNoneInput.java | 0 .../watcher/input/none/NoneInput.java | 0 .../watcher/input/none/NoneInputFactory.java | 0 .../input/search/ExecutableSearchInput.java | 0 .../watcher/input/search/SearchInput.java | 0 .../input/search/SearchInputFactory.java | 0 .../input/simple/ExecutableSimpleInput.java | 0 .../watcher/input/simple/SimpleInput.java | 0 .../input/simple/SimpleInputFactory.java | 0 .../watcher/license/LicenseModule.java | 0 .../watcher/license/WatcherLicensee.java | 0 .../watcher/rest/WatcherRestHandler.java | 0 .../rest/action/RestAckWatchAction.java | 0 .../rest/action/RestActivateWatchAction.java | 0 .../rest/action/RestDeleteWatchAction.java | 0 .../rest/action/RestExecuteWatchAction.java | 0 .../rest/action/RestGetWatchAction.java | 0 .../action/RestHijackOperationAction.java | 0 .../rest/action/RestPutWatchAction.java | 0 .../rest/action/RestWatchServiceAction.java | 0 .../rest/action/RestWatcherInfoAction.java | 0 .../rest/action/RestWatcherStatsAction.java | 0 .../watcher/shield/ShieldIntegration.java | 0 .../watcher/shield/ShieldSecretService.java | 0 .../watcher/shield/WatcherSettingsFilter.java | 0 .../watcher/shield/WatcherShieldModule.java | 0 .../watcher/shield/WatcherUserHolder.java | 0 .../watcher/support/ArrayObjectIterator.java | 0 .../watcher/support/Exceptions.java | 0 .../watcher/support/Integers.java | 0 .../elasticsearch/watcher/support/Script.java | 0 .../support/SearchRequestEquivalence.java | 0 .../watcher/support/Strings.java | 0 .../support/ThreadPoolSettingsBuilder.java | 0 .../watcher/support/Variables.java | 0 .../watcher/support/WatcherDateTimeUtils.java | 0 .../support/WatcherIndexTemplateRegistry.java | 0 .../watcher/support/WatcherUtils.java | 0 .../support/XContentFilterKeysUtils.java | 0 .../watcher/support/clock/Clock.java | 0 .../watcher/support/clock/ClockModule.java | 0 .../watcher/support/clock/HaltedClock.java | 0 .../watcher/support/clock/SystemClock.java | 0 .../support/concurrent/FairKeyedLock.java | 0 .../watcher/support/http/HttpClient.java | 0 .../support/http/HttpClientModule.java | 0 .../watcher/support/http/HttpContentType.java | 0 .../watcher/support/http/HttpMethod.java | 0 .../watcher/support/http/HttpProxy.java | 0 .../watcher/support/http/HttpRequest.java | 0 .../support/http/HttpRequestTemplate.java | 0 .../watcher/support/http/HttpResponse.java | 0 .../watcher/support/http/Scheme.java | 0 .../support/http/auth/ApplicableHttpAuth.java | 0 .../watcher/support/http/auth/HttpAuth.java | 0 .../support/http/auth/HttpAuthFactory.java | 0 .../support/http/auth/HttpAuthRegistry.java | 0 .../http/auth/basic/ApplicableBasicAuth.java | 0 .../support/http/auth/basic/BasicAuth.java | 0 .../http/auth/basic/BasicAuthFactory.java | 0 .../support/init/InitializingModule.java | 0 .../support/init/InitializingService.java | 0 .../support/init/proxy/ClientProxy.java | 0 .../init/proxy/ScriptServiceProxy.java | 0 .../watcher/support/secret/Secret.java | 0 .../watcher/support/secret/SecretModule.java | 0 .../watcher/support/secret/SecretService.java | 0 .../watcher/support/text/TextTemplate.java | 0 .../support/text/TextTemplateEngine.java | 0 .../support/text/TextTemplateModule.java | 0 .../text/xmustache/XMustacheFactory.java | 0 .../XMustacheScriptEngineService.java | 0 .../XMustacheTextTemplateEngine.java | 0 .../support/validation/Validation.java | 0 .../validation/WatcherSettingsValidation.java | 0 .../watcher/support/xcontent/ObjectPath.java | 0 .../support/xcontent/WatcherParams.java | 0 .../xcontent/WatcherXContentParser.java | 0 .../xcontent/WatcherXContentUtils.java | 0 .../support/xcontent/XContentSource.java | 0 .../transform/ExecutableTransform.java | 0 .../watcher/transform/Transform.java | 0 .../watcher/transform/TransformBuilders.java | 0 .../watcher/transform/TransformFactory.java | 0 .../watcher/transform/TransformModule.java | 0 .../watcher/transform/TransformRegistry.java | 0 .../transform/chain/ChainTransform.java | 0 .../chain/ChainTransformFactory.java | 0 .../chain/ExecutableChainTransform.java | 0 .../script/ExecutableScriptTransform.java | 0 .../transform/script/ScriptTransform.java | 0 .../script/ScriptTransformFactory.java | 0 .../search/ExecutableSearchTransform.java | 0 .../transform/search/SearchTransform.java | 0 .../search/SearchTransformFactory.java | 0 .../actions/WatcherTransportAction.java | 0 .../transport/actions/ack/AckWatchAction.java | 0 .../actions/ack/AckWatchRequest.java | 0 .../actions/ack/AckWatchRequestBuilder.java | 0 .../actions/ack/AckWatchResponse.java | 0 .../actions/ack/TransportAckWatchAction.java | 0 .../actions/activate/ActivateWatchAction.java | 0 .../activate/ActivateWatchRequest.java | 0 .../activate/ActivateWatchRequestBuilder.java | 0 .../activate/ActivateWatchResponse.java | 0 .../TransportActivateWatchAction.java | 0 .../actions/delete/DeleteWatchAction.java | 0 .../actions/delete/DeleteWatchRequest.java | 0 .../delete/DeleteWatchRequestBuilder.java | 0 .../actions/delete/DeleteWatchResponse.java | 0 .../delete/TransportDeleteWatchAction.java | 0 .../actions/execute/ExecuteWatchAction.java | 0 .../actions/execute/ExecuteWatchRequest.java | 0 .../execute/ExecuteWatchRequestBuilder.java | 0 .../actions/execute/ExecuteWatchResponse.java | 0 .../execute/TransportExecuteWatchAction.java | 0 .../transport/actions/get/GetWatchAction.java | 0 .../actions/get/GetWatchRequest.java | 0 .../actions/get/GetWatchRequestBuilder.java | 0 .../actions/get/GetWatchResponse.java | 0 .../actions/get/TransportGetWatchAction.java | 0 .../transport/actions/put/PutWatchAction.java | 0 .../actions/put/PutWatchRequest.java | 0 .../actions/put/PutWatchRequestBuilder.java | 0 .../actions/put/PutWatchResponse.java | 0 .../actions/put/TransportPutWatchAction.java | 0 .../TransportWatcherServiceAction.java | 0 .../actions/service/WatcherServiceAction.java | 0 .../service/WatcherServiceRequest.java | 0 .../service/WatcherServiceRequestBuilder.java | 0 .../service/WatcherServiceResponse.java | 0 .../stats/TransportWatcherStatsAction.java | 0 .../actions/stats/WatcherStatsAction.java | 0 .../actions/stats/WatcherStatsRequest.java | 0 .../stats/WatcherStatsRequestBuilder.java | 0 .../actions/stats/WatcherStatsResponse.java | 0 .../trigger/AbstractTriggerEngine.java | 0 .../watcher/trigger/Trigger.java | 0 .../watcher/trigger/TriggerBuilders.java | 0 .../watcher/trigger/TriggerEngine.java | 0 .../watcher/trigger/TriggerEvent.java | 0 .../watcher/trigger/TriggerModule.java | 0 .../watcher/trigger/TriggerService.java | 0 .../watcher/trigger/manual/ManualTrigger.java | 0 .../trigger/manual/ManualTriggerEngine.java | 0 .../trigger/manual/ManualTriggerEvent.java | 0 .../watcher/trigger/schedule/Cron.java | 0 .../trigger/schedule/CronSchedule.java | 0 .../trigger/schedule/CronnableSchedule.java | 0 .../trigger/schedule/DailySchedule.java | 0 .../trigger/schedule/HourlySchedule.java | 0 .../trigger/schedule/IntervalSchedule.java | 0 .../trigger/schedule/MonthlySchedule.java | 0 .../watcher/trigger/schedule/Schedule.java | 0 .../trigger/schedule/ScheduleModule.java | 0 .../trigger/schedule/ScheduleRegistry.java | 0 .../trigger/schedule/ScheduleTrigger.java | 0 .../schedule/ScheduleTriggerEngine.java | 0 .../schedule/ScheduleTriggerEvent.java | 0 .../watcher/trigger/schedule/Schedules.java | 0 .../trigger/schedule/WeeklySchedule.java | 0 .../trigger/schedule/YearlySchedule.java | 0 .../SchedulerScheduleTriggerEngine.java | 0 .../engine/TickerScheduleTriggerEngine.java | 0 .../trigger/schedule/support/DayOfWeek.java | 0 .../trigger/schedule/support/DayTimes.java | 0 .../trigger/schedule/support/Month.java | 0 .../trigger/schedule/support/MonthTimes.java | 0 .../trigger/schedule/support/Times.java | 0 .../trigger/schedule/support/WeekTimes.java | 0 .../trigger/schedule/support/YearTimes.java | 0 .../trigger/schedule/tool/CronEvalTool.java | 0 .../elasticsearch/watcher/watch/Payload.java | 0 .../elasticsearch/watcher/watch/Watch.java | 0 .../watcher/watch/WatchLockService.java | 0 .../watcher/watch/WatchModule.java | 0 .../watcher/watch/WatchStatus.java | 0 .../watcher/watch/WatchStore.java | 0 .../plugin-metadata/plugin-security.policy | 0 .../trigger/schedule/tool/croneval-eval.help | 0 .../src/main/resources/triggered_watches.json | 0 .../src/main/resources/watch_history.json | 0 .../main/resources/watcher-build.properties | 0 .../watcher}/src/main/resources/watches.json | 0 .../script/SleepScriptEngine.java | 0 .../org/elasticsearch/watcher/WatcherF.java | 0 .../watcher/WatcherLifeCycleServiceTests.java | 0 .../watcher/WatcherPluginDisableTests.java | 0 .../watcher/WatcherPluginTests.java | 0 .../watcher/WatcherServiceTests.java | 0 .../actions/ActionErrorIntegrationTests.java | 0 .../actions/TimeThrottleIntegrationTests.java | 0 .../actions/email/DataAttachmentTests.java | 0 .../email/EmailActionIntegrationTests.java | 0 .../actions/email/EmailActionTests.java | 0 .../email/EmailSecretsIntegrationTests.java | 0 .../actions/email/service/AccountTests.java | 0 .../actions/email/service/AccountsTests.java | 0 .../email/service/EmailTemplateTests.java | 0 .../actions/email/service/EmailTests.java | 0 .../email/service/HtmlSanitizerTests.java | 0 .../service/InternalEmailServiceTests.java | 0 .../ManualPublicSmtpServersTester.java | 0 .../email/service/support/EmailServer.java | 0 .../hipchat/HipChatActionFactoryTests.java | 0 .../actions/hipchat/HipChatActionTests.java | 0 .../hipchat/service/HipChatAccountsTests.java | 0 .../hipchat/service/HipChatMessageTests.java | 0 .../hipchat/service/HipChatServiceTests.java | 0 .../service/IntegrationAccountTests.java | 0 .../service/InternalHipChatServiceTests.java | 0 .../hipchat/service/UserAccountTests.java | 0 .../hipchat/service/V1AccountTests.java | 0 .../index/IndexActionIntegrationTests.java | 205 +++++ .../actions/index/IndexActionTests.java | 0 .../actions/logging/LoggingActionTests.java | 0 .../slack/SlackActionFactoryTests.java | 0 .../actions/slack/SlackActionTests.java | 0 .../slack/service/SlackAccountsTests.java | 0 .../slack/service/SlackServiceTests.java | 0 .../message/SlackMessageDefaultsTests.java | 0 .../service/message/SlackMessageTests.java | 0 .../actions/throttler/AckThrottlerTests.java | 0 .../throttler/ActionThrottleTests.java | 0 .../throttler/PeriodThrottlerTests.java | 0 .../throttler/WatchThrottlerTests.java | 0 .../actions/webhook/WebhookActionTests.java | 0 .../webhook/WebhookHttpsIntegrationTests.java | 0 .../webhook/WebhookIntegrationTests.java | 0 .../always/AlwaysConditionTests.java | 0 .../compare/CompareConditionSearchTests.java | 0 .../compare/CompareConditionTests.java | 0 .../ArrayCompareConditionSearchTests.java | 0 .../array/ArrayCompareConditionTests.java | 0 .../condition/never/NeverConditionTests.java | 0 .../script/ScriptConditionSearchTests.java | 97 ++ .../script/ScriptConditionTests.java | 219 +++++ .../execution/ExecutionServiceTests.java | 0 .../execution/ManualExecutionTests.java | 0 .../TriggeredWatchStoreLifeCycleTests.java | 0 .../execution/TriggeredWatchStoreTests.java | 0 .../execution/TriggeredWatchTests.java | 0 .../history/HistoryStoreSettingsTests.java | 0 .../watcher/history/HistoryStoreTests.java | 0 .../HistoryTemplateEmailMappingsTests.java | 0 .../HistoryTemplateHttpMappingsTests.java | 0 ...storyTemplateIndexActionMappingsTests.java | 0 ...storyTemplateSearchInputMappingsTests.java | 0 .../HistoryTemplateTimeMappingsTests.java | 0 ...HistoryTemplateTransformMappingsTests.java | 110 +++ .../watcher/input/InputRegistryTests.java | 0 .../watcher/input/chain/ChainInputTests.java | 0 .../input/chain/ChainIntegrationTests.java | 0 .../input/http/HttpInputIntegrationTests.java | 0 .../watcher/input/http/HttpInputTests.java | 0 .../input/search/SearchInputTests.java | 0 .../input/simple/SimpleInputTests.java | 0 .../watcher/license/LicenseTests.java | 0 .../watcher/shield/BasicShieldTests.java | 0 .../DynamicIndexNameIntegrationTests.java | 0 .../watcher/support/FilterXContentTests.java | 0 .../watcher/support/VariablesTests.java | 0 .../support/WatcherDateTimeUtilsTests.java | 0 .../WatcherIndexTemplateRegistryTests.java | 0 .../watcher/support/WatcherUtilsTests.java | 0 .../watcher/support/clock/ClockMock.java | 0 .../watcher/support/clock/ClockTests.java | 0 .../concurrent/FairKeyedLockTests.java | 0 .../watcher/support/http/HttpClientTests.java | 0 .../http/HttpConnectionTimeoutTests.java | 0 .../support/http/HttpReadTimeoutTests.java | 0 .../http/HttpRequestTemplateTests.java | 0 .../support/http/HttpResponseTests.java | 0 .../support/text/TextTemplateTests.java | 0 .../xmustache/XMustacheScriptEngineTests.java | 0 .../text/xmustache/XMustacheTests.java | 0 .../support/xcontent/MapPathTests.java | 0 .../support/xcontent/XContentSourceTests.java | 0 .../AbstractWatcherIntegrationTestCase.java | 0 .../watcher/test/TimeWarpedWatcherPlugin.java | 0 .../WatchExecutionContextMockBuilder.java | 0 .../watcher/test/WatcherMatchers.java | 0 .../watcher/test/WatcherTestUtils.java | 0 .../bench/ScheduleEngineTriggerBenchmark.java | 0 .../WatcherExecutorServiceBenchmark.java | 0 .../bench/WatcherScheduleEngineBenchmark.java | 0 .../test/integration/BasicWatcherTests.java | 0 .../test/integration/BootStrapTests.java | 0 .../ExecutionVarsIntegrationTests.java | 168 ++++ .../HttpSecretsIntegrationTests.java | 0 .../test/integration/NoMasterNodeTests.java | 0 .../test/integration/WatchMetadataTests.java | 0 .../WatcherSettingsFilterTests.java | 0 .../watcher/test/rest/WatcherRestIT.java | 0 .../test/rest/WatcherRestTestCase.java | 0 .../transform/TransformIntegrationTests.java | 215 +++++ .../transform/chain/ChainTransformTests.java | 0 .../script/ScriptTransformTests.java | 0 .../search/SearchTransformTests.java | 0 .../transport/action/ack/WatchAckTests.java | 0 .../action/activate/ActivateWatchTests.java | 0 .../action/delete/DeleteWatchTests.java | 0 .../action/delete/ForceDeleteWatchTests.java | 0 .../action/execute/ExecuteWatchTests.java | 0 .../ExecuteWatchWithDateMathTests.java | 0 .../transport/action/get/GetWatchTests.java | 0 .../transport/action/put/PutWatchTests.java | 0 .../action/stats/SlowWatchStatsTests.java | 0 .../action/stats/WatcherStatsTests.java | 0 .../trigger/ScheduleTriggerEngineMock.java | 0 .../trigger/schedule/CronScheduleTests.java | 0 .../trigger/schedule/DailyScheduleTests.java | 0 .../trigger/schedule/HourlyScheduleTests.java | 0 .../schedule/IntervalScheduleTests.java | 0 .../schedule/MonthlyScheduleTests.java | 0 .../schedule/ScheduleRegistryTests.java | 0 .../trigger/schedule/ScheduleTestCase.java | 0 .../schedule/ScheduleTriggerEventTests.java | 0 .../trigger/schedule/WeeklyScheduleTests.java | 0 .../trigger/schedule/YearlyScheduleTests.java | 0 .../engine/BaseTriggerEngineTestCase.java | 0 .../engine/SchedulerScheduleEngineTests.java | 0 .../engine/TickerScheduleEngineTests.java | 0 .../schedule/tool/CronEvalToolTests.java | 0 .../trigger/schedule/tool/EvalCron.java | 0 .../watcher/watch/WatchLockServiceTests.java | 0 .../watcher/watch/WatchStoreTests.java | 0 .../watcher/watch/WatchTests.java | 0 .../resources/config/scripts/my-script.groovy | 0 .../keystore/testnode-different-passwords.jks | Bin .../shield/keystore/testnode.cert | 0 .../shield/keystore/testnode.jks | Bin .../keystore/truststore-testnode-only.jks | Bin .../watcher/actions/email/service/logo.png | Bin .../scripts/test_disk_template.mustache | 0 .../scripts/test_disk_template.mustache | 0 .../rest-api-spec/api/watcher.ack_watch.json | 0 .../api/watcher.activate_watch.json | 0 .../api/watcher.deactivate_watch.json | 0 .../api/watcher.delete_watch.json | 0 .../api/watcher.execute_watch.json | 0 .../rest-api-spec/api/watcher.get_watch.json | 0 .../rest-api-spec/api/watcher.info.json | 0 .../rest-api-spec/api/watcher.put_watch.json | 0 .../rest-api-spec/api/watcher.restart.json | 0 .../rest-api-spec/api/watcher.start.json | 0 .../rest-api-spec/api/watcher.stats.json | 0 .../rest-api-spec/api/watcher.stop.json | 0 .../test/ack_watch/10_basic.yaml | 0 .../ack_watch/20_ack_individual_action.yaml | 0 .../test/activate_watch/10_basic.yaml | 0 .../test/array_compare_watch/10_basic.yaml | 0 .../test/delete_watch/10_basic.yaml | 0 .../test/get_watch/10_basic.yaml | 0 .../test/get_watch/20_missing.yaml | 0 .../10_monitor_cluster_health.yaml | 0 .../rest-api-spec/test/hijack/10_basic.yaml | 0 .../test/put_watch/10_basic.yaml | 0 .../20_put_watch_with_throttle_period.yaml | 0 ...put_watch_with_action_throttle_period.yaml | 0 .../put_watch/40_put_watch_as_inactive.yaml | 0 .../test/restart_watcher/10_basic.yaml | 0 .../test/start_watcher/10_basic.yaml | 0 .../rest-api-spec/test/stats/10_basic.yaml | 0 .../test/stop_watcher/10_basic.yaml | 0 .../test/watch_info/10_basic.yaml | 0 .../x-dev-tools}/RELEASE.md | 0 .../main/resources/ant/shield-overrides.xml | 0 .../elasticsearch_license_header.txt | 0 .../license_header_definition.xml | 0 shield/docs/private/indices_replace.asciidoc | 19 - shield/docs/private/ldap-testing.asciidoc | 93 -- .../docs/public/configuring-auditing.asciidoc | 290 ------ .../configuring-clients-integrations.asciidoc | 17 - .../hadoop.asciidoc | 8 - .../http.asciidoc | 58 -- .../java.asciidoc | 239 ----- .../kibana.asciidoc | 316 ------- .../logstash.asciidoc | 200 ----- .../marvel.asciidoc | 115 --- shield/docs/public/configuring-rbac.asciidoc | 155 ---- .../docs/public/example-deployments.asciidoc | 6 - .../example-deployments/e-commerce.asciidoc | 92 -- shield/docs/public/getting-started.asciidoc | 56 -- .../getting-started/enable-auditing.asciidoc | 17 - .../enable-basic-auth.asciidoc | 53 -- .../enable-message-authentication.asciidoc | 21 - .../public/getting-started/moving-on.asciidoc | 9 - .../public/granting-alias-privileges.asciidoc | 101 --- shield/docs/public/how-shield-works.asciidoc | 84 -- shield/docs/public/index.asciidoc | 45 - shield/docs/public/installing-shield.asciidoc | 177 ---- shield/docs/public/introduction.asciidoc | 53 -- shield/docs/public/limitations.asciidoc | 57 -- .../public/managing-shield-licenses.asciidoc | 114 --- shield/docs/public/managing-users.asciidoc | 257 ------ shield/docs/public/mapping-roles.asciidoc | 61 -- shield/docs/public/reference.asciidoc | 430 --------- shield/docs/public/release-notes.asciidoc | 230 ----- .../public/securing-communications.asciidoc | 19 - .../enabling-cipher-suites.asciidoc | 19 - .../separating-node-client-traffic.asciidoc | 65 -- .../setting-up-ssl.asciidoc | 267 ------ .../using-ip-filtering.asciidoc | 131 --- .../public/setting-up-authentication.asciidoc | 117 --- ...onfiguring-active-directory-realm.asciidoc | 244 ----- .../configuring-esusers-realm.asciidoc | 74 -- .../configuring-ldap-realm.asciidoc | 341 ------- .../configuring-pki-realm.asciidoc | 123 --- .../controlling-user-cache.asciidoc | 61 -- .../enabling-anonymous-access.asciidoc | 34 - .../integrating-other-auth-systems.asciidoc | 56 -- .../setting-up-certificate-authority.asciidoc | 190 ---- ...field-and-document-level-security.asciidoc | 162 ---- ...bmitting-requests-for-other-users.asciidoc | 31 - shield/docs/public/troubleshooting.asciidoc | 218 ----- watcher/docs/administering-watcher.asciidoc | 25 - ...configuring-default-http-timeouts.asciidoc | 21 - ...ing-default-internal-ops-timeouts.asciidoc | 22 - ...nfiguring-default-throttle-period.asciidoc | 19 - .../configuring-email.asciidoc | 270 ------ .../configuring-hipchat.asciidoc | 232 ----- .../configuring-slack.asciidoc | 87 -- .../getting-watcher-statistics.asciidoc | 46 - .../integrating-with-logstash.asciidoc | 61 -- .../integrating-with-shield.asciidoc | 103 --- .../monitoring-watch-execution.asciidoc | 177 ---- watcher/docs/customizing-watches.asciidoc | 303 ------- watcher/docs/example-watches.asciidoc | 9 - .../watching-marvel-data.asciidoc | 664 -------------- .../watching-time-series-data.asciidoc | 206 ----- watcher/docs/getting-started.asciidoc | 488 ---------- watcher/docs/how-watcher-works.asciidoc | 425 --------- .../docs/how-watcher-works/templates.asciidoc | 196 ---- watcher/docs/images/action-throttling.jpg | Bin 18426 -> 0 bytes .../docs/images/hipchat-copy-room-token.jpg | Bin 16906 -> 0 bytes .../docs/images/hipchat-copy-user-token.jpg | Bin 17177 -> 0 bytes watcher/docs/images/hipchat-copy-v1-token.jpg | Bin 15551 -> 0 bytes .../images/hipchat-generate-room-token.jpg | Bin 114702 -> 0 bytes .../images/hipchat-generate-user-token.jpg | Bin 21372 -> 0 bytes .../docs/images/hipchat-generate-v1-token.jpg | Bin 14500 -> 0 bytes .../images/hipchat-integration-example.png | Bin 150246 -> 0 bytes .../images/slack-add-webhook-integration.jpg | Bin 33234 -> 0 bytes .../docs/images/slack-copy-webhook-url.jpg | Bin 13063 -> 0 bytes watcher/docs/images/watch-execution.jpg | Bin 44137 -> 0 bytes .../docs/images/watcher-kibana-dashboard.png | Bin 172622 -> 0 bytes watcher/docs/images/watcher.graffle | Bin 6521 -> 0 bytes watcher/docs/index.asciidoc | 36 - watcher/docs/installing-watcher.asciidoc | 121 --- watcher/docs/introduction.asciidoc | 69 -- .../docs/managing-watcher-licenses.asciidoc | 96 -- watcher/docs/managing-watches.asciidoc | 15 - .../deleting-watches.asciidoc | 23 - .../managing-watches/listing-watches.asciidoc | 24 - watcher/docs/reference.asciidoc | 16 - watcher/docs/reference/actions.asciidoc | 175 ---- watcher/docs/reference/actions/email.asciidoc | 70 -- .../docs/reference/actions/hipchat.asciidoc | 127 --- watcher/docs/reference/actions/index.asciidoc | 74 -- .../docs/reference/actions/logging.asciidoc | 43 - watcher/docs/reference/actions/slack.asciidoc | 161 ---- .../docs/reference/actions/webhook.asciidoc | 145 --- watcher/docs/reference/condition.asciidoc | 22 - .../docs/reference/condition/always.asciidoc | 32 - .../condition/array-compare.asciidoc | 69 -- .../docs/reference/condition/compare.asciidoc | 98 -- .../docs/reference/condition/never.asciidoc | 24 - .../docs/reference/condition/script.asciidoc | 177 ---- watcher/docs/reference/input.asciidoc | 18 - watcher/docs/reference/input/chain.asciidoc | 43 - watcher/docs/reference/input/http.asciidoc | 166 ---- watcher/docs/reference/input/search.asciidoc | 138 --- watcher/docs/reference/input/simple.asciidoc | 52 -- watcher/docs/reference/java.asciidoc | 110 --- .../docs/reference/java/ack-watch.asciidoc | 65 -- .../reference/java/activate-watch.asciidoc | 26 - .../reference/java/deactivate-watch.asciidoc | 26 - .../docs/reference/java/delete-watch.asciidoc | 19 - .../reference/java/execute-watch.asciidoc | 54 -- .../docs/reference/java/get-watch.asciidoc | 32 - .../docs/reference/java/put-watch.asciidoc | 76 -- watcher/docs/reference/java/service.asciidoc | 24 - watcher/docs/reference/java/stats.asciidoc | 33 - watcher/docs/reference/rest.asciidoc | 32 - .../docs/reference/rest/ack-watch.asciidoc | 148 --- .../reference/rest/activate-watch.asciidoc | 52 -- .../reference/rest/deactivate-watch.asciidoc | 52 -- .../docs/reference/rest/delete-watch.asciidoc | 46 - .../reference/rest/execute-watch.asciidoc | 316 ------- .../docs/reference/rest/get-watch.asciidoc | 109 --- watcher/docs/reference/rest/info.asciidoc | 33 - .../docs/reference/rest/put-watch.asciidoc | 107 --- watcher/docs/reference/rest/restart.asciidoc | 19 - watcher/docs/reference/rest/start.asciidoc | 19 - watcher/docs/reference/rest/stats.asciidoc | 154 ---- watcher/docs/reference/rest/stop.asciidoc | 19 - watcher/docs/reference/transform.asciidoc | 60 -- .../docs/reference/transform/chain.asciidoc | 43 - .../docs/reference/transform/script.asciidoc | 62 -- .../docs/reference/transform/search.asciidoc | 164 ---- watcher/docs/reference/trigger.asciidoc | 12 - .../docs/reference/trigger/schedule.asciidoc | 52 -- .../reference/trigger/schedule/cron.asciidoc | 142 --- .../reference/trigger/schedule/daily.asciidoc | 98 -- .../trigger/schedule/hourly.asciidoc | 49 - .../trigger/schedule/interval.asciidoc | 35 - .../trigger/schedule/monthly.asciidoc | 72 -- .../trigger/schedule/weekly.asciidoc | 77 -- .../trigger/schedule/yearly.asciidoc | 80 -- watcher/docs/release-notes.asciidoc | 169 ---- watcher/docs/troubleshooting.asciidoc | 79 -- 1252 files changed, 9858 insertions(+), 13718 deletions(-) rename {qa => elasticsearch}/build.gradle (100%) create mode 100644 elasticsearch/license/README.md create mode 100644 elasticsearch/license/base/build.gradle create mode 100644 elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/CryptUtils.java create mode 100644 elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/DateUtils.java create mode 100644 elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/License.java create mode 100644 elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/LicenseVerifier.java create mode 100644 elasticsearch/license/base/src/test/java/org/elasticsearch/license/core/LicenseSerializationTests.java create mode 100644 elasticsearch/license/base/src/test/java/org/elasticsearch/license/core/TestUtils.java create mode 100644 elasticsearch/license/base/src/test/resources/log4j.properties create mode 100644 elasticsearch/license/base/src/test/resources/private.key create mode 100644 elasticsearch/license/base/src/test/resources/public.key create mode 100644 elasticsearch/license/build.gradle create mode 100644 elasticsearch/license/found-plugin/.gitignore create mode 100644 elasticsearch/license/found-plugin/build.gradle create mode 100644 elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/LicenseModule.java create mode 100644 elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/LicensePlugin.java create mode 100644 elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/core/FoundLicensesService.java create mode 100755 elasticsearch/license/licensor/bin/key-pair-generator create mode 100755 elasticsearch/license/licensor/bin/license-generator create mode 100755 elasticsearch/license/licensor/bin/verify-license create mode 100644 elasticsearch/license/licensor/build.gradle create mode 100644 elasticsearch/license/licensor/dev-tools/integration-tests.xml create mode 100644 elasticsearch/license/licensor/sample/license_spec.json create mode 100644 elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/LicenseSigner.java create mode 100644 elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/KeyPairGeneratorTool.java create mode 100644 elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/LicenseGeneratorTool.java create mode 100644 elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/LicenseVerificationTool.java create mode 100644 elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-key-pair-generator.help create mode 100644 elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-license-generator.help create mode 100644 elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-verify-license.help create mode 100644 elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/LicenseVerificationTests.java create mode 100644 elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/TestUtils.java create mode 100644 elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/KeyPairGenerationToolTests.java create mode 100644 elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/LicenseGenerationToolTests.java create mode 100644 elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/LicenseVerificationToolTests.java create mode 100644 elasticsearch/license/licensor/src/test/resources/log4j.properties create mode 100644 elasticsearch/license/licensor/src/test/resources/private.key create mode 100644 elasticsearch/license/licensor/src/test/resources/public.key create mode 100644 elasticsearch/license/plugin-api/.gitignore create mode 100644 elasticsearch/license/plugin-api/build.gradle create mode 100644 elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/AbstractLicenseeComponent.java create mode 100644 elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseState.java create mode 100644 elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseUtils.java create mode 100644 elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/Licensee.java create mode 100644 elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseeRegistry.java create mode 100644 elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicensesManagerService.java create mode 100644 elasticsearch/license/plugin-api/src/test/java/org/elasticsearch/license/plugin/core/LicenseUtilsTests.java create mode 100644 elasticsearch/license/plugin/.gitignore rename {marvel => elasticsearch/license/plugin}/LICENSE.txt (100%) rename {marvel => elasticsearch/license/plugin}/NOTICE.txt (100%) create mode 100644 elasticsearch/license/plugin/build.gradle create mode 100644 elasticsearch/license/plugin/keys/dev/public.key create mode 100644 elasticsearch/license/plugin/keys/prod/public.key create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/LicenseModule.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/LicensePlugin.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseAction.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseRequest.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseRequestBuilder.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseResponse.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/TransportDeleteLicenseAction.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseAction.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseRequest.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseRequestBuilder.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseResponse.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/TransportGetLicenseAction.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseAction.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseRequest.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseRequestBuilder.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseResponse.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/TransportPutLicenseAction.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesMetaData.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesService.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesStatus.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/TrialLicense.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestDeleteLicenseAction.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestGetLicenseAction.java create mode 100644 elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestPutLicenseAction.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesConsumerPluginIntegrationTestCase.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesIntegrationTestCase.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesEagerConsumerPluginIntegrationTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesLazyConsumerPluginIntegrationTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesPluginIntegrationTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesPluginsIntegrationTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesServiceClusterTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesServiceNodeTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesTransportTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/PutLicenseResponseTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/TestUtils.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/TrialLicenseTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationConsumerPlugin.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationPluginService.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationConsumerPlugin.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationPluginService.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/TestConsumerPluginBase.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/TestPluginServiceBase.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesAcknowledgementTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesExpirationCallbackTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesExpiryNotificationTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesManagerServiceTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesMetaDataSerializationTests.java create mode 100644 elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/rest/LicensesRestIT.java create mode 100644 elasticsearch/license/plugin/src/test/resources/log4j.properties create mode 100644 elasticsearch/license/plugin/src/test/resources/private.key create mode 100644 elasticsearch/license/plugin/src/test/resources/rest-api-spec/api/license.get.json create mode 100644 elasticsearch/license/plugin/src/test/resources/rest-api-spec/api/license.post.json create mode 100644 elasticsearch/license/plugin/src/test/resources/rest-api-spec/test/license/10_basic.yaml create mode 100644 elasticsearch/license/plugin/src/test/resources/rest-api-spec/test/license/20_put_license.yaml create mode 100644 elasticsearch/marvel/LICENSE.txt create mode 100644 elasticsearch/marvel/NOTICE.txt rename {marvel => elasticsearch/marvel}/build.gradle (83%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/MarvelPlugin.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/AgentService.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/Collector.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/CollectorModule.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoMarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateMarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateNodeMarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollector.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsMarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/DiscoveryNodeMarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollector.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryMarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsMarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsMarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsCollector.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsMarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/shards/ShardMarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/collector/shards/ShardsCollector.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportBulk.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/Exporter.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/ExporterModule.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/Exporters.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/IndexNameResolver.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelDoc.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtils.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterUtils.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/AbstractRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/Renderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererRegistry.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateNodeRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/DiscoveryNodeRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndexRecoveryRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsRenderer.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelModule.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSetting.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/license/LicenseModule.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/license/MarvelLicensee.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/shield/MarvelInternalUserHolder.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/shield/MarvelSettingsFilter.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/shield/MarvelShieldIntegration.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/shield/MarvelShieldModule.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/shield/SecuredClient.java (100%) rename {marvel => elasticsearch/marvel}/src/main/java/org/elasticsearch/marvel/support/VersionUtils.java (100%) rename {marvel => elasticsearch/marvel}/src/main/resources/marvel_index_template.json (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/MarvelF.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/MarvelPluginTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollectorTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollectorTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollectorTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollectorTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsCollectorTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/collector/shards/ShardsCollectorTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/exporter/ExportersTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterUtilsTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/RendererTestUtils.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRendererTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsRendererTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexRecoveryRendererTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexRecoveryTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRendererTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsRendererTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRendererTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsRendererTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingsTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/license/LicenseIntegrationTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/rest/MarvelRestIT.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/support/VersionUtilsTests.java (100%) rename {marvel => elasticsearch/marvel}/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java (100%) rename {marvel => elasticsearch/marvel}/src/test/resources/log4j.properties (100%) rename {marvel => elasticsearch/marvel}/src/test/resources/rest-api-spec/test/marvel/10_basic.yaml (100%) rename {marvel => elasticsearch/marvel}/src/test/resources/samples/cluster_state.json (100%) rename {marvel => elasticsearch/marvel}/src/test/resources/samples/cluster_stats.json (100%) rename {marvel => elasticsearch/marvel}/src/test/resources/samples/index_recovery.json (100%) rename {marvel => elasticsearch/marvel}/src/test/resources/samples/index_stats.json (100%) rename {marvel => elasticsearch/marvel}/src/test/resources/samples/indices_stats.json (100%) rename {marvel => elasticsearch/marvel}/src/test/resources/samples/node_stats.json (100%) rename {marvel => elasticsearch/marvel}/src/test/resources/samples/shards.json (100%) rename shield/config/shield/role_mapping.yml => elasticsearch/qa/build.gradle (100%) rename {qa => elasticsearch/qa}/messy-test-watcher-with-groovy/build.gradle (100%) rename {qa => elasticsearch/qa}/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ExecutionVarsIntegrationTests.java (100%) rename {qa => elasticsearch/qa}/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/GroovyManualExecutionTests.java (100%) rename {qa => elasticsearch/qa}/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/HistoryTemplateTransformMappingsTests.java (100%) rename {qa => elasticsearch/qa}/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/IndexActionIntegrationTests.java (100%) rename {qa => elasticsearch/qa}/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/MessyTestUtils.java (100%) rename {qa => elasticsearch/qa}/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionSearchTests.java (100%) rename {qa => elasticsearch/qa}/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionTests.java (100%) rename {qa => elasticsearch/qa}/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/TransformIntegrationTests.java (100%) rename {qa => elasticsearch/qa}/shield-audit-tests/build.gradle (76%) rename {qa => elasticsearch/qa}/shield-audit-tests/src/test/java/org/elasticsearch/shield/audit/IndexAuditIT.java (100%) rename {qa => elasticsearch/qa}/shield-client-tests/build.gradle (76%) rename {qa => elasticsearch/qa}/shield-client-tests/src/test/java/org/elasticsearch/shield/qa/ShieldTransportClientIT.java (100%) rename {qa => elasticsearch/qa}/shield-core-rest-tests/build.gradle (88%) rename {qa => elasticsearch/qa}/shield-core-rest-tests/src/test/java/org/elasticsearch/shield/RestIT.java (100%) rename {qa => elasticsearch/qa}/shield-example-realm/build.gradle (78%) rename {qa => elasticsearch/qa}/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java (100%) rename {qa => elasticsearch/qa}/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomAuthenticationFailureHandler.java (100%) rename {qa => elasticsearch/qa}/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealm.java (100%) rename {qa => elasticsearch/qa}/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealmFactory.java (100%) rename {qa => elasticsearch/qa}/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java (100%) rename {qa => elasticsearch/qa}/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java (100%) rename {qa => elasticsearch/qa}/shield-tribe-node-tests/integration-tests.xml (100%) rename {qa => elasticsearch/qa}/shield-tribe-node-tests/rest-api-spec/test/tribe_node/10_basic.yaml (100%) rename {qa => elasticsearch/qa}/shield-tribe-node-tests/shield-roles.yml (100%) rename {qa => elasticsearch/qa}/shield-tribe-node-tests/src/test/java/org/elasticsearch/ant/HttpCondition.java (100%) rename {qa => elasticsearch/qa}/shield-tribe-node-tests/src/test/java/org/elasticsearch/ant/HttpTask.java (100%) rename {qa => elasticsearch/qa}/shield-tribe-node-tests/src/test/java/org/elasticsearch/shield/RestIT.java (100%) rename {qa => elasticsearch/qa}/shield-tribe-node-tests/src/test/java/org/elasticsearch/shield/TribeRestTestCase.java (100%) rename {qa => elasticsearch/qa}/smoke-test-found-license-with-shield-and-watcher/integration-tests.xml (100%) rename {qa => elasticsearch/qa}/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/MarvelClusterInfoIT.java (100%) rename {qa => elasticsearch/qa}/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldIT.java (100%) rename {qa => elasticsearch/qa}/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldInsufficientRoleIT.java (100%) rename {qa => elasticsearch/qa}/smoke-test-found-license-with-shield-and-watcher/watcher-with-shield-roles.yml (100%) rename {qa => elasticsearch/qa}/smoke-test-plugins-ssl/build.gradle (86%) rename {qa => elasticsearch/qa}/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java (100%) rename {qa => elasticsearch/qa}/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml (100%) rename {qa => elasticsearch/qa}/smoke-test-plugins/build.gradle (77%) rename {qa => elasticsearch/qa}/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java (100%) rename {qa => elasticsearch/qa}/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml (100%) rename {qa => elasticsearch/qa}/smoke-test-watcher-with-groovy/build.gradle (53%) rename {qa => elasticsearch/qa}/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java (100%) rename {qa => elasticsearch/qa}/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherWithGroovyIT.java (100%) rename {qa => elasticsearch/qa}/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/10_basic.yaml (100%) rename {qa => elasticsearch/qa}/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/20_minimal_body.yaml (100%) rename {qa => elasticsearch/qa}/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/30_inline_watch.yaml (100%) rename {qa => elasticsearch/qa}/smoke-test-watcher-with-shield/build.gradle (73%) rename {qa => elasticsearch/qa}/smoke-test-watcher-with-shield/roles.yml (100%) rename {qa => elasticsearch/qa}/smoke-test-watcher-with-shield/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldIT.java (100%) rename {qa => elasticsearch/qa}/smoke-test-watcher-with-shield/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldInsufficientRoleIT.java (100%) rename {shield => elasticsearch/shield}/LICENSE.txt (100%) rename {shield => elasticsearch/shield}/NOTICE.txt (100%) rename {shield => elasticsearch/shield}/README.asciidoc (100%) rename {shield => elasticsearch/shield}/TESTING.asciidoc (100%) rename {shield => elasticsearch/shield}/bin/shield/.in.bat (100%) rename {shield => elasticsearch/shield}/bin/shield/esusers (100%) rename {shield => elasticsearch/shield}/bin/shield/esusers.bat (100%) rename {shield => elasticsearch/shield}/bin/shield/syskeygen (100%) rename {shield => elasticsearch/shield}/bin/shield/syskeygen.bat (100%) rename {shield => elasticsearch/shield}/build.gradle (86%) rename {shield => elasticsearch/shield}/config/shield/logging.yml (100%) rename shield/config/shield/users => elasticsearch/shield/config/shield/role_mapping.yml (100%) rename {shield => elasticsearch/shield}/config/shield/roles.yml (100%) rename shield/config/shield/users_roles => elasticsearch/shield/config/shield/users (100%) create mode 100644 elasticsearch/shield/config/shield/users_roles rename {shield => elasticsearch/shield}/dev-tools/esvm/.esvm-shield-config/role_mapping.yml (100%) rename {shield => elasticsearch/shield}/dev-tools/esvm/.esvm-shield-config/roles.yml (100%) rename {shield => elasticsearch/shield}/dev-tools/esvm/.esvm-shield-config/system_key (100%) rename {shield => elasticsearch/shield}/dev-tools/esvm/.esvm-shield-config/users (100%) rename {shield => elasticsearch/shield}/dev-tools/esvm/.esvm-shield-config/users_roles (100%) rename {shield => elasticsearch/shield}/dev-tools/esvm/.esvmrc (100%) rename {shield => elasticsearch/shield}/dev-tools/esvm/readme.txt (100%) rename {shield => elasticsearch/shield}/dev-tools/randomization.yml (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/ShieldBuild.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/ShieldDisabledModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/ShieldLifecycleService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/ShieldModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/ShieldPlugin.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/ShieldSettingsFilter.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/User.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/ShieldActionFilter.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/ShieldActionMapper.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/ShieldActionModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheAction.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequest.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequestBuilder.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheResponse.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/authc/cache/TransportClearRealmCacheAction.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/interceptor/FieldSecurityRequestInterceptor.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/interceptor/RealtimeRequestInterceptor.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/interceptor/RequestInterceptor.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/interceptor/SearchRequestInterceptor.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/action/interceptor/UpdateRequestInterceptor.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/audit/AuditTrail.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/audit/AuditTrailModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/audit/AuditTrailService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/audit/AuditUtil.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditLevel.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditTrail.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditUserHolder.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/audit/index/IndexNameResolver.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrail.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/AnonymousService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/AuthenticationFailureHandler.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/AuthenticationModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/AuthenticationService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/AuthenticationToken.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/DefaultAuthenticationFailureHandler.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/InternalAuthenticationService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/Realm.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/RealmConfig.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/Realms.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryGroupsResolver.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealm.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactory.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/esusers/ESUsersRealm.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStore.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStore.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersTool.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/ldap/LdapRealm.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/ldap/LdapSessionFactory.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactory.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/ldap/SearchGroupsResolver.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/ldap/UserAttributeGroupsResolver.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/ldap/support/AbstractLdapRealm.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapSearchScope.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapSession.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapUtils.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/ldap/support/SessionFactory.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/pki/PkiRealm.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/pki/X509AuthenticationToken.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/support/BCrypt.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/support/CachingRealm.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealm.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/support/CharArrays.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/support/DnRoleMapper.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/support/Hasher.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/support/RefreshListener.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/support/SecuredString.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordRealm.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordToken.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/AuthorizationModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/AuthorizationService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/Permission.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/Privilege.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/SystemRole.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReader.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/accesscontrol/FieldSubsetReader.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/accesscontrol/IndicesAccessControl.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/accesscontrol/OptOutQueryCache.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/accesscontrol/RequestContext.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolver.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/authz/store/RolesStore.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/client/ShieldAuthcClient.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/client/ShieldClient.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/crypto/CryptoModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/crypto/CryptoService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/crypto/tool/SystemKeyTool.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/license/LicenseModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/license/ShieldLicenseState.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/license/ShieldLicensee.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/rest/RemoteHostHeader.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/rest/ShieldRestFilter.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/rest/ShieldRestModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/rest/action/RestShieldInfoAction.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/rest/action/authc/cache/RestClearRealmCacheAction.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/ssl/AbstractSSLService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/ssl/ClientSSLService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/ssl/SSLModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/ssl/ServerSSLService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/support/AbstractShieldModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/support/AutomatonPredicate.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/support/Automatons.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/support/Exceptions.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/support/NoOpLogger.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/support/ShieldFiles.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/support/Validation.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/ClientTransportFilter.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/SSLClientAuth.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/SSLExceptionHelper.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/ServerTransportFilter.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/ShieldClientTransportService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/ShieldServerTransportService.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/ShieldTransportModule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/filter/IPFilter.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRule.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/netty/HandshakeWaitingHandler.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandler.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransport.java (100%) rename {shield => elasticsearch/shield}/src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransport.java (100%) rename {shield => elasticsearch/shield}/src/main/plugin-metadata/plugin-security.policy (100%) rename {shield => elasticsearch/shield}/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-list.help (100%) rename {shield => elasticsearch/shield}/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-passwd.help (100%) rename {shield => elasticsearch/shield}/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-roles.help (100%) rename {shield => elasticsearch/shield}/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-useradd.help (100%) rename {shield => elasticsearch/shield}/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-userdel.help (100%) rename {shield => elasticsearch/shield}/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers.help (100%) rename {shield => elasticsearch/shield}/src/main/resources/org/elasticsearch/shield/crypto/tool/syskey-generate.help (100%) rename {shield => elasticsearch/shield}/src/main/resources/shield-build.properties (100%) rename {shield => elasticsearch/shield}/src/main/resources/shield_audit_log.json (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/bench/HasherBenchmark.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/http/netty/NettyHttpMockUtil.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/LicensingTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/ScrollIdSigningTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/SettingsFilterTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/ShieldCachePermissionTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/ShieldClearScrollTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/ShieldPluginEnabledDisabledTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/ShieldPluginTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/UserTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/VersionCompatibilityTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/action/ShieldActionFilterTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/action/ShieldActionMapperTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/audit/AuditTrailServiceTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditLevelTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailEnabledTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailUpdateMappingTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/audit/index/RemoteIndexAuditTrailStartingTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/audit/logfile/CapturingLogger.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrailTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/AnonymousUserHolderTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/AnonymousUserTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/AuthenticationModuleTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/InternalAuthenticationServiceTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/RealmsTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/RunAsIntegTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryGroupsResolverTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealmTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactoryTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/esusers/ESUsersRealmTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStoreTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStoreTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersToolTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/ldap/LdapRealmTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/ldap/LdapSessionFactoryTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactoryTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/ldap/OpenLdapTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/ldap/SearchGroupsResolverTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/ldap/UserAttributeGroupsResolverTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/ldap/support/LDAPServersTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/ldap/support/LdapTestCase.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/ldap/support/SessionFactoryTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/pki/PkiAuthenticationTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/pki/PkiOptionalClientAuthTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/pki/PkiRealmTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutClientAuthenticationTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutSSLTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/support/BCryptTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/support/DnRoleMapperTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/support/HasherTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/support/SecuredStringTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authc/support/UsernamePasswordTokenTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/AnalyzeTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/PermissionTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/PrivilegeTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/SystemRoleTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldSubsetReaderTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/accesscontrol/IndicesPermissionTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesResolverTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolverIntegrationTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/crypto/InternalCryptoServiceTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/crypto/tool/SystemKeyToolTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/license/ShieldLicenseStateTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/rest/ShieldRestFilterTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/ssl/ClientSSLServiceTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/ssl/SSLSettingsTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/ssl/ServerSSLServiceTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/support/AutomatonsTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/support/ShieldFilesTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/support/ValidationTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/test/ShieldAssertions.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/test/ShieldTestUtils.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/ClientTransportFilterTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/ServerTransportFilterIntegrationTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/ServerTransportFilterTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/TransportFilterTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/filter/IPFilterTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringIntegrationTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringUpdateTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRuleTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/netty/HandshakeWaitingHandlerTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandlerTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/netty/IPHostnameVerificationTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransportTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransportTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/netty/SslHostnameVerificationTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/ssl/SslClientAuthTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/ssl/SslIntegrationTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/transport/ssl/SslMultiPortTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/shield/tribe/TribeShieldLoadedTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/test/ShieldIntegTestCase.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/test/ShieldTestsUtils.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/transport/KnownActionsTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/transport/ShieldServerTransportServiceTests.java (100%) rename {shield => elasticsearch/shield}/src/test/java/org/elasticsearch/transport/netty/NettyMockUtil.java (100%) rename {shield => elasticsearch/shield}/src/test/resources/log4j.properties (100%) rename {shield => elasticsearch/shield}/src/test/resources/logstash-shield.conf (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/activedirectory/ad-schema.ldif (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/activedirectory/ad.ldif (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/activedirectory/role_mapping.yml (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/esusers/users (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/esusers/users_roles (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldapWithGroupSearch.yaml (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldapWithRoleMapping.yaml (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldaptrust.jks (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/ldap/support/seven-seas.ldif (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/pki/role_mapping.yml (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authc/support/role_mapping.yml (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authz/store/invalid_roles.yml (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authz/store/reserved_roles.yml (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/plugin/roles.yml (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/plugin/users (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/plugin/users_roles (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/README.asciidoc (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/activedir.crt (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/openldap.crt (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/openssl_config.cnf (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.cert (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.jks (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.p12 (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.pem (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.cert (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.p12 (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.pem (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.cert (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.jks (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.p12 (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.pem (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-different-passwords.jks (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-ip-only.cert (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-ip-only.jks (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-no-subjaltname.cert (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-no-subjaltname.jks (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.p12 (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.pem (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/truststore-testnode-only.jks (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/transport/actions (100%) rename {shield => elasticsearch/shield}/src/test/resources/org/elasticsearch/transport/handlers (100%) rename {shield => elasticsearch/shield}/test-signatures.txt (100%) rename {watcher => elasticsearch/watcher}/LICENSE.txt (100%) rename {watcher => elasticsearch/watcher}/NOTICE.txt (100%) rename {watcher => elasticsearch/watcher}/README.asciidoc (100%) rename {watcher => elasticsearch/watcher}/bin/watcher/.in.bat (100%) rename {watcher => elasticsearch/watcher}/bin/watcher/croneval (100%) rename {watcher => elasticsearch/watcher}/bin/watcher/croneval.bat (100%) rename {watcher => elasticsearch/watcher}/build.gradle (88%) rename {watcher => elasticsearch/watcher}/dev-tools/randomization.yml (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/WatcherBuild.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/WatcherLifeCycleService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/WatcherMetaData.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/WatcherModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/WatcherPlugin.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/WatcherService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/WatcherState.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/Action.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/ActionBuilders.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/ActionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/ActionRegistry.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/ActionStatus.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/ActionWrapper.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/ExecutableAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/ExecutableActions.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/WatcherActionModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/DataAttachment.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/EmailAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/EmailActionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/ExecutableEmailAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/Account.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/Accounts.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/Attachment.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/Authentication.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/Email.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/EmailService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/EmailTemplate.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/HtmlSanitizer.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/Inline.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/InternalEmailService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/Profile.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/email/service/support/BodyPartSource.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/ExecutableHipChatAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccount.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccounts.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessage.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatServer.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccount.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/InternalHipChatService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/SentMessages.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccount.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/V1Account.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/index/ExecutableIndexAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/index/IndexAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/index/IndexActionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/logging/ExecutableLoggingAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingActionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingLevel.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/ExecutableSlackAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/SlackAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/SlackActionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/InternalSlackService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/SentMessages.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackAccount.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackAccounts.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/Attachment.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/DynamicAttachments.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/Field.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/MessageElement.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessage.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageDefaults.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/throttler/AckThrottler.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/throttler/ActionThrottler.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/throttler/PeriodThrottler.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/throttler/Throttler.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/webhook/ExecutableWebhookAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/webhook/WebhookAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/actions/webhook/WebhookActionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/client/WatchSourceBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/client/WatchSourceBuilders.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/client/WatcherClient.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/client/WatcherClientModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/Condition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/ConditionBuilders.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/ConditionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/ConditionModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/ConditionRegistry.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/ExecutableCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/always/AlwaysCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/always/AlwaysConditionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/always/ExecutableAlwaysCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/compare/AbstractExecutableCompareCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/compare/CompareCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/compare/CompareConditionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/compare/ExecutableCompareCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/compare/LenientCompare.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/compare/array/ExecutableArrayCompareCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/never/ExecutableNeverCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/never/NeverCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/never/NeverConditionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/script/ExecutableScriptCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/script/ScriptCondition.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/condition/script/ScriptConditionFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/ActionExecutionMode.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/AsyncTriggerListener.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/CurrentExecutions.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/ExecutionModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/ExecutionPhase.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/ExecutionService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/ExecutionState.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/InternalWatchExecutor.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/ManualExecutionContext.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/QueuedWatch.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/SyncTriggerListener.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/TriggeredExecutionContext.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatch.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionContext.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionResult.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionSnapshot.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/WatchExecutor.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/execution/Wid.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/history/HistoryModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/history/WatchRecord.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/ExecutableInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/Input.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/InputBuilders.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/InputFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/InputModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/InputRegistry.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/chain/ChainInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/chain/ChainInputFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/chain/ExecutableChainInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/http/ExecutableHttpInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/http/HttpInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/http/HttpInputFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/none/ExecutableNoneInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/none/NoneInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/none/NoneInputFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/search/ExecutableSearchInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/search/SearchInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/search/SearchInputFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/simple/ExecutableSimpleInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/simple/SimpleInput.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/input/simple/SimpleInputFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/license/LicenseModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/license/WatcherLicensee.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/WatcherRestHandler.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/action/RestAckWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/action/RestActivateWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/action/RestDeleteWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/action/RestExecuteWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/action/RestGetWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/action/RestHijackOperationAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/action/RestPutWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/action/RestWatchServiceAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/action/RestWatcherInfoAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/rest/action/RestWatcherStatsAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/shield/ShieldIntegration.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/shield/ShieldSecretService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/shield/WatcherSettingsFilter.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/shield/WatcherShieldModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/shield/WatcherUserHolder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/ArrayObjectIterator.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/Exceptions.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/Integers.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/Script.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/SearchRequestEquivalence.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/Strings.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/ThreadPoolSettingsBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/Variables.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/WatcherDateTimeUtils.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistry.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/WatcherUtils.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/XContentFilterKeysUtils.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/clock/Clock.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/clock/ClockModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/clock/HaltedClock.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/clock/SystemClock.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/concurrent/FairKeyedLock.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/HttpClient.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/HttpClientModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/HttpContentType.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/HttpMethod.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/HttpProxy.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/HttpRequest.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/HttpRequestTemplate.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/HttpResponse.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/Scheme.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/auth/ApplicableHttpAuth.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuth.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuthFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuthRegistry.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/ApplicableBasicAuth.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/BasicAuth.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/BasicAuthFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/init/InitializingModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/init/InitializingService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/init/proxy/ClientProxy.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/init/proxy/ScriptServiceProxy.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/secret/Secret.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/secret/SecretModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/secret/SecretService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/text/TextTemplate.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/text/TextTemplateEngine.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/text/TextTemplateModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTextTemplateEngine.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/validation/Validation.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/validation/WatcherSettingsValidation.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/xcontent/ObjectPath.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherParams.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentParser.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentUtils.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/support/xcontent/XContentSource.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/ExecutableTransform.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/Transform.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/TransformBuilders.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/TransformFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/TransformModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/TransformRegistry.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/chain/ChainTransform.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/chain/ChainTransformFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/chain/ExecutableChainTransform.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/script/ExecutableScriptTransform.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/script/ScriptTransform.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/script/ScriptTransformFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/search/ExecutableSearchTransform.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/search/SearchTransform.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transform/search/SearchTransformFactory.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/WatcherTransportAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchRequest.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchRequestBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchResponse.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/ack/TransportAckWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchRequest.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchRequestBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchResponse.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/activate/TransportActivateWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchRequest.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchResponse.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/delete/TransportDeleteWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchRequest.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchRequestBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchResponse.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/execute/TransportExecuteWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchRequest.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchRequestBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchResponse.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/get/TransportGetWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchRequest.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchRequestBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchResponse.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/put/TransportPutWatchAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/service/TransportWatcherServiceAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceRequest.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceRequestBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceResponse.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/stats/TransportWatcherStatsAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsAction.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsRequest.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsRequestBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsResponse.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/AbstractTriggerEngine.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/Trigger.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/TriggerBuilders.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/TriggerEngine.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/TriggerEvent.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/TriggerModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/TriggerService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTrigger.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEngine.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEvent.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/Cron.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/CronSchedule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/CronnableSchedule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/DailySchedule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/HourlySchedule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/IntervalSchedule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/MonthlySchedule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/Schedule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleRegistry.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTrigger.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEngine.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEvent.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/Schedules.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/WeeklySchedule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/YearlySchedule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleTriggerEngine.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/DayOfWeek.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/DayTimes.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/Month.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/MonthTimes.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/Times.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/WeekTimes.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/YearTimes.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/trigger/schedule/tool/CronEvalTool.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/watch/Payload.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/watch/Watch.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/watch/WatchLockService.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/watch/WatchModule.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/watch/WatchStatus.java (100%) rename {watcher => elasticsearch/watcher}/src/main/java/org/elasticsearch/watcher/watch/WatchStore.java (100%) rename {watcher => elasticsearch/watcher}/src/main/plugin-metadata/plugin-security.policy (100%) rename {watcher => elasticsearch/watcher}/src/main/resources/org/elasticsearch/watcher/trigger/schedule/tool/croneval-eval.help (100%) rename {watcher => elasticsearch/watcher}/src/main/resources/triggered_watches.json (100%) rename {watcher => elasticsearch/watcher}/src/main/resources/watch_history.json (100%) rename {watcher => elasticsearch/watcher}/src/main/resources/watcher-build.properties (100%) rename {watcher => elasticsearch/watcher}/src/main/resources/watches.json (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/script/SleepScriptEngine.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/WatcherF.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/WatcherLifeCycleServiceTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/WatcherPluginDisableTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/WatcherPluginTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/WatcherServiceTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/ActionErrorIntegrationTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/TimeThrottleIntegrationTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/DataAttachmentTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionIntegrationTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/EmailSecretsIntegrationTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/service/AccountTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/service/AccountsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/service/EmailTemplateTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/service/EmailTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/service/HtmlSanitizerTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/service/InternalEmailServiceTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/service/ManualPublicSmtpServersTester.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/email/service/support/EmailServer.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionFactoryTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccountsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessageTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatServiceTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccountTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/InternalHipChatServiceTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccountTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/V1AccountTests.java (100%) create mode 100644 elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionIntegrationTests.java rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/logging/LoggingActionTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/slack/SlackActionFactoryTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/slack/SlackActionTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/slack/service/SlackAccountsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/slack/service/SlackServiceTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageDefaultsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/throttler/AckThrottlerTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/throttler/ActionThrottleTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/throttler/PeriodThrottlerTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/throttler/WatchThrottlerTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookActionTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookHttpsIntegrationTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookIntegrationTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/condition/always/AlwaysConditionTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/condition/compare/CompareConditionSearchTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/condition/compare/CompareConditionTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionSearchTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/condition/never/NeverConditionTests.java (100%) create mode 100644 elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionSearchTests.java create mode 100644 elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionTests.java rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/execution/ExecutionServiceTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/execution/ManualExecutionTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchStoreLifeCycleTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchStoreTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/history/HistoryStoreSettingsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/history/HistoryStoreTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateEmailMappingsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateHttpMappingsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateIndexActionMappingsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateSearchInputMappingsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTimeMappingsTests.java (100%) create mode 100644 elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTransformMappingsTests.java rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/input/InputRegistryTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/input/chain/ChainInputTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/input/chain/ChainIntegrationTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/input/http/HttpInputIntegrationTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/input/http/HttpInputTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/input/search/SearchInputTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/input/simple/SimpleInputTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/license/LicenseTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/shield/BasicShieldTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/DynamicIndexNameIntegrationTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/FilterXContentTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/VariablesTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/WatcherDateTimeUtilsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistryTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/WatcherUtilsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/clock/ClockMock.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/clock/ClockTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/concurrent/FairKeyedLockTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/http/HttpClientTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/http/HttpConnectionTimeoutTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/http/HttpReadTimeoutTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/http/HttpRequestTemplateTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/http/HttpResponseTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/text/TextTemplateTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/xcontent/MapPathTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/support/xcontent/XContentSourceTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/TimeWarpedWatcherPlugin.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/WatchExecutionContextMockBuilder.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/WatcherMatchers.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/WatcherTestUtils.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/bench/ScheduleEngineTriggerBenchmark.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/bench/WatcherExecutorServiceBenchmark.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/integration/BasicWatcherTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/integration/BootStrapTests.java (100%) create mode 100644 elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/ExecutionVarsIntegrationTests.java rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/integration/HttpSecretsIntegrationTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/integration/NoMasterNodeTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/integration/WatchMetadataTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/integration/WatcherSettingsFilterTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/rest/WatcherRestIT.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/test/rest/WatcherRestTestCase.java (100%) create mode 100644 elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transform/TransformIntegrationTests.java rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transform/chain/ChainTransformTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transform/script/ScriptTransformTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transport/action/ack/WatchAckTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transport/action/activate/ActivateWatchTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transport/action/delete/DeleteWatchTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transport/action/delete/ForceDeleteWatchTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchWithDateMathTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transport/action/get/GetWatchTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transport/action/put/PutWatchTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transport/action/stats/SlowWatchStatsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/transport/action/stats/WatcherStatsTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/ScheduleTriggerEngineMock.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/CronScheduleTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/DailyScheduleTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/HourlyScheduleTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/IntervalScheduleTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/MonthlyScheduleTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleRegistryTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTestCase.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEventTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/WeeklyScheduleTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/YearlyScheduleTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/BaseTriggerEngineTestCase.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleEngineTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/tool/CronEvalToolTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/trigger/schedule/tool/EvalCron.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/watch/WatchLockServiceTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/watch/WatchStoreTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/java/org/elasticsearch/watcher/watch/WatchTests.java (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/config/scripts/my-script.groovy (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/org/elasticsearch/shield/keystore/testnode-different-passwords.jks (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/org/elasticsearch/shield/keystore/testnode.cert (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/org/elasticsearch/shield/keystore/testnode.jks (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/org/elasticsearch/shield/keystore/truststore-testnode-only.jks (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/org/elasticsearch/watcher/actions/email/service/logo.png (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/org/elasticsearch/watcher/input/search/config/scripts/test_disk_template.mustache (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/org/elasticsearch/watcher/transform/search/config/scripts/test_disk_template.mustache (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.ack_watch.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.activate_watch.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.deactivate_watch.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.delete_watch.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.execute_watch.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.get_watch.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.info.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.put_watch.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.restart.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.start.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.stats.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/api/watcher.stop.json (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/ack_watch/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/ack_watch/20_ack_individual_action.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/activate_watch/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/array_compare_watch/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/delete_watch/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/get_watch/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/get_watch/20_missing.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/getting_started/10_monitor_cluster_health.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/hijack/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/put_watch/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/put_watch/20_put_watch_with_throttle_period.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/put_watch/30_put_watch_with_action_throttle_period.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/put_watch/40_put_watch_as_inactive.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/restart_watcher/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/start_watcher/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/stats/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/stop_watcher/10_basic.yaml (100%) rename {watcher => elasticsearch/watcher}/src/test/resources/rest-api-spec/test/watch_info/10_basic.yaml (100%) rename {x-dev-tools => elasticsearch/x-dev-tools}/RELEASE.md (100%) rename {x-dev-tools => elasticsearch/x-dev-tools}/src/main/resources/ant/shield-overrides.xml (100%) rename {x-dev-tools => elasticsearch/x-dev-tools}/src/main/resources/commercial-license-check/elasticsearch_license_header.txt (100%) rename {x-dev-tools => elasticsearch/x-dev-tools}/src/main/resources/commercial-license-check/license_header_definition.xml (100%) delete mode 100644 shield/docs/private/indices_replace.asciidoc delete mode 100644 shield/docs/private/ldap-testing.asciidoc delete mode 100644 shield/docs/public/configuring-auditing.asciidoc delete mode 100644 shield/docs/public/configuring-clients-integrations.asciidoc delete mode 100644 shield/docs/public/configuring-clients-integrations/hadoop.asciidoc delete mode 100644 shield/docs/public/configuring-clients-integrations/http.asciidoc delete mode 100644 shield/docs/public/configuring-clients-integrations/java.asciidoc delete mode 100644 shield/docs/public/configuring-clients-integrations/kibana.asciidoc delete mode 100644 shield/docs/public/configuring-clients-integrations/logstash.asciidoc delete mode 100644 shield/docs/public/configuring-clients-integrations/marvel.asciidoc delete mode 100644 shield/docs/public/configuring-rbac.asciidoc delete mode 100644 shield/docs/public/example-deployments.asciidoc delete mode 100644 shield/docs/public/example-deployments/e-commerce.asciidoc delete mode 100644 shield/docs/public/getting-started.asciidoc delete mode 100644 shield/docs/public/getting-started/enable-auditing.asciidoc delete mode 100644 shield/docs/public/getting-started/enable-basic-auth.asciidoc delete mode 100644 shield/docs/public/getting-started/enable-message-authentication.asciidoc delete mode 100644 shield/docs/public/getting-started/moving-on.asciidoc delete mode 100644 shield/docs/public/granting-alias-privileges.asciidoc delete mode 100644 shield/docs/public/how-shield-works.asciidoc delete mode 100644 shield/docs/public/index.asciidoc delete mode 100644 shield/docs/public/installing-shield.asciidoc delete mode 100644 shield/docs/public/introduction.asciidoc delete mode 100644 shield/docs/public/limitations.asciidoc delete mode 100644 shield/docs/public/managing-shield-licenses.asciidoc delete mode 100644 shield/docs/public/managing-users.asciidoc delete mode 100644 shield/docs/public/mapping-roles.asciidoc delete mode 100644 shield/docs/public/reference.asciidoc delete mode 100644 shield/docs/public/release-notes.asciidoc delete mode 100644 shield/docs/public/securing-communications.asciidoc delete mode 100644 shield/docs/public/securing-communications/enabling-cipher-suites.asciidoc delete mode 100644 shield/docs/public/securing-communications/separating-node-client-traffic.asciidoc delete mode 100644 shield/docs/public/securing-communications/setting-up-ssl.asciidoc delete mode 100644 shield/docs/public/securing-communications/using-ip-filtering.asciidoc delete mode 100644 shield/docs/public/setting-up-authentication.asciidoc delete mode 100644 shield/docs/public/setting-up-authentication/configuring-active-directory-realm.asciidoc delete mode 100644 shield/docs/public/setting-up-authentication/configuring-esusers-realm.asciidoc delete mode 100644 shield/docs/public/setting-up-authentication/configuring-ldap-realm.asciidoc delete mode 100644 shield/docs/public/setting-up-authentication/configuring-pki-realm.asciidoc delete mode 100644 shield/docs/public/setting-up-authentication/controlling-user-cache.asciidoc delete mode 100644 shield/docs/public/setting-up-authentication/enabling-anonymous-access.asciidoc delete mode 100644 shield/docs/public/setting-up-authentication/integrating-other-auth-systems.asciidoc delete mode 100644 shield/docs/public/setting-up-certificate-authority.asciidoc delete mode 100644 shield/docs/public/setting-up-field-and-document-level-security.asciidoc delete mode 100644 shield/docs/public/submitting-requests-for-other-users.asciidoc delete mode 100644 shield/docs/public/troubleshooting.asciidoc delete mode 100644 watcher/docs/administering-watcher.asciidoc delete mode 100644 watcher/docs/administering-watcher/configuring-default-http-timeouts.asciidoc delete mode 100644 watcher/docs/administering-watcher/configuring-default-internal-ops-timeouts.asciidoc delete mode 100644 watcher/docs/administering-watcher/configuring-default-throttle-period.asciidoc delete mode 100644 watcher/docs/administering-watcher/configuring-email.asciidoc delete mode 100644 watcher/docs/administering-watcher/configuring-hipchat.asciidoc delete mode 100644 watcher/docs/administering-watcher/configuring-slack.asciidoc delete mode 100644 watcher/docs/administering-watcher/getting-watcher-statistics.asciidoc delete mode 100644 watcher/docs/administering-watcher/integrating-with-logstash.asciidoc delete mode 100644 watcher/docs/administering-watcher/integrating-with-shield.asciidoc delete mode 100644 watcher/docs/administering-watcher/monitoring-watch-execution.asciidoc delete mode 100644 watcher/docs/customizing-watches.asciidoc delete mode 100644 watcher/docs/example-watches.asciidoc delete mode 100644 watcher/docs/example-watches/watching-marvel-data.asciidoc delete mode 100644 watcher/docs/example-watches/watching-time-series-data.asciidoc delete mode 100644 watcher/docs/getting-started.asciidoc delete mode 100644 watcher/docs/how-watcher-works.asciidoc delete mode 100644 watcher/docs/how-watcher-works/templates.asciidoc delete mode 100644 watcher/docs/images/action-throttling.jpg delete mode 100644 watcher/docs/images/hipchat-copy-room-token.jpg delete mode 100644 watcher/docs/images/hipchat-copy-user-token.jpg delete mode 100644 watcher/docs/images/hipchat-copy-v1-token.jpg delete mode 100644 watcher/docs/images/hipchat-generate-room-token.jpg delete mode 100644 watcher/docs/images/hipchat-generate-user-token.jpg delete mode 100644 watcher/docs/images/hipchat-generate-v1-token.jpg delete mode 100644 watcher/docs/images/hipchat-integration-example.png delete mode 100644 watcher/docs/images/slack-add-webhook-integration.jpg delete mode 100644 watcher/docs/images/slack-copy-webhook-url.jpg delete mode 100644 watcher/docs/images/watch-execution.jpg delete mode 100644 watcher/docs/images/watcher-kibana-dashboard.png delete mode 100644 watcher/docs/images/watcher.graffle delete mode 100644 watcher/docs/index.asciidoc delete mode 100644 watcher/docs/installing-watcher.asciidoc delete mode 100644 watcher/docs/introduction.asciidoc delete mode 100644 watcher/docs/managing-watcher-licenses.asciidoc delete mode 100644 watcher/docs/managing-watches.asciidoc delete mode 100644 watcher/docs/managing-watches/deleting-watches.asciidoc delete mode 100644 watcher/docs/managing-watches/listing-watches.asciidoc delete mode 100644 watcher/docs/reference.asciidoc delete mode 100644 watcher/docs/reference/actions.asciidoc delete mode 100644 watcher/docs/reference/actions/email.asciidoc delete mode 100644 watcher/docs/reference/actions/hipchat.asciidoc delete mode 100644 watcher/docs/reference/actions/index.asciidoc delete mode 100644 watcher/docs/reference/actions/logging.asciidoc delete mode 100644 watcher/docs/reference/actions/slack.asciidoc delete mode 100644 watcher/docs/reference/actions/webhook.asciidoc delete mode 100644 watcher/docs/reference/condition.asciidoc delete mode 100644 watcher/docs/reference/condition/always.asciidoc delete mode 100644 watcher/docs/reference/condition/array-compare.asciidoc delete mode 100644 watcher/docs/reference/condition/compare.asciidoc delete mode 100644 watcher/docs/reference/condition/never.asciidoc delete mode 100644 watcher/docs/reference/condition/script.asciidoc delete mode 100644 watcher/docs/reference/input.asciidoc delete mode 100644 watcher/docs/reference/input/chain.asciidoc delete mode 100644 watcher/docs/reference/input/http.asciidoc delete mode 100644 watcher/docs/reference/input/search.asciidoc delete mode 100644 watcher/docs/reference/input/simple.asciidoc delete mode 100644 watcher/docs/reference/java.asciidoc delete mode 100644 watcher/docs/reference/java/ack-watch.asciidoc delete mode 100644 watcher/docs/reference/java/activate-watch.asciidoc delete mode 100644 watcher/docs/reference/java/deactivate-watch.asciidoc delete mode 100644 watcher/docs/reference/java/delete-watch.asciidoc delete mode 100644 watcher/docs/reference/java/execute-watch.asciidoc delete mode 100644 watcher/docs/reference/java/get-watch.asciidoc delete mode 100644 watcher/docs/reference/java/put-watch.asciidoc delete mode 100644 watcher/docs/reference/java/service.asciidoc delete mode 100644 watcher/docs/reference/java/stats.asciidoc delete mode 100644 watcher/docs/reference/rest.asciidoc delete mode 100644 watcher/docs/reference/rest/ack-watch.asciidoc delete mode 100644 watcher/docs/reference/rest/activate-watch.asciidoc delete mode 100644 watcher/docs/reference/rest/deactivate-watch.asciidoc delete mode 100644 watcher/docs/reference/rest/delete-watch.asciidoc delete mode 100644 watcher/docs/reference/rest/execute-watch.asciidoc delete mode 100644 watcher/docs/reference/rest/get-watch.asciidoc delete mode 100644 watcher/docs/reference/rest/info.asciidoc delete mode 100644 watcher/docs/reference/rest/put-watch.asciidoc delete mode 100644 watcher/docs/reference/rest/restart.asciidoc delete mode 100644 watcher/docs/reference/rest/start.asciidoc delete mode 100644 watcher/docs/reference/rest/stats.asciidoc delete mode 100644 watcher/docs/reference/rest/stop.asciidoc delete mode 100644 watcher/docs/reference/transform.asciidoc delete mode 100644 watcher/docs/reference/transform/chain.asciidoc delete mode 100644 watcher/docs/reference/transform/script.asciidoc delete mode 100644 watcher/docs/reference/transform/search.asciidoc delete mode 100644 watcher/docs/reference/trigger.asciidoc delete mode 100644 watcher/docs/reference/trigger/schedule.asciidoc delete mode 100644 watcher/docs/reference/trigger/schedule/cron.asciidoc delete mode 100644 watcher/docs/reference/trigger/schedule/daily.asciidoc delete mode 100644 watcher/docs/reference/trigger/schedule/hourly.asciidoc delete mode 100644 watcher/docs/reference/trigger/schedule/interval.asciidoc delete mode 100644 watcher/docs/reference/trigger/schedule/monthly.asciidoc delete mode 100644 watcher/docs/reference/trigger/schedule/weekly.asciidoc delete mode 100644 watcher/docs/reference/trigger/schedule/yearly.asciidoc delete mode 100644 watcher/docs/release-notes.asciidoc delete mode 100644 watcher/docs/troubleshooting.asciidoc diff --git a/qa/build.gradle b/elasticsearch/build.gradle similarity index 100% rename from qa/build.gradle rename to elasticsearch/build.gradle diff --git a/elasticsearch/license/README.md b/elasticsearch/license/README.md new file mode 100644 index 00000000000..37aecd4c25f --- /dev/null +++ b/elasticsearch/license/README.md @@ -0,0 +1,36 @@ +elasticsearch-license +===================== + +Elasticsearch Licensing core, tools and plugin + +## Core + +Contains core data structures, utilities used by **Licensor** and **Plugin**. + +See `core/` and `core-shaded/` + +## Licensor + +Contains a collection of tools to generate key-pairs, licenses and validate licenses. + +See `licensor/` + +see [wiki] (https://github.com/elasticsearch/elasticsearch-license/wiki) for documentation on +[Licensing Tools Usage & Reference] (https://github.com/elasticsearch/elasticsearch-license/wiki/License-Tools-Usage-&-Reference) + +## Plugin + +**NOTE**: The license plugin has to be packaged with the right public key when being deployed to public repositories in maven +or uploaded to s3. Use `-Dkeys.path=` with maven command to package the plugin with a specified key. + +See `plugin/` + +see [Getting Started] (https://github.com/elasticsearch/elasticsearch-license/blob/master/docs/getting-started.asciidoc) to install license plugin. + +see [Licensing REST APIs] (https://github.com/elasticsearch/elasticsearch-license/blob/master/docs/license.asciidoc) +to use the license plugin from an elasticsearch deployment. + +see [wiki] (https://github.com/elasticsearch/elasticsearch-license/wiki) for documentation on + - [License Plugin Consumer Interface] (https://github.com/elasticsearch/elasticsearch-license/wiki/License---Consumer-Interface) + - [License Plugin Release Process] (https://github.com/elasticsearch/elasticsearch-license/wiki/Plugin-Release-Process) + - [License Plugin Design] (https://github.com/elasticsearch/elasticsearch-license/wiki/License-Plugin--Design) diff --git a/elasticsearch/license/base/build.gradle b/elasticsearch/license/base/build.gradle new file mode 100644 index 00000000000..fefabaacf5a --- /dev/null +++ b/elasticsearch/license/base/build.gradle @@ -0,0 +1,13 @@ +apply plugin: 'elasticsearch.build' + +dependencies { + compile "org.elasticsearch:elasticsearch:${version}" + testCompile "org.elasticsearch:test-framework:${version}" +} + +dependencyLicenses.enabled = false + +jar { + baseName = 'license-core' +} + diff --git a/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/CryptUtils.java b/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/CryptUtils.java new file mode 100644 index 00000000000..939b5192828 --- /dev/null +++ b/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/CryptUtils.java @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.core; + + +import org.elasticsearch.common.Base64; + +import javax.crypto.*; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.security.*; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; + +public class CryptUtils { + private static final int minimumPadding = 20; + private static final byte[] salt = { + (byte) 0xA9, (byte) 0xA2, (byte) 0xB5, (byte) 0xDE, + (byte) 0x2A, (byte) 0x8A, (byte) 0x9A, (byte) 0xE6 + }; + private static final int iterationCount = 1024; + private static final int aesKeyLength = 128; + private static final String keyAlgorithm = "RSA"; + private static final String passHashAlgorithm = "SHA-512"; + private static final String DEFAULT_PASS_PHRASE = "elasticsearch-license"; + + private static final SecureRandom random = new SecureRandom(); + + /** + * Read encrypted private key file content with default pass phrase + */ + public static PrivateKey readEncryptedPrivateKey(byte[] fileContents) { + try { + return readEncryptedPrivateKey(fileContents, hashPassPhrase(DEFAULT_PASS_PHRASE)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /** + * Read encrypted public key file content with default pass phrase + */ + public static PublicKey readEncryptedPublicKey(byte[] fileContents) { + try { + return readEncryptedPublicKey(fileContents, hashPassPhrase(DEFAULT_PASS_PHRASE)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /** + * Returns encrypted public key file content with default pass phrase + */ + public static byte[] writeEncryptedPublicKey(PublicKey publicKey) { + try { + return writeEncryptedPublicKey(publicKey, hashPassPhrase(DEFAULT_PASS_PHRASE)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /** + * Returns encrypted private key file content with default pass phrase + */ + public static byte[] writeEncryptedPrivateKey(PrivateKey privateKey) { + try { + return writeEncryptedPrivateKey(privateKey, hashPassPhrase(DEFAULT_PASS_PHRASE)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /** + * Read encrypted private key file content with provided passPhrase + */ + public static PrivateKey readEncryptedPrivateKey(byte[] fileContents, char[] passPhrase) { + PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(decrypt(fileContents, passPhrase)); + try { + return KeyFactory.getInstance(keyAlgorithm).generatePrivate(privateKeySpec); + } catch (NoSuchAlgorithmException | InvalidKeySpecException e) { + throw new IllegalStateException(e); + } + } + + /** + * Read encrypted public key file content with provided passPhrase + */ + public static PublicKey readEncryptedPublicKey(byte[] fileContents, char[] passPhrase) { + X509EncodedKeySpec publicKeySpec = new X509EncodedKeySpec(decrypt(fileContents, passPhrase)); + try { + return KeyFactory.getInstance(CryptUtils.keyAlgorithm).generatePublic(publicKeySpec); + } catch (NoSuchAlgorithmException | InvalidKeySpecException e) { + throw new IllegalStateException(e); + } + } + + /** + * Returns encrypted public key file content with provided passPhrase + */ + public static byte[] writeEncryptedPublicKey(PublicKey publicKey, char[] passPhrase) { + X509EncodedKeySpec encodedKeySpec = new X509EncodedKeySpec(publicKey.getEncoded()); + return encrypt(encodedKeySpec.getEncoded(), passPhrase); + } + + /** + * Returns encrypted private key file content with provided passPhrase + */ + public static byte[] writeEncryptedPrivateKey(PrivateKey privateKey, char[] passPhrase) { + PKCS8EncodedKeySpec encodedKeySpec = new PKCS8EncodedKeySpec(privateKey.getEncoded()); + return encrypt(encodedKeySpec.getEncoded(), passPhrase); + } + + /** + * Encrypts provided data with DEFAULT_PASS_PHRASE + */ + public static byte[] encrypt(byte[] data) { + try { + return encrypt(data, hashPassPhrase(DEFAULT_PASS_PHRASE)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /** + * Decrypts provided encryptedData with DEFAULT_PASS_PHRASE + */ + public static byte[] decrypt(byte[] encryptedData) { + try { + return decrypt(encryptedData, hashPassPhrase(DEFAULT_PASS_PHRASE)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /** + * Encrypts provided data with passPhrase + */ + public static byte[] encrypt(byte[] data, char[] passPhrase) { + try { + final Cipher encryptionCipher = getEncryptionCipher(getSecretKey(passPhrase)); + return encryptionCipher.doFinal(pad(data, minimumPadding)); + } catch (InvalidKeySpecException | IllegalBlockSizeException | BadPaddingException e) { + throw new IllegalStateException(e); + } + } + + /** + * Decrypts provided encryptedData with passPhrase + */ + private static byte[] decrypt(byte[] encryptedData, char[] passPhrase) { + try { + final Cipher cipher = getDecryptionCipher(getSecretKey(passPhrase)); + return unPad(cipher.doFinal(encryptedData)); + } catch (IllegalBlockSizeException | BadPaddingException | InvalidKeySpecException e) { + throw new IllegalStateException(e); + } + + } + + private static SecretKey getSecretKey(char[] passPhrase) throws InvalidKeySpecException { + try { + PBEKeySpec keySpec = new PBEKeySpec(passPhrase, salt, iterationCount, aesKeyLength); + + byte[] shortKey = SecretKeyFactory.getInstance("PBEWithSHA1AndDESede"). + generateSecret(keySpec).getEncoded(); + + byte[] intermediaryKey = new byte[aesKeyLength / 8]; + for (int i = 0, j = 0; i < aesKeyLength / 8; i++) { + intermediaryKey[i] = shortKey[j]; + if (++j == shortKey.length) + j = 0; + } + + return new SecretKeySpec(intermediaryKey, "AES"); + } catch (NoSuchAlgorithmException | InvalidKeySpecException e) { + throw new IllegalStateException(e); + } + } + + private static Cipher getEncryptionCipher(SecretKey secretKey) { + return getCipher(Cipher.ENCRYPT_MODE, secretKey); + } + + private static Cipher getDecryptionCipher(SecretKey secretKey) { + return getCipher(Cipher.DECRYPT_MODE, secretKey); + } + + private static Cipher getCipher(int mode, SecretKey secretKey) { + try { + Cipher cipher = Cipher.getInstance(secretKey.getAlgorithm()); + cipher.init(mode, secretKey, random); + return cipher; + } catch (NoSuchAlgorithmException | InvalidKeyException | NoSuchPaddingException e) { + throw new IllegalStateException(e); + } + } + + private static byte[] pad(byte[] bytes, int length) { + if (bytes.length >= length) { + byte[] out = new byte[bytes.length + 1]; + System.arraycopy(bytes, 0, out, 0, bytes.length); + out[bytes.length] = (byte) 1; + return out; + } + + byte[] out = new byte[length + 1]; + + int i = 0; + for (; i < bytes.length; i++) + out[i] = bytes[i]; + + int padded = length - i; + + // fill the rest with random bytes + byte[] fill = new byte[padded - 1]; + random.nextBytes(fill); + System.arraycopy(fill, 0, out, i, padded - 1); + + out[length] = (byte) (padded + 1); + + return out; + } + + private static byte[] unPad(byte[] bytes) { + int padded = (int) bytes[bytes.length - 1]; + int targetLength = bytes.length - padded; + + byte[] out = new byte[targetLength]; + + System.arraycopy(bytes, 0, out, 0, targetLength); + + return out; + } + + private static char[] hashPassPhrase(String passPhrase) throws NoSuchAlgorithmException { + final byte[] passBytes = passPhrase.getBytes(StandardCharsets.UTF_8); + final byte[] digest = MessageDigest.getInstance(passHashAlgorithm).digest(passBytes); + return new String(Base64.encodeBytesToBytes(digest), StandardCharsets.UTF_8).toCharArray(); + } +} diff --git a/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/DateUtils.java b/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/DateUtils.java new file mode 100644 index 00000000000..80b42aa82bf --- /dev/null +++ b/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/DateUtils.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.core; + +import org.elasticsearch.common.joda.FormatDateTimeFormatter; +import org.elasticsearch.common.joda.Joda; +import org.joda.time.MutableDateTime; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +public class DateUtils { + + private final static FormatDateTimeFormatter formatDateOnlyFormatter = Joda.forPattern("yyyy-MM-dd"); + + private final static DateTimeFormatter dateOnlyFormatter = formatDateOnlyFormatter.parser().withZoneUTC(); + + private final static DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime().withZoneUTC(); + + public static long endOfTheDay(String date) { + try { + // Try parsing using complete date/time format + return dateTimeFormatter.parseDateTime(date).getMillis(); + } catch (IllegalArgumentException ex) { + // Fall back to the date only format + MutableDateTime dateTime = dateOnlyFormatter.parseMutableDateTime(date); + dateTime.millisOfDay().set(dateTime.millisOfDay().getMaximumValue()); + return dateTime.getMillis(); + } + } + + public static long beginningOfTheDay(String date) { + try { + // Try parsing using complete date/time format + return dateTimeFormatter.parseDateTime(date).getMillis(); + } catch (IllegalArgumentException ex) { + // Fall back to the date only format + return dateOnlyFormatter.parseDateTime(date).getMillis(); + } + + } +} diff --git a/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/License.java b/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/License.java new file mode 100644 index 00000000000..418a07e524d --- /dev/null +++ b/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/License.java @@ -0,0 +1,693 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.core; + +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Base64; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.*; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; + +/** + * Data structure for license. Use {@link Builder} to build a license. + * Provides serialization/deserialization & validation methods for license object + */ +public class License implements ToXContent { + public final static int VERSION_START = 1; + public final static int VERSION_NO_FEATURE_TYPE = 2; + public final static int VERSION_CURRENT = VERSION_NO_FEATURE_TYPE; + + /** + * XContent param name to deserialize license(s) with + * an additional status field, indicating whether a + * particular license is 'active' or 'expired' and no signature + * and in a human readable format + */ + public static final String REST_VIEW_MODE = "rest_view"; + /** + * XContent param name to deserialize license(s) with + * no signature + */ + public static final String LICENSE_SPEC_VIEW_MODE = "license_spec_view"; + /** + * XContent param name to deserialize licenses according + * to a specific license version + */ + public static final String LICENSE_VERSION_MODE = "license_version"; + + public final static Comparator LATEST_ISSUE_DATE_FIRST = new Comparator() { + @Override + public int compare(License right, License left) { + return Long.compare(left.issueDate(), right.issueDate()); + } + }; + + private final int version; + private final String uid; + private final String issuer; + private final String issuedTo; + private final long issueDate; + private final String type; + private final String subscriptionType; + private final String feature; + private final String signature; + private final long expiryDate; + private final int maxNodes; + private final OperationMode operationMode; + + /** + * Decouples operation mode of a license + * from the license type value + */ + public enum OperationMode { + NONE, + TRIAL, + BASIC, + GOLD, + PLATINUM; + + public static OperationMode resolve(String type) { + switch (type.toLowerCase(Locale.ROOT)) { + case "trial": + case "none": // bwc for 1.x subscription_type field + case "dev": // bwc for 1.x subscription_type field + case "development": // bwc for 1.x subscription_type field + return TRIAL; + case "basic": + return BASIC; + case "silver": + case "gold": + return GOLD; + case "platinum": + case "internal": // bwc for 1.x subscription_type field + return PLATINUM; + default: + throw new IllegalArgumentException("unknown type [" + type + "]"); + } + } + } + + private License(int version, String uid, String issuer, String issuedTo, long issueDate, String type, + String subscriptionType, String feature, String signature, long expiryDate, int maxNodes) { + this.version = version; + this.uid = uid; + this.issuer = issuer; + this.issuedTo = issuedTo; + this.issueDate = issueDate; + this.type = type; + this.subscriptionType = subscriptionType; + this.feature = feature; + this.signature = signature; + this.expiryDate = expiryDate; + this.maxNodes = maxNodes; + if (version == VERSION_START) { + // in 1.x: the acceptable values for 'subscription_type': none | dev | silver | gold | platinum + this.operationMode = OperationMode.resolve(subscriptionType); + } else { + // in 2.x: the acceptable values for 'type': trial | basic | silver | dev | gold | platinum + this.operationMode = OperationMode.resolve(type); + } + validate(); + } + + /** + * @return version of the license + */ + public int version() { + return version; + } + + /** + * @return a unique identifier for a license + */ + public String uid() { + return uid; + } + + /** + * @return type of the license [trial, subscription, internal] + */ + public String type() { + return type; + } + + /** + * @return the issueDate in milliseconds + */ + public long issueDate() { + return issueDate; + } + + /** + * @return the expiry date in milliseconds + */ + public long expiryDate() { + return expiryDate; + } + + /** + * @return the maximum number of nodes this license has been issued for + */ + public int maxNodes() { + return maxNodes; + } + + /** + * @return a string representing the entity this licenses has been issued to + */ + public String issuedTo() { + return issuedTo; + } + + /** + * @return a string representing the entity responsible for issuing this license (internal) + */ + public String issuer() { + return issuer; + } + + /** + * @return a string representing the signature of the license used for license verification + */ + public String signature() { + return signature; + } + + /** + * @return the operation mode of the license as computed from the license type + */ + public OperationMode operationMode() { + return operationMode; + } + + /** + * @return the current license's status + */ + public Status status() { + long now = System.currentTimeMillis(); + if (issueDate > now) { + return Status.INVALID; + } else if (expiryDate < now) { + return Status.EXPIRED; + } + return Status.ACTIVE; + } + + private void validate() { + if (issuer == null) { + throw new IllegalStateException("issuer can not be null"); + } else if (issuedTo == null) { + throw new IllegalStateException("issuedTo can not be null"); + } else if (issueDate == -1) { + throw new IllegalStateException("issueDate has to be set"); + } else if (type == null) { + throw new IllegalStateException("type can not be null"); + } else if (subscriptionType == null && version == VERSION_START) { + throw new IllegalStateException("subscriptionType can not be null"); + } else if (uid == null) { + throw new IllegalStateException("uid can not be null"); + } else if (feature == null && version == VERSION_START) { + throw new IllegalStateException("feature can not be null"); + } else if (maxNodes == -1) { + throw new IllegalStateException("maxNodes has to be set"); + } else if (expiryDate == -1) { + throw new IllegalStateException("expiryDate has to be set"); + } + } + + public static License readLicense(StreamInput in) throws IOException { + int version = in.readVInt(); // Version for future extensibility + if (version > VERSION_CURRENT) { + throw new ElasticsearchException("Unknown license version found, please upgrade all nodes to the latest elasticsearch-license plugin"); + } + Builder builder = builder(); + builder.version(version); + builder.uid(in.readString()); + builder.type(in.readString()); + if (version == VERSION_START) { + builder.subscriptionType(in.readString()); + } + builder.issueDate(in.readLong()); + if (version == VERSION_START) { + builder.feature(in.readString()); + } + builder.expiryDate(in.readLong()); + builder.maxNodes(in.readInt()); + builder.issuedTo(in.readString()); + builder.issuer(in.readString()); + builder.signature(in.readOptionalString()); + return builder.build(); + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(version); + out.writeString(uid); + out.writeString(type); + if (version == VERSION_START) { + out.writeString(subscriptionType); + } + out.writeLong(issueDate); + if (version == VERSION_START) { + out.writeString(feature); + } + out.writeLong(expiryDate); + out.writeInt(maxNodes); + out.writeString(issuedTo); + out.writeString(issuer); + out.writeOptionalString(signature); + } + + @Override + public String toString() { + try { + final XContentBuilder builder = XContentFactory.jsonBuilder(); + toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder.string(); + } catch (IOException e) { + return ""; + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + toInnerXContent(builder, params); + builder.endObject(); + return builder; + } + + public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException { + boolean licenseSpecMode = params.paramAsBoolean(LICENSE_SPEC_VIEW_MODE, false); + boolean restViewMode = params.paramAsBoolean(REST_VIEW_MODE, false); + boolean previouslyHumanReadable = builder.humanReadable(); + if (licenseSpecMode && restViewMode) { + throw new IllegalArgumentException("can have either " + REST_VIEW_MODE + " or " + LICENSE_SPEC_VIEW_MODE); + } else if (restViewMode) { + if (!previouslyHumanReadable) { + builder.humanReadable(true); + } + } + final int version; + if (params.param(LICENSE_VERSION_MODE) != null && restViewMode) { + version = Integer.parseInt(params.param(LICENSE_VERSION_MODE)); + } else { + version = this.version; + } + if (restViewMode) { + builder.field(XFields.STATUS, status().label()); + } + builder.field(XFields.UID, uid); + builder.field(XFields.TYPE, type); + if (version == VERSION_START) { + builder.field(XFields.SUBSCRIPTION_TYPE, subscriptionType); + } + builder.dateValueField(XFields.ISSUE_DATE_IN_MILLIS, XFields.ISSUE_DATE, issueDate); + if (version == VERSION_START) { + builder.field(XFields.FEATURE, feature); + } + builder.dateValueField(XFields.EXPIRY_DATE_IN_MILLIS, XFields.EXPIRY_DATE, expiryDate); + builder.field(XFields.MAX_NODES, maxNodes); + builder.field(XFields.ISSUED_TO, issuedTo); + builder.field(XFields.ISSUER, issuer); + if (!licenseSpecMode && !restViewMode && signature != null) { + builder.field(XFields.SIGNATURE, signature); + } + if (restViewMode) { + builder.humanReadable(previouslyHumanReadable); + } + return builder; + } + + public static License fromXContent(XContentParser parser) throws IOException { + Builder builder = new Builder(); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + String currentFieldName = parser.currentName(); + token = parser.nextToken(); + if (token.isValue()) { + if (Fields.UID.equals(currentFieldName)) { + builder.uid(parser.text()); + } else if (Fields.TYPE.equals(currentFieldName)) { + builder.type(parser.text()); + } else if (Fields.SUBSCRIPTION_TYPE.equals(currentFieldName)) { + builder.subscriptionType(parser.text()); + } else if (Fields.ISSUE_DATE.equals(currentFieldName)) { + builder.issueDate(parseDate(parser, "issue", false)); + } else if (Fields.ISSUE_DATE_IN_MILLIS.equals(currentFieldName)) { + builder.issueDate(parser.longValue()); + } else if (Fields.FEATURE.equals(currentFieldName)) { + builder.feature(parser.text()); + } else if (Fields.EXPIRY_DATE.equals(currentFieldName)) { + builder.expiryDate(parseDate(parser, "expiration", true)); + } else if (Fields.EXPIRY_DATE_IN_MILLIS.equals(currentFieldName)) { + builder.expiryDate(parser.longValue()); + } else if (Fields.MAX_NODES.equals(currentFieldName)) { + builder.maxNodes(parser.intValue()); + } else if (Fields.ISSUED_TO.equals(currentFieldName)) { + builder.issuedTo(parser.text()); + } else if (Fields.ISSUER.equals(currentFieldName)) { + builder.issuer(parser.text()); + } else if (Fields.SIGNATURE.equals(currentFieldName)) { + builder.signature(parser.text()); + } else if (Fields.VERSION.equals(currentFieldName)) { + builder.version(parser.intValue()); + } + // Ignore unknown elements - might be new version of license + } else if (token == XContentParser.Token.START_ARRAY) { + // It was probably created by newer version - ignoring + parser.skipChildren(); + } else if (token == XContentParser.Token.START_OBJECT) { + // It was probably created by newer version - ignoring + parser.skipChildren(); + } + } + } + // not a license spec + if (builder.signature != null) { + byte[] signatureBytes = Base64.decode(builder.signature); + ByteBuffer byteBuffer = ByteBuffer.wrap(signatureBytes); + int version = byteBuffer.getInt(); + // we take the absolute version, because negative versions + // mean that the license was generated by the cluster (see TrialLicense) + // and positive version means that the license was signed + if (version < 0) { + version *= -1; + } + if (version == 0) { + throw new ElasticsearchException("malformed signature for license [" + builder.uid + "]"); + } else if (version > VERSION_CURRENT) { + throw new ElasticsearchException("Unknown license version found, please upgrade all nodes to the latest elasticsearch-license plugin"); + } + // signature version is the source of truth + builder.version(version); + } + return builder.build(); + } + + /** + * Returns true if the license was auto-generated (by license plugin), + * false otherwise + */ + public static boolean isAutoGeneratedLicense(String signature) { + try { + byte[] signatureBytes = Base64.decode(signature); + ByteBuffer byteBuffer = ByteBuffer.wrap(signatureBytes); + return byteBuffer.getInt() < 0; + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + + public static License fromSource(String content) throws IOException { + return fromSource(content.getBytes(StandardCharsets.UTF_8)); + } + + public static License fromSource(byte[] bytes) throws IOException { + final XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes); + License license = null; + if (parser.nextToken() == XContentParser.Token.START_OBJECT) { + if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { + String currentFieldName = parser.currentName(); + if (Fields.LICENSES.equals(currentFieldName)) { + final List pre20Licenses = new ArrayList<>(); + if (parser.nextToken() == XContentParser.Token.START_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + pre20Licenses.add(License.fromXContent(parser)); + } + // take the latest issued unexpired license + CollectionUtil.timSort(pre20Licenses, LATEST_ISSUE_DATE_FIRST); + long now = System.currentTimeMillis(); + for (License oldLicense : pre20Licenses) { + if (oldLicense.expiryDate() > now) { + license = oldLicense; + break; + } + } + if (license == null && !pre20Licenses.isEmpty()) { + license = pre20Licenses.get(0); + } + } else { + throw new ElasticsearchParseException("failed to parse licenses expected an array of licenses"); + } + } else if (Fields.LICENSE.equals(currentFieldName)) { + license = License.fromXContent(parser); + } + // Ignore all other fields - might be created with new version + } else { + throw new ElasticsearchParseException("failed to parse licenses expected field"); + } + } else { + throw new ElasticsearchParseException("failed to parse licenses expected start object"); + } + return license; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + License license = (License) o; + + if (issueDate != license.issueDate) return false; + if (expiryDate != license.expiryDate) return false; + if (maxNodes != license.maxNodes) return false; + if (version != license.version) return false; + if (uid != null ? !uid.equals(license.uid) : license.uid != null) return false; + if (issuer != null ? !issuer.equals(license.issuer) : license.issuer != null) return false; + if (issuedTo != null ? !issuedTo.equals(license.issuedTo) : license.issuedTo != null) return false; + if (type != null ? !type.equals(license.type) : license.type != null) return false; + if (subscriptionType != null ? !subscriptionType.equals(license.subscriptionType) : license.subscriptionType != null) + return false; + if (feature != null ? !feature.equals(license.feature) : license.feature != null) return false; + return !(signature != null ? !signature.equals(license.signature) : license.signature != null); + + } + + @Override + public int hashCode() { + int result = uid != null ? uid.hashCode() : 0; + result = 31 * result + (issuer != null ? issuer.hashCode() : 0); + result = 31 * result + (issuedTo != null ? issuedTo.hashCode() : 0); + result = 31 * result + (int) (issueDate ^ (issueDate >>> 32)); + result = 31 * result + (type != null ? type.hashCode() : 0); + result = 31 * result + (subscriptionType != null ? subscriptionType.hashCode() : 0); + result = 31 * result + (feature != null ? feature.hashCode() : 0); + result = 31 * result + (signature != null ? signature.hashCode() : 0); + result = 31 * result + (int) (expiryDate ^ (expiryDate >>> 32)); + result = 31 * result + maxNodes; + result = 31 * result + version; + return result; + } + + final static class Fields { + static final String STATUS = "status"; + static final String UID = "uid"; + static final String TYPE = "type"; + static final String SUBSCRIPTION_TYPE = "subscription_type"; + static final String ISSUE_DATE_IN_MILLIS = "issue_date_in_millis"; + static final String ISSUE_DATE = "issue_date"; + static final String FEATURE = "feature"; + static final String EXPIRY_DATE_IN_MILLIS = "expiry_date_in_millis"; + static final String EXPIRY_DATE = "expiry_date"; + static final String MAX_NODES = "max_nodes"; + static final String ISSUED_TO = "issued_to"; + static final String ISSUER = "issuer"; + static final String VERSION = "version"; + static final String SIGNATURE = "signature"; + + static final String LICENSES = "licenses"; + static final String LICENSE = "license"; + + } + + public interface XFields { + XContentBuilderString STATUS = new XContentBuilderString(Fields.STATUS); + XContentBuilderString UID = new XContentBuilderString(Fields.UID); + XContentBuilderString TYPE = new XContentBuilderString(Fields.TYPE); + XContentBuilderString SUBSCRIPTION_TYPE = new XContentBuilderString(Fields.SUBSCRIPTION_TYPE); + XContentBuilderString ISSUE_DATE_IN_MILLIS = new XContentBuilderString(Fields.ISSUE_DATE_IN_MILLIS); + XContentBuilderString ISSUE_DATE = new XContentBuilderString(Fields.ISSUE_DATE); + XContentBuilderString FEATURE = new XContentBuilderString(Fields.FEATURE); + XContentBuilderString EXPIRY_DATE_IN_MILLIS = new XContentBuilderString(Fields.EXPIRY_DATE_IN_MILLIS); + XContentBuilderString EXPIRY_DATE = new XContentBuilderString(Fields.EXPIRY_DATE); + XContentBuilderString MAX_NODES = new XContentBuilderString(Fields.MAX_NODES); + XContentBuilderString ISSUED_TO = new XContentBuilderString(Fields.ISSUED_TO); + XContentBuilderString ISSUER = new XContentBuilderString(Fields.ISSUER); + XContentBuilderString SIGNATURE = new XContentBuilderString(Fields.SIGNATURE); + } + + private static long parseDate(XContentParser parser, String description, boolean endOfTheDay) throws IOException { + if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) { + return parser.longValue(); + } else { + try { + if (endOfTheDay) { + return DateUtils.endOfTheDay(parser.text()); + } else { + return DateUtils.beginningOfTheDay(parser.text()); + } + } catch (IllegalArgumentException ex) { + throw new ElasticsearchParseException("invalid " + description + " date format " + parser.text()); + } + } + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private int version = License.VERSION_CURRENT; + private String uid; + private String issuer; + private String issuedTo; + private long issueDate = -1; + private String type; + private String subscriptionType; + private String feature; + private String signature; + private long expiryDate = -1; + private int maxNodes = -1; + + public Builder uid(String uid) { + this.uid = uid; + return this; + } + + public Builder version(int version) { + this.version = version; + return this; + } + + public Builder issuer(String issuer) { + this.issuer = issuer; + return this; + } + + public Builder issuedTo(String issuedTo) { + this.issuedTo = issuedTo; + return this; + } + + public Builder issueDate(long issueDate) { + this.issueDate = issueDate; + return this; + } + + public Builder type(String type) { + this.type = type; + return this; + } + + public Builder subscriptionType(String subscriptionType) { + this.subscriptionType = subscriptionType; + return this; + } + + public Builder feature(String feature) { + this.feature = feature; + return this; + } + + public Builder expiryDate(long expiryDate) { + this.expiryDate = expiryDate; + return this; + } + + public Builder maxNodes(int maxNodes) { + this.maxNodes = maxNodes; + return this; + } + + public Builder signature(String signature) { + if (signature != null) { + this.signature = signature; + } + return this; + } + + public Builder fromLicenseSpec(License license, String signature) { + return uid(license.uid()) + .version(license.version()) + .issuedTo(license.issuedTo()) + .issueDate(license.issueDate()) + .type(license.type()) + .subscriptionType(license.subscriptionType) + .feature(license.feature) + .maxNodes(license.maxNodes()) + .expiryDate(license.expiryDate()) + .issuer(license.issuer()) + .signature(signature); + } + + /** + * Returns a builder that converts pre 2.0 licenses + * to the new license format + */ + public Builder fromPre20LicenseSpec(License pre20License) { + return uid(pre20License.uid()) + .issuedTo(pre20License.issuedTo()) + .issueDate(pre20License.issueDate()) + .maxNodes(pre20License.maxNodes()) + .expiryDate(pre20License.expiryDate()); + } + + public License build() { + return new License(version, uid, issuer, issuedTo, issueDate, type, + subscriptionType, feature, signature, expiryDate, maxNodes); + } + + public Builder validate() { + if (issuer == null) { + throw new IllegalStateException("issuer can not be null"); + } else if (issuedTo == null) { + throw new IllegalStateException("issuedTo can not be null"); + } else if (issueDate == -1) { + throw new IllegalStateException("issueDate has to be set"); + } else if (type == null) { + throw new IllegalStateException("type can not be null"); + } else if (uid == null) { + throw new IllegalStateException("uid can not be null"); + } else if (signature == null) { + throw new IllegalStateException("signature can not be null"); + } else if (maxNodes == -1) { + throw new IllegalStateException("maxNodes has to be set"); + } else if (expiryDate == -1) { + throw new IllegalStateException("expiryDate has to be set"); + } + return this; + } + } + + public enum Status { + ACTIVE("active"), + INVALID("invalid"), + EXPIRED("expired"); + + private final String label; + + Status(String label) { + this.label = label; + } + + public String label() { + return label; + } + } +} diff --git a/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/LicenseVerifier.java b/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/LicenseVerifier.java new file mode 100644 index 00000000000..51ace787491 --- /dev/null +++ b/elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/LicenseVerifier.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.core; + +import org.elasticsearch.common.Base64; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.Signature; +import java.security.SignatureException; +import java.util.Arrays; +import java.util.Collections; + +/** + * Responsible for verifying signed licenses + */ +public class LicenseVerifier { + + /** + * verifies the license content with the signature using the packaged + * public key + * @param license to verify + * @return true if valid, false otherwise + */ + public static boolean verifyLicense(final License license, byte[] encryptedPublicKeyData) { + byte[] signedContent = null; + byte[] signatureHash = null; + try { + byte[] signatureBytes = Base64.decode(license.signature()); + ByteBuffer byteBuffer = ByteBuffer.wrap(signatureBytes); + int version = byteBuffer.getInt(); + int magicLen = byteBuffer.getInt(); + byte[] magic = new byte[magicLen]; + byteBuffer.get(magic); + int hashLen = byteBuffer.getInt(); + signatureHash = new byte[hashLen]; + byteBuffer.get(signatureHash); + int signedContentLen = byteBuffer.getInt(); + signedContent = new byte[signedContentLen]; + byteBuffer.get(signedContent); + XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + license.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true"))); + Signature rsa = Signature.getInstance("SHA512withRSA"); + rsa.initVerify(CryptUtils.readEncryptedPublicKey(encryptedPublicKeyData)); + rsa.update(contentBuilder.bytes().toBytes()); + return rsa.verify(signedContent) + && Arrays.equals(Base64.encodeBytesToBytes(encryptedPublicKeyData), signatureHash); + } catch (IOException | NoSuchAlgorithmException | SignatureException | InvalidKeyException e) { + throw new IllegalStateException(e); + } finally { + Arrays.fill(encryptedPublicKeyData, (byte) 0); + if (signedContent != null) { + Arrays.fill(signedContent, (byte) 0); + } + if (signatureHash != null) { + Arrays.fill(signatureHash, (byte) 0); + } + } + } +} diff --git a/elasticsearch/license/base/src/test/java/org/elasticsearch/license/core/LicenseSerializationTests.java b/elasticsearch/license/base/src/test/java/org/elasticsearch/license/core/LicenseSerializationTests.java new file mode 100644 index 00000000000..b1947ee971b --- /dev/null +++ b/elasticsearch/license/base/src/test/java/org/elasticsearch/license/core/LicenseSerializationTests.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.core; + +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Map; + +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.IsNull.notNullValue; +import static org.hamcrest.core.IsNull.nullValue; + +public class LicenseSerializationTests extends ESTestCase { + public void testSimpleIssueExpiryDate() throws Exception { + long now = System.currentTimeMillis(); + String issueDate = TestUtils.dateMathString("now", now); + String expiryDate = TestUtils.dateMathString("now+10d/d", now); + String licenseSpecs = TestUtils.generateLicenseSpecString(new TestUtils.LicenseSpec(issueDate, expiryDate)); + License generatedLicense = License.fromSource(licenseSpecs.getBytes(StandardCharsets.UTF_8)); + assertThat(generatedLicense.issueDate(), equalTo(DateUtils.beginningOfTheDay(issueDate))); + assertThat(generatedLicense.expiryDate(), equalTo(DateUtils.endOfTheDay(expiryDate))); + } + + public void testLicensesFields() throws Exception { + TestUtils.LicenseSpec randomLicenseSpec = TestUtils.generateRandomLicenseSpec(License.VERSION_START);//randomIntBetween(License.VERSION_START, License.VERSION_CURRENT)); + String licenseSpecsSource = TestUtils.generateLicenseSpecString(randomLicenseSpec); + final License fromSource = License.fromSource(licenseSpecsSource.getBytes(StandardCharsets.UTF_8)); + TestUtils.assertLicenseSpec(randomLicenseSpec, fromSource); + } + + public void testLicenseRestView() throws Exception { + long now = System.currentTimeMillis(); + String expiredLicenseExpiryDate = TestUtils.dateMathString("now-1d/d", now); + String validLicenseIssueDate = TestUtils.dateMathString("now-10d/d", now); + String invalidLicenseIssueDate = TestUtils.dateMathString("now+1d/d", now); + String validLicenseExpiryDate = TestUtils.dateMathString("now+2d/d", now); + + License license = TestUtils.generateLicenses(new TestUtils.LicenseSpec(validLicenseIssueDate, expiredLicenseExpiryDate)); + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + license.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap(License.REST_VIEW_MODE, "true"))); + builder.flush(); + Map map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2(); + + // should have an extra status field, human readable issue_data and expiry_date + assertThat(map.get("status"), notNullValue()); + assertThat(map.get("issue_date"), notNullValue()); + assertThat(map.get("expiry_date"), notNullValue()); + assertThat(map.get("status"), equalTo("expired")); + builder = XContentFactory.contentBuilder(XContentType.JSON); + license.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.flush(); + map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2(); + assertThat(map.get("status"), nullValue()); + + license = TestUtils.generateLicenses(new TestUtils.LicenseSpec(validLicenseIssueDate, validLicenseExpiryDate)); + builder = XContentFactory.contentBuilder(XContentType.JSON); + license.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap(License.REST_VIEW_MODE, "true"))); + builder.flush(); + map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2(); + + // should have an extra status field, human readable issue_data and expiry_date + assertThat(map.get("status"), notNullValue()); + assertThat(map.get("issue_date"), notNullValue()); + assertThat(map.get("expiry_date"), notNullValue()); + assertThat(map.get("status"), equalTo("active")); + builder = XContentFactory.contentBuilder(XContentType.JSON); + license.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.flush(); + map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2(); + assertThat(map.get("status"), nullValue()); + + license = TestUtils.generateLicenses(new TestUtils.LicenseSpec(invalidLicenseIssueDate, validLicenseExpiryDate)); + builder = XContentFactory.contentBuilder(XContentType.JSON); + license.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap(License.REST_VIEW_MODE, "true"))); + builder.flush(); + map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2(); + + // should have an extra status field, human readable issue_data and expiry_date + assertThat(map.get("status"), notNullValue()); + assertThat(map.get("issue_date"), notNullValue()); + assertThat(map.get("expiry_date"), notNullValue()); + assertThat(map.get("status"), equalTo("invalid")); + builder = XContentFactory.contentBuilder(XContentType.JSON); + license.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.flush(); + map = XContentHelper.convertToMap(builder.bytesStream().bytes(), false).v2(); + assertThat(map.get("status"), nullValue()); + } +} diff --git a/elasticsearch/license/base/src/test/java/org/elasticsearch/license/core/TestUtils.java b/elasticsearch/license/base/src/test/java/org/elasticsearch/license/core/TestUtils.java new file mode 100644 index 00000000000..7f3dd01be26 --- /dev/null +++ b/elasticsearch/license/base/src/test/java/org/elasticsearch/license/core/TestUtils.java @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.core; + +import org.elasticsearch.common.joda.DateMathParser; +import org.elasticsearch.common.joda.FormatDateTimeFormatter; +import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.hamcrest.MatcherAssert; +import org.joda.time.format.DateTimeFormatter; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.Callable; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.*; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.hamcrest.core.IsEqual.equalTo; + +public class TestUtils { + + private final static FormatDateTimeFormatter formatDateTimeFormatter = Joda.forPattern("yyyy-MM-dd"); + private final static DateMathParser dateMathParser = new DateMathParser(formatDateTimeFormatter); + private final static DateTimeFormatter dateTimeFormatter = formatDateTimeFormatter.printer(); + + public static String dateMathString(String time, final long now) { + return dateTimeFormatter.print(dateMathParser.parse(time, new Callable() { + @Override + public Long call() throws Exception { + return now; + } + })); + } + + public static long dateMath(String time, final long now) { + return dateMathParser.parse(time, new Callable() { + @Override + public Long call() throws Exception { + return now; + } + }); + } + + public static LicenseSpec generateRandomLicenseSpec(int version) { + boolean datesInMillis = randomBoolean(); + long now = System.currentTimeMillis(); + String uid = UUID.randomUUID().toString(); + String feature = "feature__" + randomInt(); + String issuer = "issuer__" + randomInt(); + String issuedTo = "issuedTo__" + randomInt(); + final String type; + final String subscriptionType; + if (version < License.VERSION_NO_FEATURE_TYPE) { + subscriptionType = randomFrom("gold", "silver", "platinum"); + type = "subscription";//randomFrom("subscription", "internal", "development"); + } else { + subscriptionType = null; + type = randomFrom("basic", "dev", "gold", "silver", "platinum"); + } + int maxNodes = randomIntBetween(5, 100); + if (datesInMillis) { + long issueDateInMillis = dateMath("now", now); + long expiryDateInMillis = dateMath("now+10d/d", now); + return new LicenseSpec(version, uid, feature, issueDateInMillis, expiryDateInMillis, type, subscriptionType, issuedTo, issuer, maxNodes); + } else { + String issueDate = dateMathString("now", now); + String expiryDate = dateMathString("now+10d/d", now); + return new LicenseSpec(version, uid, feature, issueDate, expiryDate, type, subscriptionType, issuedTo, issuer, maxNodes); + } + } + + public static String generateLicenseSpecString(LicenseSpec licenseSpec) throws IOException { + XContentBuilder licenses = jsonBuilder(); + licenses.startObject(); + licenses.startArray("licenses"); + licenses.startObject() + .field("uid", licenseSpec.uid) + .field("type", licenseSpec.type) + .field("subscription_type", licenseSpec.subscriptionType) + .field("issued_to", licenseSpec.issuedTo) + .field("issuer", licenseSpec.issuer) + .field("feature", licenseSpec.feature) + .field("max_nodes", licenseSpec.maxNodes); + + if (licenseSpec.issueDate != null) { + licenses.field("issue_date", licenseSpec.issueDate); + } else { + licenses.field("issue_date_in_millis", licenseSpec.issueDateInMillis); + } + if (licenseSpec.expiryDate != null) { + licenses.field("expiry_date", licenseSpec.expiryDate); + } else { + licenses.field("expiry_date_in_millis", licenseSpec.expiryDateInMillis); + } + licenses.field("version", licenseSpec.version); + licenses.endObject(); + licenses.endArray(); + licenses.endObject(); + return licenses.string(); + } + + public static License generateLicenses(LicenseSpec spec) { + License.Builder builder = License.builder() + .uid(spec.uid) + .feature(spec.feature) + .type(spec.type) + .subscriptionType(spec.subscriptionType) + .issuedTo(spec.issuedTo) + .issuer(spec.issuer) + .maxNodes(spec.maxNodes); + + if (spec.expiryDate != null) { + builder.expiryDate(DateUtils.endOfTheDay(spec.expiryDate)); + } else { + builder.expiryDate(spec.expiryDateInMillis); + } + if (spec.issueDate != null) { + builder.issueDate(DateUtils.beginningOfTheDay(spec.issueDate)); + } else { + builder.issueDate(spec.issueDateInMillis); + } + return builder.build(); + } + + public static void assertLicenseSpec(LicenseSpec spec, License license) { + MatcherAssert.assertThat(license.uid(), equalTo(spec.uid)); + MatcherAssert.assertThat(license.issuedTo(), equalTo(spec.issuedTo)); + MatcherAssert.assertThat(license.issuer(), equalTo(spec.issuer)); + MatcherAssert.assertThat(license.type(), equalTo(spec.type)); + MatcherAssert.assertThat(license.maxNodes(), equalTo(spec.maxNodes)); + if (spec.issueDate != null) { + MatcherAssert.assertThat(license.issueDate(), equalTo(DateUtils.beginningOfTheDay(spec.issueDate))); + } else { + MatcherAssert.assertThat(license.issueDate(), equalTo(spec.issueDateInMillis)); + } + if (spec.expiryDate != null) { + MatcherAssert.assertThat(license.expiryDate(), equalTo(DateUtils.endOfTheDay(spec.expiryDate))); + } else { + MatcherAssert.assertThat(license.expiryDate(), equalTo(spec.expiryDateInMillis)); + } + } + + public static class LicenseSpec { + public final int version; + public final String feature; + public final String issueDate; + public final long issueDateInMillis; + public final String expiryDate; + public final long expiryDateInMillis; + public final String uid; + public final String type; + public final String subscriptionType; + public final String issuedTo; + public final String issuer; + public final int maxNodes; + + public LicenseSpec(String issueDate, String expiryDate) { + this(License.VERSION_CURRENT, UUID.randomUUID().toString(), "feature", issueDate, expiryDate, "trial", "none", "customer", "elasticsearch", 5); + } + + public LicenseSpec(int version, String uid, String feature, long issueDateInMillis, long expiryDateInMillis, String type, + String subscriptionType, String issuedTo, String issuer, int maxNodes) { + this.version = version; + this.feature = feature; + this.issueDateInMillis = issueDateInMillis; + this.issueDate = null; + this.expiryDateInMillis = expiryDateInMillis; + this.expiryDate = null; + this.uid = uid; + this.type = type; + this.subscriptionType = subscriptionType; + this.issuedTo = issuedTo; + this.issuer = issuer; + this.maxNodes = maxNodes; + } + + public LicenseSpec(int version, String uid, String feature, String issueDate, String expiryDate, String type, + String subscriptionType, String issuedTo, String issuer, int maxNodes) { + this.version = version; + this.feature = feature; + this.issueDate = issueDate; + this.issueDateInMillis = -1; + this.expiryDate = expiryDate; + this.expiryDateInMillis = -1; + this.uid = uid; + this.type = type; + this.subscriptionType = subscriptionType; + this.issuedTo = issuedTo; + this.issuer = issuer; + this.maxNodes = maxNodes; + } + } +} diff --git a/elasticsearch/license/base/src/test/resources/log4j.properties b/elasticsearch/license/base/src/test/resources/log4j.properties new file mode 100644 index 00000000000..76defc8660c --- /dev/null +++ b/elasticsearch/license/base/src/test/resources/log4j.properties @@ -0,0 +1,11 @@ +es.logger.level=INFO +log4j.rootLogger=${es.logger.level}, out + +log4j.logger.org.apache.http=INFO, out +log4j.additivity.org.apache.http=false + +log4j.logger.org.elasticsearch.license=TRACE + +log4j.appender.out=org.apache.log4j.ConsoleAppender +log4j.appender.out.layout=org.apache.log4j.PatternLayout +log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n diff --git a/elasticsearch/license/base/src/test/resources/private.key b/elasticsearch/license/base/src/test/resources/private.key new file mode 100644 index 0000000000000000000000000000000000000000..1f545803d875598d976b206250d5e95667c794ef GIT binary patch literal 1232 zcmV;>1TXv1%Bc_wbLO>E3}&#D<@-rDJ$(p047kR zkI3mLE>J{X5~_|GE5>Bxd81`fV@~0j8z81fwu78=uPV1?-b_fL$snD`NE}Zsv(l}S z+Dr52K=W~W#&Y(zE)IK9$=-71EtPe#X0XhttX&@1ia@?lvh&>PD=A#>rfdl2Ru)*H zKaKRTp&qI~gmO-)fq;gzrbngk1r$cUKd~25@6;rjPJf6eLtCHTaDU8Rbt#KGKlZ%g zPd`h6iZ4kWgcXo9P=@b~YoN9WQmtytp^dd%)MXVsW%n~=T9#TzSWvr*YCR25X&mo6 zJoJz>&ju@jBz@-J;se7XMiU5&x@g8>FTI*r^O6Xv2(VF_R@Vk3wQBmIE>&XqH^t=M z5_c~f_`fm)R~9ZqEv|Ma3$nQ}LvHLD`h~L2VKXBZwZnP`c6%Qwnwq8Z0!hd2+^*Gk z6R7QdYCwK?kavUU0Vd*Mq5K{M!Vk zzSLrHwA_oZsoml1+m@@Wyh8$ZA^E2_FpxTTRz7~A$x?V+XFW0!aa4^L&A*^Evlz{gN8ae8h9=J=b zC8F19FN`xNpxA!48b`dGRK3GbQMOho1>byaB*psVh{^^yqVL4J0~$j7viqxma>pYOhkH~PwpJT1j6^yHDMWGO{vw8PyI zIrYD&$F3e@0!346r3BY7JkLDD+7E5EE6oiV(gQ62K@eQRE&#pRq81Tut!V+1TCM~srn6-g`!g_`uozhhJEnN6u9$5|& zLC>$@xw*^CD*fb$ngUWU)yZ%TqX*wM!HU2(O2RRFnw&?^e*xOZWZ?n`No9lb?BLpV=@K}hkB uJ-}qCO)pTE(AWQ24*9x_mu179!iEY^J(*;5;=+=*kZC+fDbTdAaQBAù<}[Á2ÖÓàZÖ|5‹¯ÊÊ7%ØêD +Yå‘xn:¼lúLÈæHò¢«Ë2˜ŸHvEEWÇ\¦H:“6Žh9 [!š…Ûæ©Š¤+;Ö.w7Cì©_|Þ ÓªÏÁ*ñ§D`ƒÚ?‚ùxU/3>x­UÓ“+ è \ No newline at end of file diff --git a/elasticsearch/license/build.gradle b/elasticsearch/license/build.gradle new file mode 100644 index 00000000000..b4e221e7884 --- /dev/null +++ b/elasticsearch/license/build.gradle @@ -0,0 +1,7 @@ +subprojects { + project.afterEvaluate { + project.forbiddenPatterns { + exclude '**/*.key' + } + } +} diff --git a/elasticsearch/license/found-plugin/.gitignore b/elasticsearch/license/found-plugin/.gitignore new file mode 100644 index 00000000000..ab956abf6f9 --- /dev/null +++ b/elasticsearch/license/found-plugin/.gitignore @@ -0,0 +1 @@ +/eclipse-build/ diff --git a/elasticsearch/license/found-plugin/build.gradle b/elasticsearch/license/found-plugin/build.gradle new file mode 100644 index 00000000000..577ed689782 --- /dev/null +++ b/elasticsearch/license/found-plugin/build.gradle @@ -0,0 +1,19 @@ +apply plugin: 'elasticsearch.esplugin' +esplugin { + name 'found-plugin' + description 'Internal Elasticsearch Licensing Plugin for Found' + classname 'org.elasticsearch.license.plugin.LicensePlugin' +} + +dependencies { + compile project(':x-plugins:elasticsearch:license:plugin-api') + testCompile project(':x-plugins:elasticsearch:license:licensor') +} + +// no tests +test.enabled = false +integTest.enabled = false + +dependencyLicenses.enabled = false + +compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked" diff --git a/elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/LicenseModule.java b/elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/LicenseModule.java new file mode 100644 index 00000000000..626049a8e53 --- /dev/null +++ b/elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/LicenseModule.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.license.core.LicenseVerifier; +import org.elasticsearch.license.plugin.core.FoundLicensesService; +import org.elasticsearch.license.plugin.core.LicenseeRegistry; +import org.elasticsearch.license.plugin.core.LicensesManagerService; + +public class LicenseModule extends AbstractModule { + + @Override + protected void configure() { + bind(LicenseVerifier.class).asEagerSingleton(); + bind(LicenseeRegistry.class).to(FoundLicensesService.class); + bind(LicensesManagerService.class).to(FoundLicensesService.class); + } + +} \ No newline at end of file diff --git a/elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/LicensePlugin.java b/elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/LicensePlugin.java new file mode 100644 index 00000000000..82367bfad0a --- /dev/null +++ b/elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/LicensePlugin.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.common.component.LifecycleComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.core.FoundLicensesService; +import org.elasticsearch.plugins.Plugin; + +import java.util.ArrayList; +import java.util.Collection; + +public class LicensePlugin extends Plugin { + + public static final String NAME = "license"; + + @Inject + public LicensePlugin(Settings settings) { + } + + @Override + public String name() { + return NAME; + } + + @Override + public String description() { + return "Internal Elasticsearch Licensing Plugin"; + } + + @Override + public Collection> nodeServices() { + Collection> services = new ArrayList<>(); + services.add(FoundLicensesService.class); + return services; + } + + @Override + public Collection nodeModules() { + Collection modules = new ArrayList(); + modules.add(new LicenseModule()); + return modules; + } +} diff --git a/elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/core/FoundLicensesService.java b/elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/core/FoundLicensesService.java new file mode 100644 index 00000000000..af1273697b3 --- /dev/null +++ b/elasticsearch/license/found-plugin/src/main/java/org/elasticsearch/license/plugin/core/FoundLicensesService.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.Singleton; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.core.License; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.UUID; + +@Singleton +public class FoundLicensesService extends AbstractLifecycleComponent implements LicenseeRegistry, LicensesManagerService { + + @Inject + public FoundLicensesService(Settings settings) { + super(settings); + } + + /** + * {@inheritDoc} + */ + @Override + public void register(Licensee licensee) { + licensee.onChange(new Licensee.Status(License.OperationMode.PLATINUM, LicenseState.ENABLED)); + } + + @Override + protected void doStart() throws ElasticsearchException { + } + + @Override + protected void doStop() throws ElasticsearchException { + } + + @Override + protected void doClose() throws ElasticsearchException { + } + + @Override + public List licenseesWithState(LicenseState state) { + return Collections.emptyList(); + } + + @Override + public License getLicense() { + return null; + } +} diff --git a/elasticsearch/license/licensor/bin/key-pair-generator b/elasticsearch/license/licensor/bin/key-pair-generator new file mode 100755 index 00000000000..7c6308673fa --- /dev/null +++ b/elasticsearch/license/licensor/bin/key-pair-generator @@ -0,0 +1,58 @@ +#!/bin/sh + +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +CDPATH="" +SCRIPT="$0" + +# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path. +while [ -h "$SCRIPT" ] ; do + ls=`ls -ld "$SCRIPT"` + # Drop everything prior to -> + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + SCRIPT="$link" + else + SCRIPT=`dirname "$SCRIPT"`/"$link" + fi +done + +# determine license home +LICENSE_HOME=`dirname "$SCRIPT"`/.. + +# make LICENSE_HOME absolute +LICENSE_HOME=`cd "$LICENSE_HOME"; pwd` + +# setup classpath +LICENSE_CLASSPATH=$LICENSE_CLASSPATH:$LICENSE_HOME/lib/${project.artifactId}-${project.version}-exec.jar:$LICENSE_HOME/lib/* + +if [ -x "$JAVA_HOME/bin/java" ]; then + JAVA=$JAVA_HOME/bin/java +else + JAVA=`which java` +fi + + +# Parse any long getopt options and put them into properties before calling getopt below +# Be dash compatible to make sure running under ubuntu works +ARGCOUNT=$# +COUNT=0 +while [ $COUNT -lt $ARGCOUNT ] +do + case $1 in + -D*=*) + properties="$properties $1" + shift 1; COUNT=$(($COUNT+1)) + ;; + -D*) + properties="$properties $1=$2" + shift ; shift; COUNT=$(($COUNT+2)) + ;; + *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1)) + esac +done + +exec "$JAVA" $JAVA_OPTS -Xmx64m -Xms16m $properties -cp "$LICENSE_CLASSPATH" -Des.path.home="`pwd`" org.elasticsearch.license.licensor.tools.KeyPairGeneratorTool "$@" + diff --git a/elasticsearch/license/licensor/bin/license-generator b/elasticsearch/license/licensor/bin/license-generator new file mode 100755 index 00000000000..d51ac254ee1 --- /dev/null +++ b/elasticsearch/license/licensor/bin/license-generator @@ -0,0 +1,56 @@ +#!/bin/sh + +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +CDPATH="" +SCRIPT="$0" + +# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path. +while [ -h "$SCRIPT" ] ; do + ls=`ls -ld "$SCRIPT"` + # Drop everything prior to -> + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + SCRIPT="$link" + else + SCRIPT=`dirname "$SCRIPT"`/"$link" + fi +done + +# determine license home +LICENSE_HOME=`dirname "$SCRIPT"`/.. + +# make LICENSE_HOME absolute +LICENSE_HOME=`cd "$LICENSE_HOME"; pwd` + +# setup classpath +LICENSE_CLASSPATH=$LICENSE_CLASSPATH:$LICENSE_HOME/lib/${project.artifactId}-${project.version}-exec.jar:$LICENSE_HOME/lib/* + +if [ -x "$JAVA_HOME/bin/java" ]; then + JAVA=$JAVA_HOME/bin/java +else + JAVA=`which java` +fi + +# Parse any long getopt options and put them into properties before calling getopt below +# Be dash compatible to make sure running under ubuntu works +ARGCOUNT=$# +COUNT=0 +while [ $COUNT -lt $ARGCOUNT ] +do + case $1 in + -D*=*) + properties="$properties $1" + shift 1; COUNT=$(($COUNT+1)) + ;; + -D*) + properties="$properties $1=$2" + shift ; shift; COUNT=$(($COUNT+2)) + ;; + *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1)) + esac +done + +exec "$JAVA" $JAVA_OPTS -Xmx64m -Xms16m $properties -cp "$LICENSE_CLASSPATH" -Des.path.home="`pwd`" org.elasticsearch.license.licensor.tools.LicenseGeneratorTool "$@" diff --git a/elasticsearch/license/licensor/bin/verify-license b/elasticsearch/license/licensor/bin/verify-license new file mode 100755 index 00000000000..49762803123 --- /dev/null +++ b/elasticsearch/license/licensor/bin/verify-license @@ -0,0 +1,57 @@ +#!/bin/sh + +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +CDPATH="" +SCRIPT="$0" + +# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path. +while [ -h "$SCRIPT" ] ; do + ls=`ls -ld "$SCRIPT"` + # Drop everything prior to -> + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + SCRIPT="$link" + else + SCRIPT=`dirname "$SCRIPT"`/"$link" + fi +done + +# determine license home +LICENSE_HOME=`dirname "$SCRIPT"`/.. + +# make LICENSE_HOME absolute +LICENSE_HOME=`cd "$LICENSE_HOME"; pwd` + +# setup classpath +LICENSE_CLASSPATH=$LICENSE_CLASSPATH:$LICENSE_HOME/lib/${project.artifactId}-${project.version}-exec.jar:$LICENSE_HOME/lib/* + +if [ -x "$JAVA_HOME/bin/java" ]; then + JAVA=$JAVA_HOME/bin/java +else + JAVA=`which java` +fi + +# Parse any long getopt options and put them into properties before calling getopt below +# Be dash compatible to make sure running under ubuntu works +ARGCOUNT=$# +COUNT=0 +while [ $COUNT -lt $ARGCOUNT ] +do + case $1 in + -D*=*) + properties="$properties $1" + shift 1; COUNT=$(($COUNT+1)) + ;; + -D*) + properties="$properties $1=$2" + shift ; shift; COUNT=$(($COUNT+2)) + ;; + *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1)) + esac +done + +exec "$JAVA" $JAVA_OPTS -Xmx64m -Xms16m $properties -cp "$LICENSE_CLASSPATH" -Des.path.home="`pwd`" org.elasticsearch.license.licensor.tools.LicenseVerificationTool "$@" + diff --git a/elasticsearch/license/licensor/build.gradle b/elasticsearch/license/licensor/build.gradle new file mode 100644 index 00000000000..e14cd0045d8 --- /dev/null +++ b/elasticsearch/license/licensor/build.gradle @@ -0,0 +1,9 @@ +apply plugin: 'elasticsearch.build' + +dependencies { + compile project(':x-plugins:elasticsearch:license:base') + compile "org.elasticsearch:elasticsearch:${version}" + testCompile "org.elasticsearch:test-framework:${version}" +} + +dependencyLicenses.enabled = false diff --git a/elasticsearch/license/licensor/dev-tools/integration-tests.xml b/elasticsearch/license/licensor/dev-tools/integration-tests.xml new file mode 100644 index 00000000000..153253fd27d --- /dev/null +++ b/elasticsearch/license/licensor/dev-tools/integration-tests.xml @@ -0,0 +1,122 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/elasticsearch/license/licensor/sample/license_spec.json b/elasticsearch/license/licensor/sample/license_spec.json new file mode 100644 index 00000000000..cff074c0e7d --- /dev/null +++ b/elasticsearch/license/licensor/sample/license_spec.json @@ -0,0 +1 @@ +{"licenses":[{"uid": "893361dc-9749-4997-93cb-802e3d7fa4a8", "type":"basic","issued_to":"issuedTo","issuer":"issuer","issue_date":"2014-09-29","expiry_date":"2030-08-29","max_nodes":1}]} diff --git a/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/LicenseSigner.java b/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/LicenseSigner.java new file mode 100644 index 00000000000..4f634978b1b --- /dev/null +++ b/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/LicenseSigner.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.licensor; + +import org.elasticsearch.common.Base64; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.core.CryptUtils; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.*; +import java.util.Collections; + +/** + * Responsible for generating a license signature according to + * the signature spec and sign it with the provided encrypted private key + */ +public class LicenseSigner { + + private final static int MAGIC_LENGTH = 13; + + private final Path publicKeyPath; + + private final Path privateKeyPath; + + public LicenseSigner(final Path privateKeyPath, final Path publicKeyPath) { + this.publicKeyPath = publicKeyPath; + this.privateKeyPath = privateKeyPath; + } + + /** + * Generates a signature for the licenseSpec. + * Signature structure: + * | VERSION | MAGIC | PUB_KEY_DIGEST | SIGNED_LICENSE_CONTENT | + * + * @return a signed License + */ + public License sign(License licenseSpec) throws IOException { + XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + licenseSpec.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true"))); + final byte[] signedContent; + try { + final Signature rsa = Signature.getInstance("SHA512withRSA"); + rsa.initSign(CryptUtils.readEncryptedPrivateKey(Files.readAllBytes(privateKeyPath))); + rsa.update(contentBuilder.bytes().toBytes()); + signedContent = rsa.sign(); + } catch (InvalidKeyException | IOException | NoSuchAlgorithmException | SignatureException e) { + throw new IllegalStateException(e); + } + final byte[] magic = new byte[MAGIC_LENGTH]; + SecureRandom random = new SecureRandom(); + random.nextBytes(magic); + final byte[] hash = Base64.encodeBytesToBytes(Files.readAllBytes(publicKeyPath)); + assert hash != null; + byte[] bytes = new byte[4 + 4 + MAGIC_LENGTH + 4 + hash.length + 4 + signedContent.length]; + ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); + byteBuffer.putInt(licenseSpec.version()) + .putInt(magic.length) + .put(magic) + .putInt(hash.length) + .put(hash) + .putInt(signedContent.length) + .put(signedContent); + + return License.builder() + .fromLicenseSpec(licenseSpec, Base64.encodeBytes(bytes)) + .build(); + } +} diff --git a/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/KeyPairGeneratorTool.java b/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/KeyPairGeneratorTool.java new file mode 100644 index 00000000000..73c911e76fd --- /dev/null +++ b/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/KeyPairGeneratorTool.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.licensor.tools; + +import org.apache.commons.cli.CommandLine; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.cli.CliTool; +import org.elasticsearch.common.cli.CliToolConfig; +import org.elasticsearch.common.cli.Terminal; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; + +import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd; +import static org.elasticsearch.common.cli.CliToolConfig.Builder.option; +import static org.elasticsearch.common.cli.CliToolConfig.config; +import static org.elasticsearch.license.core.CryptUtils.writeEncryptedPrivateKey; +import static org.elasticsearch.license.core.CryptUtils.writeEncryptedPublicKey; + +public class KeyPairGeneratorTool extends CliTool { + + public static final String NAME = "key-pair-generator"; + private static final CliToolConfig CONFIG = config("licensor", KeyPairGeneratorTool.class) + .cmds(KeyGenerator.CMD) + .build(); + + public KeyPairGeneratorTool() { + super(CONFIG); + } + + @Override + protected Command parse(String s, CommandLine commandLine) throws Exception { + return KeyGenerator.parse(terminal, commandLine, env); + } + + public static class KeyGenerator extends Command { + + private static final CliToolConfig.Cmd CMD = cmd(NAME, KeyGenerator.class) + .options( + option("pub", "publicKeyPath").required(true).hasArg(true), + option("pri", "privateKeyPath").required(true).hasArg(true) + ).build(); + + public final Path publicKeyPath; + public final Path privateKeyPath; + + protected KeyGenerator(Terminal terminal, Path publicKeyPath, Path privateKeyPath) { + super(terminal); + this.privateKeyPath = privateKeyPath; + this.publicKeyPath = publicKeyPath; + } + + public static Command parse(Terminal terminal, CommandLine commandLine, Environment environment) { + Path publicKeyPath = environment.binFile().getParent().resolve(commandLine.getOptionValue("publicKeyPath")); + Path privateKeyPath = environment.binFile().getParent().resolve(commandLine.getOptionValue("privateKeyPath")); + + if (Files.exists(privateKeyPath)) { + return exitCmd(ExitStatus.USAGE, terminal, privateKeyPath + " already exists"); + } else if (Files.exists(publicKeyPath)) { + return exitCmd(ExitStatus.USAGE, terminal, publicKeyPath + " already exists"); + } + return new KeyGenerator(terminal, publicKeyPath, privateKeyPath); + } + + @Override + public ExitStatus execute(Settings settings, Environment env) throws Exception { + KeyPair keyPair = generateKeyPair(privateKeyPath, publicKeyPath); + terminal.println(Terminal.Verbosity.VERBOSE, "generating key pair [public key: " + publicKeyPath + ", private key: " + privateKeyPath + "]"); + return (keyPair != null) ? ExitStatus.OK : ExitStatus.CANT_CREATE; + } + + private static KeyPair generateKeyPair(Path privateKeyPath, Path publicKeyPath) throws IOException, NoSuchAlgorithmException { + SecureRandom random = new SecureRandom(); + + KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(2048, random); + KeyPair keyPair = keyGen.generateKeyPair(); + + saveKeyPairToFiles(keyPair, privateKeyPath, publicKeyPath); + return keyPair; + } + } + + private static void saveKeyPairToFiles(KeyPair keyPair, Path privateKeyPath, Path publicKeyPath) throws IOException { + Files.write(privateKeyPath, writeEncryptedPrivateKey(keyPair.getPrivate())); + Files.write(publicKeyPath, writeEncryptedPublicKey(keyPair.getPublic())); + } + + public static void main(String[] args) throws Exception { + ExitStatus exitStatus = new KeyPairGeneratorTool().execute(args); + exit(exitStatus.status()); + } + + @SuppressForbidden(reason = "Allowed to exit explicitly from #main()") + private static void exit(int status) { + System.exit(status); + } +} diff --git a/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/LicenseGeneratorTool.java b/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/LicenseGeneratorTool.java new file mode 100644 index 00000000000..0064a3a7fa4 --- /dev/null +++ b/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/LicenseGeneratorTool.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.licensor.tools; + +import org.apache.commons.cli.CommandLine; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.cli.CliTool; +import org.elasticsearch.common.cli.CliToolConfig; +import org.elasticsearch.common.cli.Terminal; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.env.Environment; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.licensor.LicenseSigner; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd; +import static org.elasticsearch.common.cli.CliToolConfig.Builder.option; +import static org.elasticsearch.common.cli.CliToolConfig.config; + +public class LicenseGeneratorTool extends CliTool { + public static final String NAME = "license-generator"; + + private static final CliToolConfig CONFIG = config("licensor", LicenseGeneratorTool.class) + .cmds(LicenseGenerator.CMD) + .build(); + + public LicenseGeneratorTool() { + super(CONFIG); + } + + @Override + protected Command parse(String s, CommandLine commandLine) throws Exception { + return LicenseGenerator.parse(terminal, commandLine, env); + } + + public static class LicenseGenerator extends Command { + + private static final CliToolConfig.Cmd CMD = cmd(NAME, LicenseGenerator.class) + .options( + option("pub", "publicKeyPath").required(true).hasArg(true), + option("pri", "privateKeyPath").required(true).hasArg(true), + option("l", "license").required(false).hasArg(true), + option("lf", "licenseFile").required(false).hasArg(true) + ).build(); + + public final License licenseSpec; + public final Path publicKeyFilePath; + public final Path privateKeyFilePath; + + public LicenseGenerator(Terminal terminal, Path publicKeyFilePath, Path privateKeyFilePath, License licenseSpec) { + super(terminal); + this.licenseSpec = licenseSpec; + this.privateKeyFilePath = privateKeyFilePath; + this.publicKeyFilePath = publicKeyFilePath; + } + + public static Command parse(Terminal terminal, CommandLine commandLine, Environment environment) throws IOException { + Path publicKeyPath = environment.binFile().getParent().resolve(commandLine.getOptionValue("publicKeyPath")); + Path privateKeyPath = environment.binFile().getParent().resolve(commandLine.getOptionValue("privateKeyPath")); + String licenseSpecSource = commandLine.getOptionValue("license"); + String licenseSpecSourceFile = commandLine.getOptionValue("licenseFile"); + + if (!Files.exists(privateKeyPath)) { + return exitCmd(ExitStatus.USAGE, terminal, privateKeyPath + " does not exist"); + } else if (!Files.exists(publicKeyPath)) { + return exitCmd(ExitStatus.USAGE, terminal, publicKeyPath + " does not exist"); + } + + License license = null; + if (licenseSpecSource != null) { + license = License.fromSource(licenseSpecSource); + } else if (licenseSpecSourceFile != null) { + Path licenseSpecPath = environment.binFile().getParent().resolve(licenseSpecSourceFile); + if (!Files.exists(licenseSpecPath)) { + return exitCmd(ExitStatus.USAGE, terminal, licenseSpecSourceFile + " does not exist"); + } + license = License.fromSource(Files.readAllBytes(licenseSpecPath)); + } + + if (license == null) { + return exitCmd(ExitStatus.USAGE, terminal, "no license spec provided"); + } + return new LicenseGenerator(terminal, publicKeyPath, privateKeyPath, license); + } + + @Override + public ExitStatus execute(Settings settings, Environment env) throws Exception { + + // sign + License license = new LicenseSigner(privateKeyFilePath, publicKeyFilePath).sign(licenseSpec); + + // dump + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.startObject(); + builder.startObject("license"); + license.toInnerXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + builder.endObject(); + builder.flush(); + terminal.print(builder.string()); + + return ExitStatus.OK; + } + } + + public static void main(String[] args) throws Exception { + ExitStatus exitStatus = new LicenseGeneratorTool().execute(args); + exit(exitStatus.status()); + } + + @SuppressForbidden(reason = "Allowed to exit explicitly from #main()") + private static void exit(int status) { + System.exit(status); + } +} diff --git a/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/LicenseVerificationTool.java b/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/LicenseVerificationTool.java new file mode 100644 index 00000000000..58666e02bd8 --- /dev/null +++ b/elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/LicenseVerificationTool.java @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.licensor.tools; + +import org.apache.commons.cli.CommandLine; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.cli.CliTool; +import org.elasticsearch.common.cli.CliToolConfig; +import org.elasticsearch.common.cli.Terminal; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.env.Environment; +import org.elasticsearch.license.core.License; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd; +import static org.elasticsearch.common.cli.CliToolConfig.Builder.option; +import static org.elasticsearch.common.cli.CliToolConfig.config; + +public class LicenseVerificationTool extends CliTool { + public static final String NAME = "verify-license"; + + private static final CliToolConfig CONFIG = config("licensor", LicenseVerificationTool.class) + .cmds(LicenseVerifier.CMD) + .build(); + + public LicenseVerificationTool() { + super(CONFIG); + } + + @Override + protected Command parse(String s, CommandLine commandLine) throws Exception { + return LicenseVerifier.parse(terminal, commandLine, env); + } + + public static class LicenseVerifier extends Command { + + private static final CliToolConfig.Cmd CMD = cmd(NAME, LicenseVerifier.class) + .options( + option("pub", "publicKeyPath").required(true).hasArg(true), + option("l", "license").required(false).hasArg(true), + option("lf", "licenseFile").required(false).hasArg(true) + ).build(); + + public final License license; + public final Path publicKeyPath; + + public LicenseVerifier(Terminal terminal, License license, Path publicKeyPath) { + super(terminal); + this.license = license; + this.publicKeyPath = publicKeyPath; + } + + public static Command parse(Terminal terminal, CommandLine commandLine, Environment environment) throws IOException { + String publicKeyPathString = commandLine.getOptionValue("publicKeyPath"); + String licenseSource = commandLine.getOptionValue("license"); + String licenseSourceFile = commandLine.getOptionValue("licenseFile"); + + License license = null; + if (licenseSource != null) { + license = License.fromSource(licenseSource); + } else if (licenseSourceFile != null) { + Path licenseSpecPath = environment.binFile().getParent().resolve(licenseSourceFile); + if (!Files.exists(licenseSpecPath)) { + return exitCmd(ExitStatus.USAGE, terminal, licenseSourceFile + " does not exist"); + } + license = License.fromSource(Files.readAllBytes(licenseSpecPath)); + } + if (license == null) { + return exitCmd(ExitStatus.USAGE, terminal, "no license spec provided"); + } + Path publicKeyPath = environment.binFile().getParent().resolve(publicKeyPathString); + if (!Files.exists(publicKeyPath)) { + return exitCmd(ExitStatus.USAGE, terminal, publicKeyPath + " does not exist"); + } + return new LicenseVerifier(terminal, license, publicKeyPath); + } + + @Override + public ExitStatus execute(Settings settings, Environment env) throws Exception { + + // verify + if (!org.elasticsearch.license.core.LicenseVerifier.verifyLicense(license, Files.readAllBytes(publicKeyPath))) { + terminal.println("Invalid License!"); + return ExitStatus.DATA_ERROR; + } + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.startObject(); + builder.startObject("license"); + license.toInnerXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + builder.endObject(); + builder.flush(); + terminal.print(builder.string()); + return ExitStatus.OK; + } + } + + public static void main(String[] args) throws Exception { + ExitStatus exitStatus = new LicenseVerificationTool().execute(args); + exit(exitStatus.status()); + } + + @SuppressForbidden(reason = "Allowed to exit explicitly from #main()") + private static void exit(int status) { + System.exit(status); + } +} diff --git a/elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-key-pair-generator.help b/elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-key-pair-generator.help new file mode 100644 index 00000000000..c41ed5c0ed7 --- /dev/null +++ b/elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-key-pair-generator.help @@ -0,0 +1,22 @@ +NAME + + key-pair-generator - generates a key pair with RSA 2048-bit security + +SYNOPSIS + + key-pair-generator -pub publicKeyPath -pri privateKeyPath + +DESCRIPTION + + This tool generates and saves a key pair to the provided publicKeyPath + and privateKeyPath. The tool checks the existence of the provided key paths + and will not override if any existing keys are found. + +OPTIONS + + -h,--help Shows this message + + -pub,--publicKeyPath Save the generated public key to path + + -pri,--privateKeyPath Save the generated private key to path + diff --git a/elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-license-generator.help b/elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-license-generator.help new file mode 100644 index 00000000000..75d507f7dd9 --- /dev/null +++ b/elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-license-generator.help @@ -0,0 +1,26 @@ +NAME + + license-generator - generates signed elasticsearch license(s) for a given license spec(s) + +SYNOPSIS + + license-generator -l licenseSpec -pub publicKeyPath -pri privateKeyPath + +DESCRIPTION + + This tool generate elasticsearch license(s) for the provided license spec(s). The tool + can take arbitrary number of `--license` and/or `--licenseFile` to generate corrosponding + signed license(s). + +OPTIONS + + -h,--help Shows this message + + -l,--license License spec to generate a signed license from + + -lf,--licenseFile Path to a license spec file + + -pub,--publicKeyPath Path to public key to be used + + -pri,--privateKeyPath Path to private key to be used + diff --git a/elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-verify-license.help b/elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-verify-license.help new file mode 100644 index 00000000000..e36007f3838 --- /dev/null +++ b/elasticsearch/license/licensor/src/main/resources/org/elasticsearch/license/licensor/tools/licensor-verify-license.help @@ -0,0 +1,28 @@ +NAME + + verify-license - verifies the integrity of elasticsearch signed license(s) + +SYNOPSIS + + verify-license -l signedLicense -pub publicKeyPath + +DESCRIPTION + + This tool assumes the configured public key to be the same as that of the production license plugin public key. + The tool can take arbitrary number of `--license` and/or `--licenseFile` for verifying signed license(s). If any + of the provided license(s) are invalid, the tool will error out, otherwise it will output a effective licenses file. + + Effective Licenses: + A set of licenses that only has one effective sub-license for every feature provided through the input license file. + Where effective sub-licenses are identified as the sub-licenses with the latest `expiry_date` for a `feature` + and the sub-license has not already expired. + +OPTIONS + + -h,--help Shows this message + + -l,--license signed license(s) string + + -lf,--licenseFile Path to signed license(s) file + + -pub,--publicKeyPath Path to public key to verify against diff --git a/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/LicenseVerificationTests.java b/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/LicenseVerificationTests.java new file mode 100644 index 00000000000..4e081953d6e --- /dev/null +++ b/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/LicenseVerificationTests.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.licensor; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.license.core.DateUtils; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.core.LicenseVerifier; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; +import org.junit.Before; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.hamcrest.Matchers.equalTo; + +public class LicenseVerificationTests extends ESTestCase { + protected Path pubKeyPath = null; + protected Path priKeyPath = null; + + @Before + public void setup() throws Exception { + pubKeyPath = getDataPath("/public.key"); + priKeyPath = getDataPath("/private.key"); + } + + @After + public void cleanUp() { + pubKeyPath = null; + priKeyPath = null; + } + + public void testGeneratedLicenses() throws Exception { + assertThat(LicenseVerifier.verifyLicense(TestUtils.generateSignedLicense(TimeValue.timeValueHours(2 * 24), pubKeyPath, priKeyPath), Files.readAllBytes(pubKeyPath)), equalTo(true)); + } + + public void testLicenseTampering() throws Exception { + License license = TestUtils.generateSignedLicense(TimeValue.timeValueHours(2), pubKeyPath, priKeyPath); + + final License tamperedLicense = License.builder() + .fromLicenseSpec(license, license.signature()) + .expiryDate(license.expiryDate() + 10 * 24 * 60 * 60 * 1000l) + .validate() + .build(); + + assertThat(LicenseVerifier.verifyLicense(tamperedLicense, Files.readAllBytes(pubKeyPath)), equalTo(false)); + } + + public void testRandomLicenseVerification() throws Exception { + TestUtils.LicenseSpec licenseSpec = TestUtils.generateRandomLicenseSpec(randomIntBetween(License.VERSION_START, License.VERSION_CURRENT)); + License generatedLicense = generateSignedLicense(licenseSpec, pubKeyPath, priKeyPath); + assertThat(LicenseVerifier.verifyLicense(generatedLicense, Files.readAllBytes(pubKeyPath)), equalTo(true)); + } + + private static License generateSignedLicense(TestUtils.LicenseSpec spec, Path pubKeyPath, Path priKeyPath) throws Exception { + LicenseSigner signer = new LicenseSigner(priKeyPath, pubKeyPath); + License.Builder builder = License.builder() + .uid(spec.uid) + .feature(spec.feature) + .type(spec.type) + .subscriptionType(spec.subscriptionType) + .issuedTo(spec.issuedTo) + .issuer(spec.issuer) + .maxNodes(spec.maxNodes); + + if (spec.expiryDate != null) { + builder.expiryDate(DateUtils.endOfTheDay(spec.expiryDate)); + } else { + builder.expiryDate(spec.expiryDateInMillis); + } + if (spec.issueDate != null) { + builder.issueDate(DateUtils.beginningOfTheDay(spec.issueDate)); + } else { + builder.issueDate(spec.issueDateInMillis); + } + builder.version(spec.version); + return signer.sign(builder.build()); + } +} diff --git a/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/TestUtils.java b/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/TestUtils.java new file mode 100644 index 00000000000..4ca5fdb07e5 --- /dev/null +++ b/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/TestUtils.java @@ -0,0 +1,214 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.licensor; + +import org.elasticsearch.common.joda.DateMathParser; +import org.elasticsearch.common.joda.FormatDateTimeFormatter; +import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.license.core.DateUtils; +import org.elasticsearch.license.core.License; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.MatcherAssert; +import org.joda.time.format.DateTimeFormatter; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.Callable; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.*; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.hamcrest.core.IsEqual.equalTo; + +public class TestUtils { + + public static final String PUBLIC_KEY_RESOURCE = "/public.key"; + public static final String PRIVATE_KEY_RESOURCE = "/private.key"; + + private final static FormatDateTimeFormatter formatDateTimeFormatter = Joda.forPattern("yyyy-MM-dd"); + private final static DateMathParser dateMathParser = new DateMathParser(formatDateTimeFormatter); + private final static DateTimeFormatter dateTimeFormatter = formatDateTimeFormatter.printer(); + + public static String dumpLicense(License license) throws Exception { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.startObject(); + builder.startObject("license"); + license.toInnerXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + builder.endObject(); + return builder.string(); + } + + public static String dateMathString(String time, final long now) { + return dateTimeFormatter.print(dateMathParser.parse(time, new Callable() { + @Override + public Long call() throws Exception { + return now; + } + })); + } + + public static long dateMath(String time, final long now) { + return dateMathParser.parse(time, new Callable() { + @Override + public Long call() throws Exception { + return now; + } + }); + } + + public static LicenseSpec generateRandomLicenseSpec(int version) { + boolean datesInMillis = randomBoolean(); + long now = System.currentTimeMillis(); + String uid = UUID.randomUUID().toString(); + String issuer = "issuer__" + randomInt(); + String issuedTo = "issuedTo__" + randomInt(); + String type = version < License.VERSION_NO_FEATURE_TYPE ? + randomFrom("subscription", "internal", "development") : + randomFrom("basic", "silver", "dev", "gold", "platinum"); + final String subscriptionType; + final String feature; + if (version < License.VERSION_NO_FEATURE_TYPE) { + subscriptionType = randomFrom("gold", "silver", "platinum"); + feature = "feature__" + randomInt(); + } else { + subscriptionType = null; + feature = null; + } + int maxNodes = randomIntBetween(5, 100); + if (datesInMillis) { + long issueDateInMillis = dateMath("now", now); + long expiryDateInMillis = dateMath("now+10d/d", now); + return new LicenseSpec(version, uid, feature, issueDateInMillis, expiryDateInMillis, type, subscriptionType, issuedTo, issuer, maxNodes); + } else { + String issueDate = dateMathString("now", now); + String expiryDate = dateMathString("now+10d/d", now); + return new LicenseSpec(version, uid, feature, issueDate, expiryDate, type, subscriptionType, issuedTo, issuer, maxNodes); + } + } + + public static String generateLicenseSpecString(LicenseSpec licenseSpec) throws IOException { + XContentBuilder licenses = jsonBuilder(); + licenses.startObject(); + licenses.startObject("license") + .field("uid", licenseSpec.uid) + .field("type", licenseSpec.type) + .field("subscription_type", licenseSpec.subscriptionType) + .field("issued_to", licenseSpec.issuedTo) + .field("issuer", licenseSpec.issuer) + .field("feature", licenseSpec.feature) + .field("max_nodes", licenseSpec.maxNodes); + + if (licenseSpec.issueDate != null) { + licenses.field("issue_date", licenseSpec.issueDate); + } else { + licenses.field("issue_date_in_millis", licenseSpec.issueDateInMillis); + } + if (licenseSpec.expiryDate != null) { + licenses.field("expiry_date", licenseSpec.expiryDate); + } else { + licenses.field("expiry_date_in_millis", licenseSpec.expiryDateInMillis); + } + licenses.field("version", licenseSpec.version); + licenses.endObject(); + licenses.endObject(); + return licenses.string(); + } + + public static void assertLicenseSpec(LicenseSpec spec, License license) { + MatcherAssert.assertThat(license.uid(), equalTo(spec.uid)); + MatcherAssert.assertThat(license.issuedTo(), equalTo(spec.issuedTo)); + MatcherAssert.assertThat(license.issuer(), equalTo(spec.issuer)); + MatcherAssert.assertThat(license.type(), equalTo(spec.type)); + MatcherAssert.assertThat(license.maxNodes(), equalTo(spec.maxNodes)); + if (spec.issueDate != null) { + MatcherAssert.assertThat(license.issueDate(), equalTo(DateUtils.beginningOfTheDay(spec.issueDate))); + } else { + MatcherAssert.assertThat(license.issueDate(), equalTo(spec.issueDateInMillis)); + } + if (spec.expiryDate != null) { + MatcherAssert.assertThat(license.expiryDate(), equalTo(DateUtils.endOfTheDay(spec.expiryDate))); + } else { + MatcherAssert.assertThat(license.expiryDate(), equalTo(spec.expiryDateInMillis)); + } + } + + public static License generateSignedLicense(TimeValue expiryDuration, Path pubKeyPath, Path priKeyPath) throws Exception { + long issue = System.currentTimeMillis(); + int version = ESTestCase.randomIntBetween(License.VERSION_START, License.VERSION_CURRENT); + String type = version < License.VERSION_NO_FEATURE_TYPE ? + randomFrom("subscription", "internal", "development") : + randomFrom("trial", "basic", "silver", "dev", "gold", "platinum"); + final License.Builder builder = License.builder() + .uid(UUID.randomUUID().toString()) + .expiryDate(issue + expiryDuration.getMillis()) + .issueDate(issue) + .version(version) + .type(type) + .issuedTo("customer") + .issuer("elasticsearch") + .maxNodes(5); + if (version == License.VERSION_START) { + builder.subscriptionType(randomFrom("dev", "gold", "platinum", "silver")); + builder.feature(ESTestCase.randomAsciiOfLength(10)); + } + LicenseSigner signer = new LicenseSigner(priKeyPath, pubKeyPath); + return signer.sign(builder.build()); + } + + public static class LicenseSpec { + public final int version; + public final String feature; + public final String issueDate; + public final long issueDateInMillis; + public final String expiryDate; + public final long expiryDateInMillis; + public final String uid; + public final String type; + public final String subscriptionType; + public final String issuedTo; + public final String issuer; + public final int maxNodes; + + public LicenseSpec(int version, String uid, String feature, long issueDateInMillis, long expiryDateInMillis, String type, + String subscriptionType, String issuedTo, String issuer, int maxNodes) { + this.version = version; + this.feature = feature; + this.issueDateInMillis = issueDateInMillis; + this.issueDate = null; + this.expiryDateInMillis = expiryDateInMillis; + this.expiryDate = null; + this.uid = uid; + this.type = type; + this.subscriptionType = subscriptionType; + this.issuedTo = issuedTo; + this.issuer = issuer; + this.maxNodes = maxNodes; + } + + public LicenseSpec(int version, String uid, String feature, String issueDate, String expiryDate, String type, + String subscriptionType, String issuedTo, String issuer, int maxNodes) { + this.version = version; + this.feature = feature; + this.issueDate = issueDate; + this.issueDateInMillis = -1; + this.expiryDate = expiryDate; + this.expiryDateInMillis = -1; + this.uid = uid; + this.type = type; + this.subscriptionType = subscriptionType; + this.issuedTo = issuedTo; + this.issuer = issuer; + this.maxNodes = maxNodes; + } + } +} diff --git a/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/KeyPairGenerationToolTests.java b/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/KeyPairGenerationToolTests.java new file mode 100644 index 00000000000..a70e646e292 --- /dev/null +++ b/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/KeyPairGenerationToolTests.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.licensor.tools; + +import org.apache.commons.cli.MissingOptionException; +import org.elasticsearch.common.cli.CliTool.Command; +import org.elasticsearch.common.cli.CliTool.ExitStatus; +import org.elasticsearch.common.cli.CliToolTestCase; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.license.licensor.tools.KeyPairGeneratorTool.KeyGenerator; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.core.IsEqual.equalTo; + +public class KeyPairGenerationToolTests extends CliToolTestCase { + public void testParsingMissingPath() throws Exception { + KeyPairGeneratorTool keyPairGeneratorTool = new KeyPairGeneratorTool(); + Path tempFile = createTempFile(); + try { + keyPairGeneratorTool.parse(KeyPairGeneratorTool.NAME, + new String[] { "--privateKeyPath", tempFile.toAbsolutePath().toString() }); + fail("no public key path provided"); + } catch (MissingOptionException e) { + assertThat(e.getMessage(), containsString("pub")); + } + try { + keyPairGeneratorTool.parse(KeyPairGeneratorTool.NAME, + new String[] { "--publicKeyPath", tempFile.toAbsolutePath().toString() }); + fail("no private key path provided"); + } catch (MissingOptionException e) { + assertThat(e.getMessage(), containsString("pri")); + } + } + + public void testParsingNeverOverrideKey() throws Exception { + KeyPairGeneratorTool keyPairGeneratorTool = new KeyPairGeneratorTool(); + Path tempFile = createTempFile(); + Path tempFile2 = createTempFile(); + String nonExistentFilePath = tempFile2.toAbsolutePath().toString(); + Files.delete(tempFile2); + assertThat(Files.exists(tempFile2), equalTo(false)); + + Command command = keyPairGeneratorTool.parse(KeyPairGeneratorTool.NAME, new String[] {"--privateKeyPath", tempFile.toAbsolutePath().toString(), + "--publicKeyPath", nonExistentFilePath }); + + assertThat(command, instanceOf(Command.Exit.class)); + Command.Exit exitCommand = (Command.Exit) command; + assertThat(exitCommand.status(), equalTo(ExitStatus.USAGE)); + + command = keyPairGeneratorTool.parse(KeyPairGeneratorTool.NAME, new String[] {"--publicKeyPath", tempFile.toAbsolutePath().toString(), + "--privateKeyPath", nonExistentFilePath }); + + assertThat(command, instanceOf(Command.Exit.class)); + exitCommand = (Command.Exit) command; + assertThat(exitCommand.status(), equalTo(ExitStatus.USAGE)); + } + + public void testToolSimple() throws Exception { + KeyPairGeneratorTool keyPairGeneratorTool = new KeyPairGeneratorTool(); + Path publicKeyFilePath = createTempFile().toAbsolutePath(); + Path privateKeyFilePath = createTempFile().toAbsolutePath(); + Settings settings = Settings.builder().put("path.home", createTempDir("KeyPairGenerationToolTests")).build(); + + Files.delete(publicKeyFilePath); + Files.delete(privateKeyFilePath); + assertThat(Files.exists(publicKeyFilePath), equalTo(false)); + assertThat(Files.exists(privateKeyFilePath), equalTo(false)); + + Command command = keyPairGeneratorTool.parse(KeyPairGeneratorTool.NAME, new String[] { "--privateKeyPath", privateKeyFilePath.toString(), + "--publicKeyPath", publicKeyFilePath.toString() }); + + assertThat(command, instanceOf(KeyGenerator.class)); + KeyGenerator keyGenerator = (KeyGenerator) command; + assertThat(keyGenerator.privateKeyPath, equalTo(privateKeyFilePath)); + assertThat(keyGenerator.publicKeyPath, equalTo(publicKeyFilePath)); + + assertThat(Files.exists(publicKeyFilePath), equalTo(false)); + assertThat(Files.exists(privateKeyFilePath), equalTo(false)); + + assertThat(keyGenerator.execute(settings, new Environment(settings)), equalTo(ExitStatus.OK)); + assertThat(Files.exists(publicKeyFilePath), equalTo(true)); + assertThat(Files.exists(privateKeyFilePath), equalTo(true)); + + Files.delete(publicKeyFilePath); + Files.delete(privateKeyFilePath); + } +} diff --git a/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/LicenseGenerationToolTests.java b/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/LicenseGenerationToolTests.java new file mode 100644 index 00000000000..578b6b42a71 --- /dev/null +++ b/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/LicenseGenerationToolTests.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.licensor.tools; + +import org.apache.commons.cli.MissingOptionException; +import org.elasticsearch.common.cli.CliTool.Command; +import org.elasticsearch.common.cli.CliTool.ExitStatus; +import org.elasticsearch.common.cli.CliToolTestCase; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.licensor.TestUtils; +import org.elasticsearch.license.licensor.tools.LicenseGeneratorTool.LicenseGenerator; +import org.junit.Before; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.core.IsEqual.equalTo; + +public class LicenseGenerationToolTests extends CliToolTestCase { + protected Path pubKeyPath = null; + protected Path priKeyPath = null; + + @Before + public void setup() throws Exception { + logger.error("project.basedir [{}]", System.getProperty("project.basedir")); + pubKeyPath = getDataPath(TestUtils.PUBLIC_KEY_RESOURCE); + priKeyPath = getDataPath(TestUtils.PRIVATE_KEY_RESOURCE); + } + + public void testParsingNonExistentKeyFile() throws Exception { + TestUtils.LicenseSpec inputLicenseSpec = TestUtils.generateRandomLicenseSpec(License.VERSION_CURRENT); + LicenseGeneratorTool licenseGeneratorTool = new LicenseGeneratorTool(); + Command command = licenseGeneratorTool.parse(LicenseGeneratorTool.NAME, + new String[] {"--license", TestUtils.generateLicenseSpecString(inputLicenseSpec), + "--publicKeyPath", pubKeyPath.toString().concat("invalid"), + "--privateKeyPath", priKeyPath.toString() }); + + assertThat(command, instanceOf(Command.Exit.class)); + Command.Exit exitCommand = (Command.Exit) command; + assertThat(exitCommand.status(), equalTo(ExitStatus.USAGE)); + + command = licenseGeneratorTool.parse(LicenseGeneratorTool.NAME, + new String[] {"--license", TestUtils.generateLicenseSpecString(inputLicenseSpec), + "--privateKeyPath", priKeyPath.toString().concat("invalid"), + "--publicKeyPath", pubKeyPath.toString() }); + + assertThat(command, instanceOf(Command.Exit.class)); + exitCommand = (Command.Exit) command; + assertThat(exitCommand.status(), equalTo(ExitStatus.USAGE)); + } + + public void testParsingMissingLicenseSpec() throws Exception { + LicenseGeneratorTool licenseGeneratorTool = new LicenseGeneratorTool(); + Command command = licenseGeneratorTool.parse(LicenseGeneratorTool.NAME, + new String[] { "--publicKeyPath", pubKeyPath.toString(), + "--privateKeyPath", priKeyPath.toString() }); + + assertThat(command, instanceOf(Command.Exit.class)); + Command.Exit exitCommand = (Command.Exit) command; + assertThat(exitCommand.status(), equalTo(ExitStatus.USAGE)); + } + + public void testParsingMissingArgs() throws Exception { + TestUtils.LicenseSpec inputLicenseSpec = TestUtils.generateRandomLicenseSpec(License.VERSION_CURRENT); + LicenseGeneratorTool licenseGeneratorTool = new LicenseGeneratorTool(); + boolean pubKeyMissing = randomBoolean(); + try { + licenseGeneratorTool.parse(LicenseGeneratorTool.NAME, + new String[] { "--license", TestUtils.generateLicenseSpecString(inputLicenseSpec), + ((pubKeyMissing) ? "--privateKeyPath" : "--publicKeyPath"), + ((pubKeyMissing) ? priKeyPath.toString() : pubKeyPath.toString()) }); + fail("missing argument: " + ((pubKeyMissing) ? "publicKeyPath" : "privateKeyPath") + " should throw an exception"); + } catch (MissingOptionException e) { + assertThat(e.getMessage(), containsString((pubKeyMissing) ? "pub" : "pri")); + } + } + + public void testParsingSimple() throws Exception { + TestUtils.LicenseSpec inputLicenseSpec = TestUtils.generateRandomLicenseSpec(License.VERSION_CURRENT); + LicenseGeneratorTool licenseGeneratorTool = new LicenseGeneratorTool(); + Command command = licenseGeneratorTool.parse(LicenseGeneratorTool.NAME, + new String[]{"--license", TestUtils.generateLicenseSpecString(inputLicenseSpec), + "--publicKeyPath", pubKeyPath.toString(), + "--privateKeyPath", priKeyPath.toString() }); + + assertThat(command, instanceOf(LicenseGenerator.class)); + LicenseGenerator licenseGenerator = (LicenseGenerator) command; + assertThat(licenseGenerator.publicKeyFilePath, equalTo(pubKeyPath)); + assertThat(licenseGenerator.privateKeyFilePath, equalTo(priKeyPath)); + TestUtils.assertLicenseSpec(inputLicenseSpec, licenseGenerator.licenseSpec); + } + + public void testParsingLicenseFile() throws Exception { + TestUtils.LicenseSpec inputLicenseSpec = TestUtils.generateRandomLicenseSpec(License.VERSION_CURRENT); + Path tempFile = createTempFile(); + Files.write(tempFile, TestUtils.generateLicenseSpecString(inputLicenseSpec).getBytes(StandardCharsets.UTF_8)); + + LicenseGeneratorTool licenseGeneratorTool = new LicenseGeneratorTool(); + Command command = licenseGeneratorTool.parse(LicenseGeneratorTool.NAME, + new String[] { "--licenseFile", tempFile.toAbsolutePath().toString(), + "--publicKeyPath", pubKeyPath.toString(), + "--privateKeyPath", priKeyPath.toString() }); + + assertThat(command, instanceOf(LicenseGenerator.class)); + LicenseGenerator licenseGenerator = (LicenseGenerator) command; + assertThat(licenseGenerator.publicKeyFilePath, equalTo(pubKeyPath)); + assertThat(licenseGenerator.privateKeyFilePath, equalTo(priKeyPath)); + TestUtils.assertLicenseSpec(inputLicenseSpec, licenseGenerator.licenseSpec); + } + + public void testTool() throws Exception { + TestUtils.LicenseSpec licenseSpec = TestUtils.generateRandomLicenseSpec(License.VERSION_CURRENT); + License license = License.fromSource(TestUtils.generateLicenseSpecString(licenseSpec).getBytes(StandardCharsets.UTF_8)); + String output = runLicenseGenerationTool(pubKeyPath, priKeyPath, license, ExitStatus.OK); + License outputLicense = License.fromSource(output.getBytes(StandardCharsets.UTF_8)); + TestUtils.assertLicenseSpec(licenseSpec, outputLicense); + } + + private String runLicenseGenerationTool(Path pubKeyPath, Path priKeyPath, License licenseSpec, ExitStatus expectedExitStatus) throws Exception { + CaptureOutputTerminal outputTerminal = new CaptureOutputTerminal(); + Settings settings = Settings.builder().put("path.home", createTempDir("LicenseGenerationToolTests")).build(); + LicenseGenerator licenseGenerator = new LicenseGenerator(outputTerminal, pubKeyPath, priKeyPath, licenseSpec); + assertThat(execute(licenseGenerator, settings), equalTo(expectedExitStatus)); + assertThat(outputTerminal.getTerminalOutput().size(), equalTo(1)); + return outputTerminal.getTerminalOutput().get(0); + } + + private ExitStatus execute(Command cmd, Settings settings) throws Exception { + Environment env = new Environment(settings); + return cmd.execute(settings, env); + } +} diff --git a/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/LicenseVerificationToolTests.java b/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/LicenseVerificationToolTests.java new file mode 100644 index 00000000000..16a5bb70b00 --- /dev/null +++ b/elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/LicenseVerificationToolTests.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.licensor.tools; + +import org.apache.commons.cli.MissingOptionException; +import org.elasticsearch.common.cli.CliTool.Command; +import org.elasticsearch.common.cli.CliTool.ExitStatus; +import org.elasticsearch.common.cli.CliToolTestCase; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.env.Environment; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.licensor.TestUtils; +import org.elasticsearch.license.licensor.tools.LicenseVerificationTool.LicenseVerifier; +import org.hamcrest.CoreMatchers; +import org.junit.Before; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.core.IsEqual.equalTo; + +public class LicenseVerificationToolTests extends CliToolTestCase { + protected Path pubKeyPath = null; + protected Path priKeyPath = null; + + @Before + public void setup() throws Exception { + logger.error("project.basedir [{}]", System.getProperty("project.basedir")); + pubKeyPath = getDataPath(TestUtils.PUBLIC_KEY_RESOURCE); + priKeyPath = getDataPath(TestUtils.PRIVATE_KEY_RESOURCE); + } + + public void testParsingMissingLicense() throws Exception { + LicenseVerificationTool licenseVerificationTool = new LicenseVerificationTool(); + Path path = getDataPath(TestUtils.PUBLIC_KEY_RESOURCE); + Command command = licenseVerificationTool.parse(LicenseVerificationTool.NAME, new String[] { "--publicKeyPath", path.toString() }); + + assertThat(command, instanceOf(Command.Exit.class)); + Command.Exit exitCommand = (Command.Exit) command; + assertThat(exitCommand.status(), equalTo(ExitStatus.USAGE)); + } + + public void testParsingMissingPublicKeyPath() throws Exception { + License inputLicense = TestUtils.generateSignedLicense(TimeValue.timeValueHours(1), pubKeyPath, priKeyPath); + LicenseVerificationTool licenseVerificationTool = new LicenseVerificationTool(); + try { + licenseVerificationTool.parse(LicenseVerificationTool.NAME, + new String[] { "--license", TestUtils.dumpLicense(inputLicense) }); + } catch (MissingOptionException e) { + assertThat(e.getMessage(), containsString("pub")); + } + } + + public void testParsingNonExistentPublicKeyPath() throws Exception { + LicenseVerificationTool licenseVerificationTool = new LicenseVerificationTool(); + Path path = getDataPath(TestUtils.PUBLIC_KEY_RESOURCE); + Command command = licenseVerificationTool.parse(LicenseVerificationTool.NAME, + new String[] { "--publicKeyPath", path.toString().concat(".invalid") }); + + assertThat(command, instanceOf(Command.Exit.class)); + Command.Exit exitCommand = (Command.Exit) command; + assertThat(exitCommand.status(), equalTo(ExitStatus.USAGE)); + } + + public void testParsingSimple() throws Exception { + License inputLicense = TestUtils.generateSignedLicense(TimeValue.timeValueHours(1), pubKeyPath, priKeyPath); + LicenseVerificationTool licenseVerificationTool = new LicenseVerificationTool(); + Command command = licenseVerificationTool.parse(LicenseVerificationTool.NAME, + new String[] { "--license", TestUtils.dumpLicense(inputLicense), + "--publicKeyPath", getDataPath(TestUtils.PUBLIC_KEY_RESOURCE).toString() }); + assertThat(command, instanceOf(LicenseVerifier.class)); + LicenseVerifier licenseVerifier = (LicenseVerifier) command; + assertThat(inputLicense, equalTo(licenseVerifier.license)); + } + + public void testParsingLicenseFile() throws Exception { + License inputLicense = TestUtils.generateSignedLicense(TimeValue.timeValueHours(1), pubKeyPath, priKeyPath); + + LicenseVerificationTool licenseVerificationTool = new LicenseVerificationTool(); + Command command = licenseVerificationTool.parse(LicenseVerificationTool.NAME, + new String[]{"--licenseFile", dumpLicenseAsFile(inputLicense), + "--publicKeyPath", getDataPath(TestUtils.PUBLIC_KEY_RESOURCE).toString()}); + assertThat(command, instanceOf(LicenseVerifier.class)); + LicenseVerifier licenseVerifier = (LicenseVerifier) command; + assertThat(inputLicense, equalTo(licenseVerifier.license)); + + } + + public void testParsingMultipleLicense() throws Exception { + License license = TestUtils.generateSignedLicense(TimeValue.timeValueHours(1), pubKeyPath, priKeyPath); + List arguments = new ArrayList<>(); + arguments.add("--license"); + arguments.add(TestUtils.dumpLicense(license)); + arguments.add("--publicKeyPath"); + arguments.add(getDataPath(TestUtils.PUBLIC_KEY_RESOURCE).toString()); + LicenseVerificationTool licenseVerificationTool = new LicenseVerificationTool(); + Command command = licenseVerificationTool.parse(LicenseVerificationTool.NAME, arguments.toArray(new String[arguments.size()])); + + assertThat(command, instanceOf(LicenseVerifier.class)); + LicenseVerifier licenseVerifier = (LicenseVerifier) command; + assertThat(licenseVerifier.license, equalTo(license)); + } + + public void testToolSimple() throws Exception { + License license = TestUtils.generateSignedLicense(TimeValue.timeValueHours(1), pubKeyPath, priKeyPath); + String output = runLicenseVerificationTool(license, getDataPath(TestUtils.PUBLIC_KEY_RESOURCE), ExitStatus.OK); + License outputLicense = License.fromSource(output.getBytes(StandardCharsets.UTF_8)); + assertThat(outputLicense, CoreMatchers.equalTo(license)); + } + + public void testToolInvalidLicense() throws Exception { + License signedLicense = TestUtils.generateSignedLicense(TimeValue.timeValueHours(1), pubKeyPath, priKeyPath); + + License tamperedLicense = License.builder() + .fromLicenseSpec(signedLicense, signedLicense.signature()) + .expiryDate(signedLicense.expiryDate() + randomIntBetween(1, 1000)).build(); + + runLicenseVerificationTool(tamperedLicense, getDataPath(TestUtils.PUBLIC_KEY_RESOURCE), ExitStatus.DATA_ERROR); + } + + private String dumpLicenseAsFile(License license) throws Exception { + Path tempFile = createTempFile(); + Files.write(tempFile, TestUtils.dumpLicense(license).getBytes(StandardCharsets.UTF_8)); + return tempFile.toAbsolutePath().toString(); + } + + private String runLicenseVerificationTool(License license, Path publicKeyPath, ExitStatus expectedExitStatus) throws Exception { + CaptureOutputTerminal outputTerminal = new CaptureOutputTerminal(); + Settings settings = Settings.builder().put("path.home", createTempDir("LicenseVerificationToolTests")).build(); + LicenseVerifier licenseVerifier = new LicenseVerifier(outputTerminal, license, publicKeyPath); + assertThat(execute(licenseVerifier, settings), equalTo(expectedExitStatus)); + if (expectedExitStatus == ExitStatus.OK) { + assertThat(outputTerminal.getTerminalOutput().size(), equalTo(1)); + + return outputTerminal.getTerminalOutput().get(0); + } else { + return null; + } + } + + private ExitStatus execute(Command cmd, Settings settings) throws Exception { + Environment env = new Environment(settings); + return cmd.execute(settings, env); + } +} diff --git a/elasticsearch/license/licensor/src/test/resources/log4j.properties b/elasticsearch/license/licensor/src/test/resources/log4j.properties new file mode 100644 index 00000000000..76defc8660c --- /dev/null +++ b/elasticsearch/license/licensor/src/test/resources/log4j.properties @@ -0,0 +1,11 @@ +es.logger.level=INFO +log4j.rootLogger=${es.logger.level}, out + +log4j.logger.org.apache.http=INFO, out +log4j.additivity.org.apache.http=false + +log4j.logger.org.elasticsearch.license=TRACE + +log4j.appender.out=org.apache.log4j.ConsoleAppender +log4j.appender.out.layout=org.apache.log4j.PatternLayout +log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n diff --git a/elasticsearch/license/licensor/src/test/resources/private.key b/elasticsearch/license/licensor/src/test/resources/private.key new file mode 100644 index 0000000000000000000000000000000000000000..1f545803d875598d976b206250d5e95667c794ef GIT binary patch literal 1232 zcmV;>1TXv1%Bc_wbLO>E3}&#D<@-rDJ$(p047kR zkI3mLE>J{X5~_|GE5>Bxd81`fV@~0j8z81fwu78=uPV1?-b_fL$snD`NE}Zsv(l}S z+Dr52K=W~W#&Y(zE)IK9$=-71EtPe#X0XhttX&@1ia@?lvh&>PD=A#>rfdl2Ru)*H zKaKRTp&qI~gmO-)fq;gzrbngk1r$cUKd~25@6;rjPJf6eLtCHTaDU8Rbt#KGKlZ%g zPd`h6iZ4kWgcXo9P=@b~YoN9WQmtytp^dd%)MXVsW%n~=T9#TzSWvr*YCR25X&mo6 zJoJz>&ju@jBz@-J;se7XMiU5&x@g8>FTI*r^O6Xv2(VF_R@Vk3wQBmIE>&XqH^t=M z5_c~f_`fm)R~9ZqEv|Ma3$nQ}LvHLD`h~L2VKXBZwZnP`c6%Qwnwq8Z0!hd2+^*Gk z6R7QdYCwK?kavUU0Vd*Mq5K{M!Vk zzSLrHwA_oZsoml1+m@@Wyh8$ZA^E2_FpxTTRz7~A$x?V+XFW0!aa4^L&A*^Evlz{gN8ae8h9=J=b zC8F19FN`xNpxA!48b`dGRK3GbQMOho1>byaB*psVh{^^yqVL4J0~$j7viqxma>pYOhkH~PwpJT1j6^yHDMWGO{vw8PyI zIrYD&$F3e@0!346r3BY7JkLDD+7E5EE6oiV(gQ62K@eQRE&#pRq81Tut!V+1TCM~srn6-g`!g_`uozhhJEnN6u9$5|& zLC>$@xw*^CD*fb$ngUWU)yZ%TqX*wM!HU2(O2RRFnw&?^e*xOZWZ?n`No9lb?BLpV=@K}hkB uJ-}qCO)pTE(AWQ24*9x_mu179!iEY^J(*;5;=+=*kZC+fDbTdAaQBAù<}[Á2ÖÓàZÖ|5‹¯ÊÊ7%ØêD +Yå‘xn:¼lúLÈæHò¢«Ë2˜ŸHvEEWÇ\¦H:“6Žh9 [!š…Ûæ©Š¤+;Ö.w7Cì©_|Þ ÓªÏÁ*ñ§D`ƒÚ?‚ùxU/3>x­UÓ“+ è \ No newline at end of file diff --git a/elasticsearch/license/plugin-api/.gitignore b/elasticsearch/license/plugin-api/.gitignore new file mode 100644 index 00000000000..ab956abf6f9 --- /dev/null +++ b/elasticsearch/license/plugin-api/.gitignore @@ -0,0 +1 @@ +/eclipse-build/ diff --git a/elasticsearch/license/plugin-api/build.gradle b/elasticsearch/license/plugin-api/build.gradle new file mode 100644 index 00000000000..7e38efa068d --- /dev/null +++ b/elasticsearch/license/plugin-api/build.gradle @@ -0,0 +1,15 @@ +apply plugin: 'elasticsearch.build' + +dependencies { + compile project(':x-plugins:elasticsearch:license:base') + compile "org.elasticsearch:elasticsearch:${version}" + testCompile project(':x-plugins:elasticsearch:license:licensor') + testCompile "org.elasticsearch:test-framework:${version}" +} + +dependencyLicenses.enabled = false + +jar { + baseName = 'license-plugin-api' +} + diff --git a/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/AbstractLicenseeComponent.java b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/AbstractLicenseeComponent.java new file mode 100644 index 00000000000..9c8fd1ed6bd --- /dev/null +++ b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/AbstractLicenseeComponent.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.settings.Settings; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * A supporting base class for injectable Licensee components. + */ +public abstract class AbstractLicenseeComponent> extends AbstractLifecycleComponent implements Licensee { + + private final String id; + private final LicenseeRegistry clientService; + private final List listeners = new CopyOnWriteArrayList<>(); + + // we initialize the licensee state to enabled with trial operation mode + protected volatile Status status = Status.ENABLED; + + protected AbstractLicenseeComponent(Settings settings, String id, LicenseeRegistry clientService) { + super(settings); + this.id = id; + this.clientService = clientService; + } + + @Override + protected void doStart() { + clientService.register(this); + } + + @Override + protected void doStop() { + } + + @Override + protected void doClose() { + } + + @Override + public final String id() { + return id; + } + + /** + * @return the current status of this licensee (can never be null) + */ + public Status getStatus() { + return status; + } + + public void add(Listener listener) { + listeners.add(listener); + } + + @Override + public void onChange(Status status) { + this.status = status; + logger.trace("[{}] is running in [{}] mode", id(), status); + for (Listener listener : listeners) { + listener.onChange(status); + } + } + + public interface Listener { + void onChange(Status status); + } + +} diff --git a/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseState.java b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseState.java new file mode 100644 index 00000000000..a6eba86c532 --- /dev/null +++ b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseState.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +/** + * States of a registered licensee + * based on the current license + */ +public enum LicenseState { + + /** + * Active license is valid. + * + * When license expires + * changes to {@link #GRACE_PERIOD} + */ + ENABLED, + + /** + * Active license expired + * but grace period has not. + * + * When grace period expires + * changes to {@link #DISABLED}. + * When valid license is installed + * changes back to {@link #ENABLED} + */ + GRACE_PERIOD, + + /** + * Grace period for active license + * expired. + * + * When a valid license is installed + * changes to {@link #ENABLED}, otherwise + * remains unchanged + */ + DISABLED +} diff --git a/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseUtils.java b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseUtils.java new file mode 100644 index 00000000000..e90166b87ed --- /dev/null +++ b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseUtils.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.rest.RestStatus; + +public class LicenseUtils { + + public final static String EXPIRED_FEATURE_HEADER = "es.license.expired.feature"; + + /** + * Exception to be thrown when a feature action requires a valid license, but license + * has expired + * + * feature accessible through {@link #EXPIRED_FEATURE_HEADER} in the + * exception's rest header + */ + public static ElasticsearchSecurityException newComplianceException(String feature) { + ElasticsearchSecurityException e = new ElasticsearchSecurityException("current license is non-compliant for [{}]", RestStatus.UNAUTHORIZED, feature); + e.addHeader(EXPIRED_FEATURE_HEADER, feature); + return e; + } + + /** + * Checks if a given {@link ElasticsearchSecurityException} refers to a feature that + * requires a valid license, but the license has expired. + */ + public static boolean isLicenseExpiredException(ElasticsearchSecurityException exception) { + return (exception != null) && (exception.getHeader(EXPIRED_FEATURE_HEADER) != null); + } +} diff --git a/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/Licensee.java b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/Licensee.java new file mode 100644 index 00000000000..00d3bc6bef3 --- /dev/null +++ b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/Licensee.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.core.License.OperationMode; + +import java.util.Locale; + +public interface Licensee { + + /** + * Unique id used to log expiry and + * acknowledgment messages + */ + String id(); + + /** + * Messages to be printed when + * logging license expiry warnings + */ + String[] expirationMessages(); + + /** + * Messages to be returned when + * installing newLicense + * when oldLicense is + * active + */ + String[] acknowledgmentMessages(License currentLicense, License newLicense); + + /** + * Notifies when a new license is activated + * or when a license state change has occurred + */ + void onChange(Status status); + + class Status { + + public static Status ENABLED = new Status(OperationMode.TRIAL, LicenseState.ENABLED); + + private final OperationMode mode; + private final LicenseState licenseState; + + public Status(OperationMode mode, LicenseState licenseState) { + this.mode = mode; + this.licenseState = licenseState; + } + + /** + * Returns the operation mode of the license + * responsible for the current licenseState + */ + public OperationMode getMode() { + return mode; + } + + /** + * When a license is active, the state is + * {@link LicenseState#ENABLED}, upon license expiry + * the state changes to {@link LicenseState#GRACE_PERIOD} + * and after the grace period has ended the state changes + * to {@link LicenseState#DISABLED} + */ + public LicenseState getLicenseState() { + return licenseState; + } + + @Override + public String toString() { + if (mode == OperationMode.NONE) { + return "disabled"; + } + switch (licenseState) { + case DISABLED: + return "disabled " + mode.name().toLowerCase(Locale.ROOT); + case GRACE_PERIOD: + return mode.name().toLowerCase(Locale.ROOT) + " grace period"; + default: + return mode.name().toLowerCase(Locale.ROOT); + } + } + } +} diff --git a/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseeRegistry.java b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseeRegistry.java new file mode 100644 index 00000000000..53a855fa637 --- /dev/null +++ b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicenseeRegistry.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +public interface LicenseeRegistry { + + /** + * Registers a licensee for license notifications + */ + void register(Licensee licensee); +} diff --git a/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicensesManagerService.java b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicensesManagerService.java new file mode 100644 index 00000000000..f80d7dac03f --- /dev/null +++ b/elasticsearch/license/plugin-api/src/main/java/org/elasticsearch/license/plugin/core/LicensesManagerService.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.license.core.License; + +import java.util.List; + +public interface LicensesManagerService { + + /** + * @return the id of registered licensees currently in state + */ + List licenseesWithState(LicenseState state); + + /** + * @return the currently active license, or {@code null} if no license is currently installed + */ + License getLicense(); +} diff --git a/elasticsearch/license/plugin-api/src/test/java/org/elasticsearch/license/plugin/core/LicenseUtilsTests.java b/elasticsearch/license/plugin-api/src/test/java/org/elasticsearch/license/plugin/core/LicenseUtilsTests.java new file mode 100644 index 00000000000..4f7f526ca81 --- /dev/null +++ b/elasticsearch/license/plugin-api/src/test/java/org/elasticsearch/license/plugin/core/LicenseUtilsTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; + +import static org.hamcrest.Matchers.*; + +public class LicenseUtilsTests extends ESTestCase { + + public void testNewExpirationException() { + for (String feature : Arrays.asList("feature", randomAsciiOfLength(5), null, "")) { + ElasticsearchSecurityException exception = LicenseUtils.newComplianceException(feature); + assertNotNull(exception); + assertThat(exception.getHeaderKeys(), contains(LicenseUtils.EXPIRED_FEATURE_HEADER)); + assertThat(exception.getHeader(LicenseUtils.EXPIRED_FEATURE_HEADER), hasSize(1)); + assertThat(exception.getHeader(LicenseUtils.EXPIRED_FEATURE_HEADER).iterator().next(), equalTo(feature)); + } + } + + public void testIsLicenseExpiredException() { + ElasticsearchSecurityException exception = LicenseUtils.newComplianceException("feature"); + assertTrue(LicenseUtils.isLicenseExpiredException(exception)); + + exception = new ElasticsearchSecurityException("msg"); + assertFalse(LicenseUtils.isLicenseExpiredException(exception)); + } +} diff --git a/elasticsearch/license/plugin/.gitignore b/elasticsearch/license/plugin/.gitignore new file mode 100644 index 00000000000..ab956abf6f9 --- /dev/null +++ b/elasticsearch/license/plugin/.gitignore @@ -0,0 +1 @@ +/eclipse-build/ diff --git a/marvel/LICENSE.txt b/elasticsearch/license/plugin/LICENSE.txt similarity index 100% rename from marvel/LICENSE.txt rename to elasticsearch/license/plugin/LICENSE.txt diff --git a/marvel/NOTICE.txt b/elasticsearch/license/plugin/NOTICE.txt similarity index 100% rename from marvel/NOTICE.txt rename to elasticsearch/license/plugin/NOTICE.txt diff --git a/elasticsearch/license/plugin/build.gradle b/elasticsearch/license/plugin/build.gradle new file mode 100644 index 00000000000..da1bbed3223 --- /dev/null +++ b/elasticsearch/license/plugin/build.gradle @@ -0,0 +1,26 @@ +apply plugin: 'elasticsearch.esplugin' +esplugin { + name 'license' + description 'Internal Elasticsearch Licensing Plugin' + classname 'org.elasticsearch.license.plugin.LicensePlugin' + isolated false +} + +dependencies { + compile project(':x-plugins:elasticsearch:license:plugin-api') + testCompile project(':x-plugins:elasticsearch:license:licensor') +} + +dependencyLicenses.enabled = false + +processResources { + String licenseKeyName = System.getProperty('license.key', 'dev') + String licenseKeyPath = "keys/${licenseKeyName}/public.key" + if (file(licenseKeyPath).exists() == false) { + throw new GradleException("no public key found for '${licenseKeyName}'") + } + from licenseKeyPath +} + +compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked" +compileTestJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked" diff --git a/elasticsearch/license/plugin/keys/dev/public.key b/elasticsearch/license/plugin/keys/dev/public.key new file mode 100644 index 00000000000..2a9f272e0b3 --- /dev/null +++ b/elasticsearch/license/plugin/keys/dev/public.key @@ -0,0 +1,3 @@ +ýŽÇqÝnêÄÌgŠœwM}¹‡UiKŠ•0âbÖ2غqö]â쇴¯ÖÏÃcÌ+IÒðÔ &IJ†fÉ~ßlj ˆº]d™}o§Oè¾Id®È +5A(ìµ´^ØöW©DªJµë}ù-Oîë?u N5¾ÛvpÛ{’¼²Áô­œát–¤7ùøÃê #²Vqöó»ktwm’Œ]ÏLõ£z"| Q‹lŸòQðsâ>ù<}[Á2ÖÓàZÖ|5‹¯ÊÊ7%ØêD +Yå‘xn:¼lúLÈæHò¢«Ë2˜ŸHvEEWÇ\¦H:“6Žh9 [!š…Ûæ©Š¤+;Ö.w7Cì©_|Þ ÓªÏÁ*ñ§D`ƒÚ?‚ùxU/3>x­UÓ“+ è \ No newline at end of file diff --git a/elasticsearch/license/plugin/keys/prod/public.key b/elasticsearch/license/plugin/keys/prod/public.key new file mode 100644 index 00000000000..cf62043826c --- /dev/null +++ b/elasticsearch/license/plugin/keys/prod/public.key @@ -0,0 +1,3 @@ +ýŽÇqÝnêÄÌgŠœwM}¹‡UiKŠ•0âbÖ2ˆŒµô!Â[ÔCצ°ê4O9 š0'‰Q]b‰ÉဖÓ0ëèîqI´™0H7ÔÚaTuîl0ÿˆà5¿ø›&ŒÌjƒŒvÐ¥°[ä®]%Q'–Ì/‰Q;Qý` Ê[È1ûïX^sA€’v½îÿlñ<ÄàTÁÜ¡©ž±ùxh¢¡è÷²?1ì ÅàN3óÑ78ê'D["“Ö½¶GüY!:`j1YŸ…Gõyé@&I‚Ïü²b1Û¡re‰Ø£V!¸ÿæMÍô êÇÌ{ƒ³í[ÜMs“ ½M E?e…Sø¿TË +Y¬¥i:ÄR;«R2HäYkW}šcL׆8 +ÎÁfîs \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/LicenseModule.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/LicenseModule.java new file mode 100644 index 00000000000..b50a18a09ea --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/LicenseModule.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.license.core.LicenseVerifier; +import org.elasticsearch.license.plugin.core.LicenseeRegistry; +import org.elasticsearch.license.plugin.core.LicensesManagerService; +import org.elasticsearch.license.plugin.core.LicensesService; + +public class LicenseModule extends AbstractModule { + + @Override + protected void configure() { + bind(LicenseVerifier.class).asEagerSingleton(); + bind(LicensesService.class).asEagerSingleton(); + bind(LicenseeRegistry.class).to(LicensesService.class); + bind(LicensesManagerService.class).to(LicensesService.class); + } + +} \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/LicensePlugin.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/LicensePlugin.java new file mode 100644 index 00000000000..b4556f40fe1 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/LicensePlugin.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.action.ActionModule; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.component.LifecycleComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseAction; +import org.elasticsearch.license.plugin.action.delete.TransportDeleteLicenseAction; +import org.elasticsearch.license.plugin.action.get.GetLicenseAction; +import org.elasticsearch.license.plugin.action.get.TransportGetLicenseAction; +import org.elasticsearch.license.plugin.action.put.PutLicenseAction; +import org.elasticsearch.license.plugin.action.put.TransportPutLicenseAction; +import org.elasticsearch.license.plugin.core.LicensesMetaData; +import org.elasticsearch.license.plugin.core.LicensesService; +import org.elasticsearch.license.plugin.rest.RestDeleteLicenseAction; +import org.elasticsearch.license.plugin.rest.RestGetLicenseAction; +import org.elasticsearch.license.plugin.rest.RestPutLicenseAction; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestModule; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; + +public class LicensePlugin extends Plugin { + + public static final String NAME = "license"; + private final boolean isEnabled; + + static { + MetaData.registerPrototype(LicensesMetaData.TYPE, LicensesMetaData.PROTO); + } + + @Inject + public LicensePlugin(Settings settings) { + if (DiscoveryNode.clientNode(settings)) { + // Enable plugin only on node clients + this.isEnabled = "node".equals(settings.get(Client.CLIENT_TYPE_SETTING)); + } else { + this.isEnabled = true; + } + } + + @Override + public String name() { + return NAME; + } + + @Override + public String description() { + return "Internal Elasticsearch Licensing Plugin"; + } + + public void onModule(RestModule module) { + // Register REST endpoint + module.addRestAction(RestPutLicenseAction.class); + module.addRestAction(RestGetLicenseAction.class); + module.addRestAction(RestDeleteLicenseAction.class); + } + + public void onModule(ActionModule module) { + module.registerAction(PutLicenseAction.INSTANCE, TransportPutLicenseAction.class); + module.registerAction(GetLicenseAction.INSTANCE, TransportGetLicenseAction.class); + module.registerAction(DeleteLicenseAction.INSTANCE, TransportDeleteLicenseAction.class); + } + + @Override + public Collection> nodeServices() { + Collection> services = new ArrayList<>(); + if (isEnabled) { + services.add(LicensesService.class); + } + return services; + } + + + @Override + public Collection nodeModules() { + if (isEnabled) { + return Collections.singletonList(new LicenseModule()); + } + return Collections.emptyList(); + } +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseAction.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseAction.java new file mode 100644 index 00000000000..c7c8a675f84 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseAction.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.delete; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; + +public class DeleteLicenseAction extends Action { + + public static final DeleteLicenseAction INSTANCE = new DeleteLicenseAction(); + public static final String NAME = "cluster:admin/plugin/license/delete"; + + private DeleteLicenseAction() { + super(NAME); + } + + @Override + public DeleteLicenseResponse newResponse() { + return new DeleteLicenseResponse(); + } + + @Override + public DeleteLicenseRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new DeleteLicenseRequestBuilder(client, this); + } +} \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseRequest.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseRequest.java new file mode 100644 index 00000000000..acfc1e6a9bf --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseRequest.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.delete; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Set; + + +public class DeleteLicenseRequest extends AcknowledgedRequest { + + public DeleteLicenseRequest() { + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } +} \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseRequestBuilder.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseRequestBuilder.java new file mode 100644 index 00000000000..c210d54c13f --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseRequestBuilder.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.delete; + +import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; + +import java.util.Set; + +public class DeleteLicenseRequestBuilder extends AcknowledgedRequestBuilder { + + /** + * Creates new get licenses request builder + * + * @param client elasticsearch client + */ + public DeleteLicenseRequestBuilder(ElasticsearchClient client, DeleteLicenseAction action) { + super(client, action, new DeleteLicenseRequest()); + } +} \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseResponse.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseResponse.java new file mode 100644 index 00000000000..2b5d35edeae --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseResponse.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.delete; + +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class DeleteLicenseResponse extends AcknowledgedResponse { + + DeleteLicenseResponse() { + } + + DeleteLicenseResponse(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/TransportDeleteLicenseAction.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/TransportDeleteLicenseAction.java new file mode 100644 index 00000000000..a5c5594cb05 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/TransportDeleteLicenseAction.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.delete; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.core.LicensesManagerService; +import org.elasticsearch.license.plugin.core.LicensesService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +public class TransportDeleteLicenseAction extends TransportMasterNodeAction { + + private final LicensesService licensesService; + + @Inject + public TransportDeleteLicenseAction(Settings settings, TransportService transportService, ClusterService clusterService, LicensesService licensesService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, DeleteLicenseAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, DeleteLicenseRequest::new); + this.licensesService = licensesService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected DeleteLicenseResponse newResponse() { + return new DeleteLicenseResponse(); + } + + @Override + protected ClusterBlockException checkBlock(DeleteLicenseRequest request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected void masterOperation(final DeleteLicenseRequest request, ClusterState state, final ActionListener listener) throws ElasticsearchException { + licensesService.removeLicense(request, new ActionListener() { + @Override + public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) { + listener.onResponse(new DeleteLicenseResponse(clusterStateUpdateResponse.isAcknowledged())); + } + + @Override + public void onFailure(Throwable e) { + listener.onFailure(e); + } + }); + } +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseAction.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseAction.java new file mode 100644 index 00000000000..d426b1b8aab --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseAction.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.get; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; + +public class GetLicenseAction extends Action { + + public static final GetLicenseAction INSTANCE = new GetLicenseAction(); + public static final String NAME = "cluster:admin/plugin/license/get"; + + private GetLicenseAction() { + super(NAME); + } + + @Override + public GetLicenseResponse newResponse() { + return new GetLicenseResponse(); + } + + @Override + public GetLicenseRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new GetLicenseRequestBuilder(client, this); + } +} \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseRequest.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseRequest.java new file mode 100644 index 00000000000..7f8d20512a1 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseRequest.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.get; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; + + +public class GetLicenseRequest extends MasterNodeReadRequest { + + public GetLicenseRequest() { + } + + @Override + public ActionRequestValidationException validate() { + return null; + } +} \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseRequestBuilder.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseRequestBuilder.java new file mode 100644 index 00000000000..a837141e194 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseRequestBuilder.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.get; + +import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; + +public class GetLicenseRequestBuilder extends MasterNodeReadOperationRequestBuilder { + + /** + * Creates new get licenses request builder + * + * @param client elasticsearch client + */ + public GetLicenseRequestBuilder(ElasticsearchClient client, GetLicenseAction action) { + super(client, action, new GetLicenseRequest()); + } +} \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseResponse.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseResponse.java new file mode 100644 index 00000000000..95f4799e618 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseResponse.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.get; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.license.core.License; + +import java.io.IOException; + +public class GetLicenseResponse extends ActionResponse { + + private License license; + + GetLicenseResponse() { + } + + GetLicenseResponse(License license) { + this.license = license; + } + + public License license() { + return license; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + if (in.readBoolean()) { + license = License.readLicense(in); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + if (license == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + license.writeTo(out); + } + } + +} \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/TransportGetLicenseAction.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/TransportGetLicenseAction.java new file mode 100644 index 00000000000..3a9a0c24a0b --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/get/TransportGetLicenseAction.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.get; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.core.LicensesManagerService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +public class TransportGetLicenseAction extends TransportMasterNodeReadAction { + + private final LicensesManagerService licensesManagerService; + + @Inject + public TransportGetLicenseAction(Settings settings, TransportService transportService, ClusterService clusterService, LicensesManagerService licensesManagerService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, GetLicenseAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, GetLicenseRequest::new); + this.licensesManagerService = licensesManagerService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected GetLicenseResponse newResponse() { + return new GetLicenseResponse(); + } + + @Override + protected ClusterBlockException checkBlock(GetLicenseRequest request, ClusterState state) { + return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_READ, ""); + } + + @Override + protected void masterOperation(final GetLicenseRequest request, ClusterState state, final ActionListener listener) throws ElasticsearchException { + listener.onResponse(new GetLicenseResponse(licensesManagerService.getLicense())); + } +} \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseAction.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseAction.java new file mode 100644 index 00000000000..e90523b9c1d --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseAction.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.put; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; + +public class PutLicenseAction extends Action { + + public static final PutLicenseAction INSTANCE = new PutLicenseAction(); + public static final String NAME = "cluster:admin/plugin/license/put"; + + private PutLicenseAction() { + super(NAME); + } + + @Override + public PutLicenseResponse newResponse() { + return new PutLicenseResponse(); + } + + @Override + public PutLicenseRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new PutLicenseRequestBuilder(client, this); + } +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseRequest.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseRequest.java new file mode 100644 index 00000000000..7780908cce9 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseRequest.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.put; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ValidateActions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.license.core.License; + +import java.io.IOException; + + +public class PutLicenseRequest extends AcknowledgedRequest { + + private License license; + private boolean acknowledge = false; + + public PutLicenseRequest() { + } + + @Override + public ActionRequestValidationException validate() { + return (license == null) ? ValidateActions.addValidationError("license is missing", null) : null; + } + + /** + * Parses license from json format to an instance of {@link org.elasticsearch.license.core.License} + * + * @param licenseDefinition licenses definition + */ + public PutLicenseRequest license(String licenseDefinition) { + try { + return license(License.fromSource(licenseDefinition)); + } catch (IOException e) { + throw new IllegalArgumentException("failed to parse license source", e); + } + } + + public PutLicenseRequest license(License license) { + this.license = license; + return this; + } + + public License license() { + return license; + } + + public PutLicenseRequest acknowledge(boolean acknowledge) { + this.acknowledge = acknowledge; + return this; + } + + public boolean acknowledged() { + return acknowledge; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + license = License.readLicense(in); + acknowledge = in.readBoolean(); + readTimeout(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + license.writeTo(out); + out.writeBoolean(acknowledge); + writeTimeout(out); + } +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseRequestBuilder.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseRequestBuilder.java new file mode 100644 index 00000000000..b4a3a5e8499 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseRequestBuilder.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.put; + +import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.license.core.License; + +/** + * Register license request builder + */ +public class PutLicenseRequestBuilder extends AcknowledgedRequestBuilder { + + /** + * Constructs register license request + * + * @param client elasticsearch client + */ + public PutLicenseRequestBuilder(ElasticsearchClient client, PutLicenseAction action) { + super(client, action, new PutLicenseRequest()); + } + + /** + * Sets the license + * + * @param license license + * @return this builder + */ + public PutLicenseRequestBuilder setLicense(License license) { + request.license(license); + return this; + } + + public PutLicenseRequestBuilder setLicense(String licenseSource) { + request.license(licenseSource); + return this; + } + + public PutLicenseRequestBuilder setAcknowledge(boolean acknowledge) { + request.acknowledge(acknowledge); + return this; + } +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseResponse.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseResponse.java new file mode 100644 index 00000000000..0bd0839b5da --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseResponse.java @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.put; + +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.license.plugin.core.LicensesStatus; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PutLicenseResponse extends AcknowledgedResponse implements ToXContent { + + private LicensesStatus status; + private Map acknowledgeMessages; + private String acknowledgeHeader; + + PutLicenseResponse() { + } + + public PutLicenseResponse(boolean acknowledged, LicensesStatus status, String acknowledgeHeader, Map acknowledgeMessages) { + super(acknowledged); + this.status = status; + this.acknowledgeHeader = acknowledgeHeader; + this.acknowledgeMessages = acknowledgeMessages; + } + + public LicensesStatus status() { + return status; + } + + public Map acknowledgeMessages() { + return acknowledgeMessages; + } + + public String acknowledgeHeader() { + return acknowledgeHeader; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + status = LicensesStatus.fromId(in.readVInt()); + acknowledgeHeader = in.readOptionalString(); + int size = in.readVInt(); + Map acknowledgeMessages = new HashMap<>(size); + for (int i = 0; i < size; i++) { + String feature = in.readString(); + int nMessages = in.readVInt(); + String[] messages = new String[nMessages]; + for (int j = 0; j < nMessages; j++) { + messages[i] = in.readString(); + } + acknowledgeMessages.put(feature, messages); + } + this.acknowledgeMessages = acknowledgeMessages; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + out.writeVInt(status.id()); + out.writeOptionalString(acknowledgeHeader); + out.writeVInt(acknowledgeMessages.size()); + for (Map.Entry entry : acknowledgeMessages.entrySet()) { + out.writeString(entry.getKey()); + out.writeVInt(entry.getValue().length); + for (String message : entry.getValue()) { + out.writeString(message); + } + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("acknowledged", isAcknowledged()); + switch (status) { + case VALID: + builder.field("license_status", "valid"); + break; + case INVALID: + builder.field("license_status", "invalid"); + break; + case EXPIRED: + builder.field("license_status", "expired"); + break; + default: + throw new IllegalArgumentException("unknown status [" + status + "] found"); + } + if (!acknowledgeMessages.isEmpty()) { + builder.startObject("acknowledge"); + builder.field("message", acknowledgeHeader); + for (Map.Entry entry : acknowledgeMessages.entrySet()) { + builder.startArray(entry.getKey()); + for (String message : entry.getValue()) { + builder.value(message); + } + builder.endArray(); + } + builder.endObject(); + } + return builder; + } + + @Override + public String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); + builder.startObject(); + toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + return builder.string(); + } catch (IOException e) { + return "{ \"error\" : \"" + e.getMessage() + "\"}"; + } + } +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/TransportPutLicenseAction.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/TransportPutLicenseAction.java new file mode 100644 index 00000000000..8eda9c4ffb4 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/action/put/TransportPutLicenseAction.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.action.put; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.core.LicensesManagerService; +import org.elasticsearch.license.plugin.core.LicensesService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import static org.elasticsearch.license.plugin.core.LicensesService.LicensesUpdateResponse; + +public class TransportPutLicenseAction extends TransportMasterNodeAction { + + private final LicensesService licensesService; + + @Inject + public TransportPutLicenseAction(Settings settings, TransportService transportService, ClusterService clusterService, + LicensesService licensesService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, PutLicenseAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, PutLicenseRequest::new); + this.licensesService = licensesService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected PutLicenseResponse newResponse() { + return new PutLicenseResponse(); + } + + @Override + protected ClusterBlockException checkBlock(PutLicenseRequest request, ClusterState state) { + return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, ""); + } + + @Override + protected void masterOperation(final PutLicenseRequest request, ClusterState state, final ActionListener listener) throws ElasticsearchException { + licensesService.registerLicense(request, new ActionListener() { + @Override + public void onResponse(LicensesUpdateResponse licensesUpdateResponse) { + listener.onResponse(new PutLicenseResponse(licensesUpdateResponse.isAcknowledged(), licensesUpdateResponse.status(), + licensesUpdateResponse.acknowledgementHeader(), licensesUpdateResponse.acknowledgeMessages())); + } + + @Override + public void onFailure(Throwable e) { + listener.onFailure(e); + } + }); + } + +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesMetaData.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesMetaData.java new file mode 100644 index 00000000000..03c58bcdf54 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesMetaData.java @@ -0,0 +1,237 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.Base64; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.license.core.License; + +import java.io.IOException; +import java.util.*; + +import static org.elasticsearch.license.core.CryptUtils.decrypt; +import static org.elasticsearch.license.core.CryptUtils.encrypt; + +/** + * Contains metadata about registered licenses + */ +public class LicensesMetaData extends AbstractDiffable implements MetaData.Custom { + + public static final String TYPE = "licenses"; + + /** + * When license is explicitly removed by a user, LICENSE_TOMBSTONE + * is used as a placeholder in the license metadata. This enables + * us to distinguish between the scenario when a cluster never + * had a license (null) and when a license was removed explicitly + * (LICENSE_TOMBSTONE). + * We rely on this to decide whether to generate a unsigned trial + * license or not. we should only generate a license if no license + * ever existed in the cluster state + */ + public static final License LICENSE_TOMBSTONE = License.builder() + .type("trial") + .issuer("elasticsearch") + .uid("TOMBSTONE") + .issuedTo("") + .maxNodes(0) + .issueDate(0) + .expiryDate(0) + .build(); + + public static final LicensesMetaData PROTO = new LicensesMetaData(null); + + private License license; + + public LicensesMetaData(License license) { + this.license = license; + } + + public License getLicense() { + return license; + } + + @Override + public String toString() { + if (license != null) { + return license.toString(); + } + return ""; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LicensesMetaData that = (LicensesMetaData) o; + return !(license != null ? !license.equals(that.license) : that.license != null); + } + + @Override + public int hashCode() { + return license != null ? license.hashCode() : 0; + } + + @Override + public String type() { + return TYPE; + } + + @Override + public EnumSet context() { + return EnumSet.of(MetaData.XContentContext.GATEWAY); + } + + @Override + public LicensesMetaData fromXContent(XContentParser parser) throws IOException { + List pre20Licenses = new ArrayList<>(1); + License license = LICENSE_TOMBSTONE; + while (parser.currentToken() != XContentParser.Token.END_OBJECT) { + XContentParser.Token token = parser.nextToken(); + if (token == XContentParser.Token.FIELD_NAME) { + String fieldName = parser.text(); + if (fieldName != null) { + // for back compat with 1.x license metadata + if (fieldName.equals(Fields.TRIAL_LICENSES) || fieldName.equals(Fields.SIGNED_LICENCES)) { + token = parser.nextToken(); + if (token == XContentParser.Token.START_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + if (parser.currentToken().isValue()) { + // trial license + byte[] data = decrypt(Base64.decode(parser.text())); + try (XContentParser trialLicenseParser = XContentFactory.xContent(XContentType.JSON).createParser(data)) { + trialLicenseParser.nextToken(); + License pre20TrialLicense = License.fromXContent(trialLicenseParser); + pre20Licenses.add(TrialLicense.create(License.builder().fromPre20LicenseSpec(pre20TrialLicense))); + } + } else { + // signed license + pre20Licenses.add(License.fromXContent(parser)); + } + } + } + } else if (fieldName.equals(Fields.LICENSE)) { + token = parser.nextToken(); + if (token == XContentParser.Token.START_OBJECT) { + license = License.fromXContent(parser); + } else if (token == XContentParser.Token.VALUE_NULL) { + license = LICENSE_TOMBSTONE; + } + } + } + } + } + // when we see old license metadata, + // we try to choose the license that has the latest issue date that is not expired + if (!pre20Licenses.isEmpty()) { + // take the best unexpired license + CollectionUtil.timSort(pre20Licenses, License.LATEST_ISSUE_DATE_FIRST); + long now = System.currentTimeMillis(); + for (License oldLicense : pre20Licenses) { + if (oldLicense.expiryDate() > now) { + license = oldLicense; + break; + } + } + // take the best expired license + if (license == LICENSE_TOMBSTONE && !pre20Licenses.isEmpty()) { + license = pre20Licenses.get(0); + } + } + return new LicensesMetaData(license); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (license == LICENSE_TOMBSTONE) { + builder.nullField(Fields.LICENSE); + } else { + builder.startObject(Fields.LICENSE); + license.toInnerXContent(builder, params); + builder.endObject(); + } + return builder; + } + + @Override + public void writeTo(StreamOutput streamOutput) throws IOException { + if (streamOutput.getVersion().before(Version.V_2_0_0)) { + if (license == LICENSE_TOMBSTONE) { + streamOutput.writeVInt(0); // no signed license + streamOutput.writeVInt(0); // no trial license + } else if (!License.isAutoGeneratedLicense(license.signature())) { + streamOutput.writeVInt(1); // one signed license + license.writeTo(streamOutput); + streamOutput.writeVInt(0); // no trial license + } else { + streamOutput.writeVInt(0); // no signed license + streamOutput.writeVInt(1); // one trial license + XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + license.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true"))); + streamOutput.writeString(Base64.encodeBytes(encrypt(contentBuilder.bytes().toBytes()))); + } + } else { + if (license == LICENSE_TOMBSTONE) { + streamOutput.writeBoolean(false); // no license + } else { + streamOutput.writeBoolean(true); // has a license + license.writeTo(streamOutput); + } + } + } + + @Override + public LicensesMetaData readFrom(StreamInput streamInput) throws IOException { + License license = LICENSE_TOMBSTONE; + if (streamInput.getVersion().before(Version.V_2_0_0)) { + int size = streamInput.readVInt(); + List licenses = new ArrayList<>(); + for (int i = 0; i < size; i++) { + licenses.add(License.readLicense(streamInput)); + } + int numTrialLicenses = streamInput.readVInt(); + for (int i = 0; i < numTrialLicenses; i++) { + byte[] data = decrypt(Base64.decode(streamInput.readString())); + try (XContentParser trialLicenseParser = XContentFactory.xContent(XContentType.JSON).createParser(data)) { + trialLicenseParser.nextToken(); + License pre20TrialLicense = License.fromXContent(trialLicenseParser); + licenses.add(TrialLicense.create(License.builder().fromPre20LicenseSpec(pre20TrialLicense))); + } + } + // when we see read licenses from old pre v2.0, + // we try to choose the license that has the latest issue date that is not expired + CollectionUtil.timSort(licenses, License.LATEST_ISSUE_DATE_FIRST); + long now = System.currentTimeMillis(); + for (License oldLicense : licenses) { + if (oldLicense.expiryDate() > now) { + license = oldLicense; + break; + } + } + // take the best expired license + if (license == LICENSE_TOMBSTONE && !licenses.isEmpty()) { + license = licenses.get(0); + } + } else { + if (streamInput.readBoolean()) { + license = License.readLicense(streamInput); + } + } + return new LicensesMetaData(license); + } + + private final static class Fields { + private static final String SIGNED_LICENCES = "signed_licenses"; + private static final String TRIAL_LICENSES = "trial_licenses"; + private static final String LICENSE = "license"; + } +} \ No newline at end of file diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesService.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesService.java new file mode 100644 index 00000000000..95bde3a6d86 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesService.java @@ -0,0 +1,845 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.*; +import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.Singleton; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.joda.FormatDateTimeFormatter; +import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.logging.support.LoggerMessageFormat; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.util.concurrent.FutureUtils; +import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.core.LicenseVerifier; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseRequest; +import org.elasticsearch.license.plugin.action.put.PutLicenseRequest; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.*; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Service responsible for managing {@link LicensesMetaData} + * Interfaces through which this is exposed are: + * - LicensesManagerService - responsible for managing signed and one-time-trial licenses + * - LicensesClientService - responsible for listener registration of consumer plugin(s) + *

+ * Registration Scheme: + *

+ * A consumer plugin is registered with {@link LicenseeRegistry#register(Licensee)} + * This method can be called at any time during the life-cycle of the consumer plugin. + * If the listener can not be registered immediately, it is queued up and registered on the first clusterChanged event with + * no {@link org.elasticsearch.gateway.GatewayService#STATE_NOT_RECOVERED_BLOCK} block + * Upon successful registration, the listeners are notified appropriately using the notification scheme + *

+ * Notification Scheme: + *

+ * All registered listeners are notified of the current license upon registration or when a new license is installed in the cluster state. + * When a new license is notified as enabled to the registered listener, a notification is scheduled at the time of license expiry. + * Registered listeners are notified using {@link #notifyAndSchedule(LicensesMetaData)} + */ +@Singleton +public class LicensesService extends AbstractLifecycleComponent implements ClusterStateListener, LicensesManagerService, LicenseeRegistry { + + public static final String REGISTER_TRIAL_LICENSE_ACTION_NAME = "internal:plugin/license/cluster/register_trial_license"; + + private final ClusterService clusterService; + + private final ThreadPool threadPool; + + private final TransportService transportService; + + /** + * Currently active consumers to notify to + */ + private final List registeredLicensees = new CopyOnWriteArrayList<>(); + + /** + * Currently active expiry notifications + */ + private final Queue expiryNotifications = new ConcurrentLinkedQueue<>(); + + /** + * Currently active event notifications for every registered listener + */ + private final Queue eventNotifications = new ConcurrentLinkedQueue<>(); + + /** + * Currently active license + */ + private final AtomicReference currentLicense = new AtomicReference<>(); + + /** + * Callbacks to notify relative to license expiry + */ + private List expirationCallbacks = new ArrayList<>(); + + /** + * Duration of generated trial license + */ + private TimeValue trialLicenseDuration = TimeValue.timeValueHours(30 * 24); + + /** + * Max number of nodes licensed by generated trial license + */ + private int trialLicenseMaxNodes = 1000; + + /** + * Duration of grace period after a license has expired + */ + private TimeValue gracePeriodDuration = days(7); + + private static final FormatDateTimeFormatter DATE_FORMATTER = Joda.forPattern("EEEE, MMMMM dd, yyyy", Locale.ROOT); + + private static final String ACKNOWLEDGEMENT_HEADER = "This license update requires acknowledgement. To acknowledge the license, please read the following messages and update the license again, this time with the \"acknowledge=true\" parameter:"; + + @Inject + public LicensesService(Settings settings, ClusterService clusterService, ThreadPool threadPool, TransportService transportService) { + super(settings); + this.clusterService = clusterService; + this.threadPool = threadPool; + this.transportService = transportService; + if (DiscoveryNode.masterNode(settings)) { + transportService.registerRequestHandler(REGISTER_TRIAL_LICENSE_ACTION_NAME, TransportRequest.Empty::new, + ThreadPool.Names.SAME, new RegisterTrialLicenseRequestHandler()); + } + populateExpirationCallbacks(); + } + + private void populateExpirationCallbacks() { + expirationCallbacks.add(new ExpirationCallback.Pre(days(7), days(30), days(1)) { + @Override + public void on(License license) { + String general = LoggerMessageFormat.format(null, "\n" + + "#\n" + + "# License will expire on [{}]. If you have a new license, please update it.\n" + + "# Otherwise, please reach out to your support contact.\n" + + "# ", DATE_FORMATTER.printer().print(license.expiryDate())); + if (!registeredLicensees.isEmpty()) { + StringBuilder builder = new StringBuilder(general); + builder.append(System.lineSeparator()); + builder.append("# Commercial plugins operate with reduced functionality on license expiration:"); + for (InternalLicensee licensee : registeredLicensees) { + if (licensee.expirationMessages().length > 0) { + builder.append(System.lineSeparator()); + builder.append("# - "); + builder.append(licensee.id()); + for (String message : licensee.expirationMessages()) { + builder.append(System.lineSeparator()); + builder.append("# - "); + builder.append(message); + } + } + } + logger.error(builder.toString()); + } else { + logger.error(general); + } + } + } + ); + expirationCallbacks.add(new ExpirationCallback.Pre(days(0), days(7), TimeValue.timeValueMinutes(10)) { + @Override + public void on(License license) { + String general = LoggerMessageFormat.format(null, "\n" + + "#\n" + + "# License will expire on [{}]. If you have a new license, please update it.\n" + + "# Otherwise, please reach out to your support contact.\n" + + "# ", DATE_FORMATTER.printer().print(license.expiryDate())); + if (!registeredLicensees.isEmpty()) { + StringBuilder builder = new StringBuilder(general); + builder.append(System.lineSeparator()); + builder.append("# Commercial plugins operate with reduced functionality on license expiration:"); + for (InternalLicensee licensee : registeredLicensees) { + if (licensee.expirationMessages().length > 0) { + builder.append(System.lineSeparator()); + builder.append("# - "); + builder.append(licensee.id()); + for (String message : licensee.expirationMessages()) { + builder.append(System.lineSeparator()); + builder.append("# - "); + builder.append(message); + } + } + } + logger.error(builder.toString()); + } else { + logger.error(general); + } + } + } + ); + expirationCallbacks.add(new ExpirationCallback.Post(days(0), null, TimeValue.timeValueMinutes(10)) { + @Override + public void on(License license) { + // logged when grace period begins + String general = LoggerMessageFormat.format(null, "\n" + + "#\n" + + "# LICENSE EXPIRED ON [{}]. IF YOU HAVE A NEW LICENSE, PLEASE\n" + + "# UPDATE IT. OTHERWISE, PLEASE REACH OUT TO YOUR SUPPORT CONTACT.\n" + + "# ", DATE_FORMATTER.printer().print(license.expiryDate())); + if (!registeredLicensees.isEmpty()) { + StringBuilder builder = new StringBuilder(general); + builder.append(System.lineSeparator()); + builder.append("# COMMERCIAL PLUGINS OPERATING WITH REDUCED FUNCTIONALITY"); + for (InternalLicensee licensee : registeredLicensees) { + if (licensee.expirationMessages().length > 0) { + builder.append(System.lineSeparator()); + builder.append("# - "); + builder.append(licensee.id()); + for (String message : licensee.expirationMessages()) { + builder.append(System.lineSeparator()); + builder.append("# - "); + builder.append(message); + } + } + } + logger.error(builder.toString()); + } else { + logger.error(general); + } + } + } + ); + } + + /** + * Registers new license in the cluster + * Master only operation. Installs a new license on the master provided it is VALID + */ + public void registerLicense(final PutLicenseRequest request, final ActionListener listener) { + final License newLicense = request.license(); + final long now = System.currentTimeMillis(); + if (!verifyLicense(newLicense) || newLicense.issueDate() > now) { + listener.onResponse(new LicensesUpdateResponse(true, LicensesStatus.INVALID)); + } else if (newLicense.expiryDate() < now) { + listener.onResponse(new LicensesUpdateResponse(true, LicensesStatus.EXPIRED)); + } else { + if (!request.acknowledged()) { + final LicensesMetaData currentMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE); + final License currentLicense = getLicense(currentMetaData); + Map acknowledgeMessages = new HashMap<>(registeredLicensees.size() + 1); + if (currentLicense != null && !License.isAutoGeneratedLicense(currentLicense.signature()) // when current license is not an auto-generated license + && currentLicense.issueDate() > newLicense.issueDate()) { // and has a later issue date + acknowledgeMessages.put("license", + new String[] { "The new license is older than the currently installed license. Are you sure you want to override the current license?" }); + } + for (InternalLicensee licensee : registeredLicensees) { + String[] listenerAcknowledgeMessages = licensee.acknowledgmentMessages(currentLicense, newLicense); + if (listenerAcknowledgeMessages.length > 0) { + acknowledgeMessages.put(licensee.id(), listenerAcknowledgeMessages); + } + } + if (!acknowledgeMessages.isEmpty()) { + // needs acknowledgement + listener.onResponse(new LicensesUpdateResponse(false, LicensesStatus.VALID, ACKNOWLEDGEMENT_HEADER, acknowledgeMessages)); + return; + } + } + clusterService.submitStateUpdateTask("register license [" + newLicense.uid() + "]", new AckedClusterStateUpdateTask(request, listener) { + @Override + protected LicensesUpdateResponse newResponse(boolean acknowledged) { + return new LicensesUpdateResponse(acknowledged, LicensesStatus.VALID); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); + mdBuilder.putCustom(LicensesMetaData.TYPE, new LicensesMetaData(newLicense)); + return ClusterState.builder(currentState).metaData(mdBuilder).build(); + } + }); + } + } + + private boolean verifyLicense(final License license) { + final byte[] publicKeyBytes; + try (InputStream is = LicensesService.class.getResourceAsStream("/public.key")) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(is, out); + publicKeyBytes = out.toByteArray(); + } catch (IOException ex) { + throw new IllegalStateException(ex); + } + return LicenseVerifier.verifyLicense(license, publicKeyBytes); + } + + static TimeValue days(int days) { + return TimeValue.timeValueHours(days * 24); + } + + public static class LicensesUpdateResponse extends ClusterStateUpdateResponse { + private final LicensesStatus status; + private final String acknowledgementHeader; + private final Map acknowledgeMessages; + + public LicensesUpdateResponse(boolean acknowledged, LicensesStatus status) { + this(acknowledged, status, null, Collections.emptyMap()); + } + + public LicensesUpdateResponse(boolean acknowledged, LicensesStatus status, String acknowledgementHeader, Map acknowledgeMessages) { + super(acknowledged); + this.status = status; + this.acknowledgeMessages = acknowledgeMessages; + this.acknowledgementHeader = acknowledgementHeader; + } + + public LicensesStatus status() { + return status; + } + + public String acknowledgementHeader() { + return acknowledgementHeader; + } + + public Map acknowledgeMessages() { + return acknowledgeMessages; + } + } + + /** + * Remove license from the cluster state metadata + */ + public void removeLicense(final DeleteLicenseRequest request, final ActionListener listener) { + clusterService.submitStateUpdateTask("delete license", new AckedClusterStateUpdateTask(request, listener) { + @Override + protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { + return new ClusterStateUpdateResponse(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + MetaData metaData = currentState.metaData(); + final LicensesMetaData currentLicenses = metaData.custom(LicensesMetaData.TYPE); + if (currentLicenses.getLicense() != LicensesMetaData.LICENSE_TOMBSTONE) { + MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); + mdBuilder.putCustom(LicensesMetaData.TYPE, new LicensesMetaData(LicensesMetaData.LICENSE_TOMBSTONE)); + return ClusterState.builder(currentState).metaData(mdBuilder).build(); + } else { + return currentState; + } + } + }); + } + + @Override + public List licenseesWithState(LicenseState state) { + List licensees = new ArrayList<>(registeredLicensees.size()); + for (InternalLicensee licensee : registeredLicensees) { + if (licensee.currentLicenseState == state) { + licensees.add(licensee.id()); + } + } + return licensees; + } + + @Override + public License getLicense() { + final LicensesMetaData metaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE); + return getLicense(metaData); + } + + /** + * Master-only operation to generate a one-time global trial license. + * The trial license is only generated and stored if the current cluster state metaData + * has no signed/trial license + */ + private void registerTrialLicense() { + clusterService.submitStateUpdateTask("generate trial license for [" + trialLicenseDuration + "]", new ClusterStateUpdateTask() { + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + LicensesMetaData licensesMetaData = newState.metaData().custom(LicensesMetaData.TYPE); + if (logger.isDebugEnabled()) { + logger.debug("registered trial license", licensesMetaData); + } + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + final MetaData metaData = currentState.metaData(); + final LicensesMetaData currentLicensesMetaData = metaData.custom(LicensesMetaData.TYPE); + MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); + // do not generate a trial license if any license is present + if (currentLicensesMetaData == null) { + long issueDate = System.currentTimeMillis(); + License.Builder specBuilder = License.builder() + .uid(UUID.randomUUID().toString()) + .issuedTo(clusterService.state().getClusterName().value()) + .maxNodes(trialLicenseMaxNodes) + .issueDate(issueDate) + .expiryDate(issueDate + trialLicenseDuration.getMillis()); + License trialLicense = TrialLicense.create(specBuilder); + mdBuilder.putCustom(LicensesMetaData.TYPE, new LicensesMetaData(trialLicense)); + return ClusterState.builder(currentState).metaData(mdBuilder).build(); + } + return currentState; + } + + @Override + public void onFailure(String source, @Nullable Throwable t) { + logger.error("unexpected failure during [{}]", t, source); + } + + }); + } + + @Override + protected void doStart() throws ElasticsearchException { + clusterService.add(this); + } + + @Override + protected void doStop() throws ElasticsearchException { + clusterService.remove(this); + + // cancel all notifications + for (ScheduledFuture scheduledNotification : expiryNotifications) { + FutureUtils.cancel(scheduledNotification); + } + for (ScheduledFuture eventNotification : eventNotifications) { + FutureUtils.cancel(eventNotification); + } + + // clear all handlers + registeredLicensees.clear(); + + // empty out notification queue + expiryNotifications.clear(); + + // clear current license + currentLicense.set(null); + } + + @Override + protected void doClose() throws ElasticsearchException { + transportService.removeHandler(REGISTER_TRIAL_LICENSE_ACTION_NAME); + } + + /** + * When there is no global block on {@link org.elasticsearch.gateway.GatewayService#STATE_NOT_RECOVERED_BLOCK} + * notify licensees and issue auto-generated license if no license has been installed/issued yet. + */ + @Override + public void clusterChanged(ClusterChangedEvent event) { + final ClusterState previousClusterState = event.previousState(); + final ClusterState currentClusterState = event.state(); + if (!currentClusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + final LicensesMetaData prevLicensesMetaData = previousClusterState.getMetaData().custom(LicensesMetaData.TYPE); + final LicensesMetaData currentLicensesMetaData = currentClusterState.getMetaData().custom(LicensesMetaData.TYPE); + if (logger.isDebugEnabled()) { + logger.debug("previous [{}]", prevLicensesMetaData); + logger.debug("current [{}]", currentLicensesMetaData); + } + // notify all interested plugins + if (previousClusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + notifyAndSchedule(currentLicensesMetaData); + } else { + if (prevLicensesMetaData == null) { + if (currentLicensesMetaData != null) { + notifyAndSchedule(currentLicensesMetaData); + } + } else if (!prevLicensesMetaData.equals(currentLicensesMetaData)) { + notifyAndSchedule(currentLicensesMetaData); + } + } + // auto-generate license if no licenses ever existed + // this will trigger a subsequent cluster changed event + if (prevLicensesMetaData == null + && (currentLicensesMetaData == null || currentLicensesMetaData.getLicense() == null)) { + requestTrialLicense(currentClusterState); + } + } else if (logger.isDebugEnabled()) { + logger.debug("skipped license notifications reason: [{}]", GatewayService.STATE_NOT_RECOVERED_BLOCK); + } + } + + /** + * Notifies registered licensees of license state change and/or new active license + * based on the license in currentLicensesMetaData. + * Additionally schedules license expiry notifications and event callbacks + * relative to the current license's expiry + */ + private void notifyAndSchedule(final LicensesMetaData currentLicensesMetaData) { + final License license = getLicense(currentLicensesMetaData); + if (license != null) { + logger.debug("notifying [{}] listeners", registeredLicensees.size()); + long now = System.currentTimeMillis(); + if (license.issueDate() > now) { + logger.info("license [{}] - invalid", license.uid()); + return; + } + long expiryDuration = license.expiryDate() - now; + if (license.expiryDate() > now) { + for (InternalLicensee licensee : registeredLicensees) { + licensee.onChange(license, LicenseState.ENABLED); + } + logger.info("license [{}] - valid", license.uid()); + final TimeValue delay = TimeValue.timeValueMillis(expiryDuration); + // cancel any previous notifications + cancelNotifications(expiryNotifications); + try { + logger.debug("schedule grace notification after [{}] for license [{}]", delay.toString(), license.uid()); + expiryNotifications.add(threadPool.schedule(delay, executorName(), new LicensingClientNotificationJob())); + } catch (EsRejectedExecutionException ex) { + logger.debug("couldn't schedule grace notification", ex); + } + } else if ((license.expiryDate() + gracePeriodDuration.getMillis()) > now) { + for (InternalLicensee licensee : registeredLicensees) { + licensee.onChange(license, LicenseState.GRACE_PERIOD); + } + logger.info("license [{}] - grace", license.uid()); + final TimeValue delay = TimeValue.timeValueMillis(expiryDuration + gracePeriodDuration.getMillis()); + // cancel any previous notifications + cancelNotifications(expiryNotifications); + try { + logger.debug("schedule expiry notification after [{}] for license [{}]", delay.toString(), license.uid()); + expiryNotifications.add(threadPool.schedule(delay, executorName(), new LicensingClientNotificationJob())); + } catch (EsRejectedExecutionException ex) { + logger.debug("couldn't schedule expiry notification", ex); + } + } else { + for (InternalLicensee licensee : registeredLicensees) { + licensee.onChange(license, LicenseState.DISABLED); + } + logger.info("license [{}] - expired", license.uid()); + } + if (!license.equals(currentLicense.get())) { + currentLicense.set(license); + // cancel all scheduled event notifications + cancelNotifications(eventNotifications); + // schedule expiry callbacks + for (ExpirationCallback expirationCallback : this.expirationCallbacks) { + final TimeValue delay; + if (expirationCallback.matches(license.expiryDate(), now)) { + expirationCallback.on(license); + TimeValue frequency = expirationCallback.frequency(); + delay = frequency != null ? frequency : expirationCallback.delay(expiryDuration); + } else { + delay = expirationCallback.delay(expiryDuration); + } + if (delay != null) { + eventNotifications.add(threadPool.schedule(delay, executorName(), new EventNotificationJob(expirationCallback))); + } + if (logger.isDebugEnabled()) { + logger.debug("schedule [{}] after [{}]", expirationCallback, delay); + } + } + logger.debug("scheduled expiry callbacks for [{}] expiring after [{}]", license.uid(), TimeValue.timeValueMillis(expiryDuration)); + } + } + } + + private class LicensingClientNotificationJob implements Runnable { + @Override + public void run() { + logger.debug("running expiry notification"); + final ClusterState currentClusterState = clusterService.state(); + if (!currentClusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + final LicensesMetaData currentLicensesMetaData = currentClusterState.metaData().custom(LicensesMetaData.TYPE); + notifyAndSchedule(currentLicensesMetaData); + } else if (logger.isDebugEnabled()) { + // next clusterChanged event will deal with the missed notifications + logger.debug("skip expiry notification [{}]", GatewayService.STATE_NOT_RECOVERED_BLOCK); + } + } + } + + private class EventNotificationJob implements Runnable { + private final ExpirationCallback expirationCallback; + + EventNotificationJob(ExpirationCallback expirationCallback) { + this.expirationCallback = expirationCallback; + } + + @Override + public void run() { + logger.debug("running event notification for [{}]", expirationCallback); + LicensesMetaData currentLicensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE); + License license = getLicense(currentLicensesMetaData); + if (license != null) { + long now = System.currentTimeMillis(); + if (expirationCallback.matches(license.expiryDate(), now)) { + expirationCallback.on(license); + if (expirationCallback.frequency() != null) { + // schedule next event + eventNotifications.add(threadPool.schedule(expirationCallback.frequency(), executorName(), this)); + } + } else if (logger.isDebugEnabled()) { + logger.debug("skip scheduling notification for [{}] with license expiring after [{}]", expirationCallback, + TimeValue.timeValueMillis(license.expiryDate() - now)); + } + } + // clear out any finished event notifications + while (!eventNotifications.isEmpty()) { + ScheduledFuture notification = eventNotifications.peek(); + if (notification != null && notification.isDone()) { + // remove the notifications that are done + eventNotifications.poll(); + } else { + // stop emptying out the queue as soon as the first undone future hits + break; + } + } + } + } + + public static abstract class ExpirationCallback { + + public enum Orientation { PRE, POST } + + public static abstract class Pre extends ExpirationCallback { + + /** + * Callback schedule prior to license expiry + * + * @param min latest relative time to execute before license expiry + * @param max earliest relative time to execute before license expiry + * @param frequency interval between execution + */ + public Pre(TimeValue min, TimeValue max, TimeValue frequency) { + super(Orientation.PRE, min, max, frequency); + } + + @Override + public boolean matches(long expirationDate, long now) { + long expiryDuration = expirationDate - now; + if (expiryDuration > 0l) { + if (expiryDuration <= max.getMillis()) { + return expiryDuration >= min.getMillis(); + } + } + return false; + } + + @Override + public TimeValue delay(long expiryDuration) { + return TimeValue.timeValueMillis(expiryDuration - max.getMillis()); + } + } + + public static abstract class Post extends ExpirationCallback { + + /** + * Callback schedule after license expiry + * + * @param min earliest relative time to execute after license expiry + * @param max latest relative time to execute after license expiry + * @param frequency interval between execution + */ + public Post(TimeValue min, TimeValue max, TimeValue frequency) { + super(Orientation.POST, min, max, frequency); + } + + @Override + public boolean matches(long expirationDate, long now) { + long postExpiryDuration = now - expirationDate; + if (postExpiryDuration > 0l) { + if (postExpiryDuration <= max.getMillis()) { + return postExpiryDuration >= min.getMillis(); + } + } + return false; + } + + @Override + public TimeValue delay(long expiryDuration) { + final long delay; + if (expiryDuration >= 0l) { + delay = expiryDuration + min.getMillis(); + } else { + delay = (-1l * expiryDuration) - min.getMillis(); + } + if (delay > 0l) { + return TimeValue.timeValueMillis(delay); + } else { + return null; + } + } + } + + protected final Orientation orientation; + protected final TimeValue min; + protected final TimeValue max; + private final TimeValue frequency; + + private ExpirationCallback(Orientation orientation, TimeValue min, TimeValue max, TimeValue frequency) { + this.orientation = orientation; + this.min = (min == null) ? TimeValue.timeValueMillis(0) : min; + this.max = (max == null) ? TimeValue.timeValueMillis(Long.MAX_VALUE) : max; + this.frequency = frequency; + } + + public TimeValue frequency() { + return frequency; + } + + public abstract TimeValue delay(long expiryDuration); + + public abstract boolean matches(long expirationDate, long now); + + public abstract void on(License license); + + @Override + public String toString() { + return LoggerMessageFormat.format(null, "ExpirationCallback:(orientation [{}], min [{}], max [{}], freq [{}])", orientation.name(), min, max, frequency); + } + } + + @Override + public void register(Licensee licensee) { + for (final InternalLicensee existingLicensee : registeredLicensees) { + if (existingLicensee.id().equals(licensee.id())) { + throw new IllegalStateException("listener: [" + licensee.id() + "] has been already registered"); + } + } + logger.debug("registering licensee [{}]", licensee.id()); + registeredLicensees.add(new InternalLicensee(licensee)); + final ClusterState clusterState = clusterService.state(); + if (clusterService.lifecycleState() == Lifecycle.State.STARTED + && clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK) == false + && clusterState.nodes().masterNode() != null) { + final LicensesMetaData currentMetaData = clusterState.metaData().custom(LicensesMetaData.TYPE); + if (currentMetaData == null || currentMetaData.getLicense() == null) { + // triggers a cluster changed event + // eventually notifying the current licensee + requestTrialLicense(clusterState); + } else { + notifyAndSchedule(currentMetaData); + } + } + } + + private void requestTrialLicense(final ClusterState currentState) { + DiscoveryNode masterNode = currentState.nodes().masterNode(); + if (masterNode == null) { + throw new IllegalStateException("master not available when registering auto-generated license"); + } + transportService.sendRequest(masterNode, + REGISTER_TRIAL_LICENSE_ACTION_NAME, TransportRequest.Empty.INSTANCE, EmptyTransportResponseHandler.INSTANCE_SAME); + } + + public License getLicense(final LicensesMetaData metaData) { + if (metaData != null) { + License license = metaData.getLicense(); + if (license != LicensesMetaData.LICENSE_TOMBSTONE) { + boolean autoGeneratedLicense = License.isAutoGeneratedLicense(license.signature()); + if ((autoGeneratedLicense && TrialLicense.verify(license)) + || (!autoGeneratedLicense && verifyLicense(license))) { + return license; + } + } + } + return null; + } + + /** + * Cancels out all notification futures + */ + private static void cancelNotifications(Queue scheduledNotifications) { + // clear out notification queue + while (!scheduledNotifications.isEmpty()) { + ScheduledFuture notification = scheduledNotifications.peek(); + if (notification != null) { + // cancel + FutureUtils.cancel(notification); + scheduledNotifications.poll(); + } + } + } + + private String executorName() { + return ThreadPool.Names.GENERIC; + } + + /** + * Stores acknowledgement, expiration and license notification callbacks + * for a registered listener + */ + private class InternalLicensee { + volatile License currentLicense = null; + volatile LicenseState currentLicenseState = LicenseState.DISABLED; + private final Licensee licensee; + + private InternalLicensee(Licensee licensee) { + this.licensee = licensee; + } + + @Override + public String toString() { + return "(listener: " + licensee.id() + ", state: " + currentLicenseState.name() + ")"; + } + + public String id() { + return licensee.id(); + } + + public String[] expirationMessages() { + return licensee.expirationMessages(); + } + + public String[] acknowledgmentMessages(License currentLicense, License newLicense) { + return licensee.acknowledgmentMessages(currentLicense, newLicense); + } + + public void onChange(License license, LicenseState state) { + synchronized (this) { + if (currentLicense == null // not yet initialized + || !currentLicense.equals(license) // current license has changed + || currentLicenseState != state) { // same license but state has changed + logger.debug("licensee [{}] notified", licensee.id()); + licensee.onChange(new Licensee.Status(license.operationMode(), state)); + currentLicense = license; + currentLicenseState = state; + } + } + } + } + + /** + * Request handler for trial license generation to master + */ + private class RegisterTrialLicenseRequestHandler implements TransportRequestHandler { + + @Override + public void messageReceived(TransportRequest.Empty empty, TransportChannel channel) throws Exception { + registerTrialLicense(); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + } + + // TODO - temporary hack for tests, should be removed once we introduce `ClockMock` + public void setGracePeriodDuration(TimeValue gracePeriodDuration) { + this.gracePeriodDuration = gracePeriodDuration; + } + // only for adding expiration callbacks for tests + public void setExpirationCallbacks(List expirationCallbacks) { + this.expirationCallbacks = expirationCallbacks; + } + // TODO - temporary hack for tests, should be removed once we introduce `ClockMock` + public void setTrialLicenseDuration(TimeValue trialLicenseDuration) { + this.trialLicenseDuration = trialLicenseDuration; + } +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesStatus.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesStatus.java new file mode 100644 index 00000000000..3bd18d58770 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesStatus.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +public enum LicensesStatus { + VALID((byte) 0), + INVALID((byte) 1), + EXPIRED((byte) 2); + + private final byte id; + + LicensesStatus(byte id) { + this.id = id; + } + + public int id() { + return id; + } + + public static LicensesStatus fromId(int id) { + if (id == 0) { + return VALID; + } else if (id == 1) { + return INVALID; + } else if (id == 2) { + return EXPIRED; + } else { + throw new IllegalStateException("no valid LicensesStatus for id=" + id); + } + } +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/TrialLicense.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/TrialLicense.java new file mode 100644 index 00000000000..c528c452a4e --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/core/TrialLicense.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.common.Base64; +import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.license.core.License; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; + +import static org.elasticsearch.license.core.CryptUtils.decrypt; +import static org.elasticsearch.license.core.CryptUtils.encrypt; + +public class TrialLicense { + + public static License create(License.Builder specBuilder) { + License spec = specBuilder + .type("trial") + .issuer("elasticsearch") + .version(License.VERSION_CURRENT) + .build(); + final String signature; + try { + XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + spec.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true"))); + byte[] encrypt = encrypt(contentBuilder.bytes().toBytes()); + byte[] bytes = new byte[4 + 4 + encrypt.length]; + ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); + // always generate license version -VERSION_CURRENT + byteBuffer.putInt(-License.VERSION_CURRENT) + .putInt(encrypt.length) + .put(encrypt); + signature = Base64.encodeBytes(bytes); + } catch (IOException e) { + throw new IllegalStateException(e); + } + return License.builder().fromLicenseSpec(spec, signature).build(); + } + + public static boolean verify(final License license) { + try { + byte[] signatureBytes = Base64.decode(license.signature()); + ByteBuffer byteBuffer = ByteBuffer.wrap(signatureBytes); + int version = byteBuffer.getInt(); + int contentLen = byteBuffer.getInt(); + byte[] content = new byte[contentLen]; + byteBuffer.get(content); + final License expectedLicense; + try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(decrypt(content))) { + parser.nextToken(); + expectedLicense = License.builder().fromLicenseSpec(License.fromXContent(parser), license.signature()).version(-version).build(); + } + return license.equals(expectedLicense); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestDeleteLicenseAction.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestDeleteLicenseAction.java new file mode 100644 index 00000000000..7919ba8c121 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestDeleteLicenseAction.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.rest; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseAction; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseRequest; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseResponse; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.support.AcknowledgedRestListener; + +import static org.elasticsearch.rest.RestRequest.Method.DELETE; + +public class RestDeleteLicenseAction extends BaseRestHandler { + + @Inject + public RestDeleteLicenseAction(Settings settings, RestController controller, Client client) { + super(settings, controller, client); + controller.registerHandler(DELETE, "/_license", this); + } + + @Override + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + client.admin().cluster().execute(DeleteLicenseAction.INSTANCE, new DeleteLicenseRequest(), new AcknowledgedRestListener(channel)); + } +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestGetLicenseAction.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestGetLicenseAction.java new file mode 100644 index 00000000000..6d9a31b56f1 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestGetLicenseAction.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.rest; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.plugin.action.get.GetLicenseAction; +import org.elasticsearch.license.plugin.action.get.GetLicenseRequest; +import org.elasticsearch.license.plugin.action.get.GetLicenseResponse; +import org.elasticsearch.rest.*; +import org.elasticsearch.rest.action.support.RestBuilderListener; + +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestStatus.NOT_FOUND; +import static org.elasticsearch.rest.RestStatus.OK; + +public class RestGetLicenseAction extends BaseRestHandler { + + @Inject + public RestGetLicenseAction(Settings settings, RestController controller, Client client) { + super(settings, controller, client); + controller.registerHandler(GET, "/_license", this); + } + + /** + * There will be only one license displayed per feature, the selected license will have the latest expiry_date + * out of all other licenses for the feature. + *

+ * The licenses are sorted by latest issue_date + */ + @Override + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + final Map overrideParams = new HashMap<>(2); + overrideParams.put(License.REST_VIEW_MODE, "true"); + overrideParams.put(License.LICENSE_VERSION_MODE, String.valueOf(License.VERSION_CURRENT)); + final ToXContent.Params params = new ToXContent.DelegatingMapParams(overrideParams, request); + GetLicenseRequest getLicenseRequest = new GetLicenseRequest(); + getLicenseRequest.local(request.paramAsBoolean("local", getLicenseRequest.local())); + client.admin().cluster().execute(GetLicenseAction.INSTANCE, getLicenseRequest, new RestBuilderListener(channel) { + @Override + public RestResponse buildResponse(GetLicenseResponse response, XContentBuilder builder) throws Exception { + // Default to pretty printing, but allow ?pretty=false to disable + if (!request.hasParam("pretty")) { + builder.prettyPrint().lfAtEnd(); + } + builder.startObject(); + if (response.license() != null) { + builder.startObject("license"); + response.license().toInnerXContent(builder, params); + builder.endObject(); + } + builder.endObject(); + return new BytesRestResponse(OK, builder); + } + }); + } + +} diff --git a/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestPutLicenseAction.java b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestPutLicenseAction.java new file mode 100644 index 00000000000..f355a610791 --- /dev/null +++ b/elasticsearch/license/plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestPutLicenseAction.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.rest; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.license.plugin.action.put.PutLicenseAction; +import org.elasticsearch.license.plugin.action.put.PutLicenseRequest; +import org.elasticsearch.license.plugin.action.put.PutLicenseResponse; +import org.elasticsearch.rest.*; +import org.elasticsearch.rest.action.support.RestBuilderListener; + +import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +public class RestPutLicenseAction extends BaseRestHandler { + + @Inject + public RestPutLicenseAction(Settings settings, RestController controller, Client client) { + super(settings, controller, client); + controller.registerHandler(PUT, "/_license", this); + controller.registerHandler(POST, "/_license", this); + } + + @Override + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + PutLicenseRequest putLicenseRequest = new PutLicenseRequest(); + putLicenseRequest.license(request.content().toUtf8()); + putLicenseRequest.acknowledge(request.paramAsBoolean("acknowledge", false)); + client.admin().cluster().execute(PutLicenseAction.INSTANCE, putLicenseRequest, new RestBuilderListener(channel) { + @Override + public RestResponse buildResponse(PutLicenseResponse response, XContentBuilder builder) throws Exception { + builder.startObject(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + return new BytesRestResponse(RestStatus.OK, builder); + } + }); + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesConsumerPluginIntegrationTestCase.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesConsumerPluginIntegrationTestCase.java new file mode 100644 index 00000000000..ca755c3934b --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesConsumerPluginIntegrationTestCase.java @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.license.plugin.consumer.TestConsumerPluginBase; +import org.elasticsearch.license.plugin.consumer.TestPluginServiceBase; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.InternalTestCluster; +import org.junit.After; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; + +/** + * Framework to test licensing plugin integration for existing/new consumer plugins + * see {@link org.elasticsearch.license.plugin.LicensesEagerConsumerPluginIntegrationTests} and {@link org.elasticsearch.license.plugin.LicensesLazyConsumerPluginIntegrationTests} + * for example usage + */ +@ClusterScope(scope = TEST, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0) +public abstract class AbstractLicensesConsumerPluginIntegrationTestCase extends AbstractLicensesIntegrationTestCase { + protected final TestConsumerPluginBase consumerPlugin; + + public AbstractLicensesConsumerPluginIntegrationTestCase(TestConsumerPluginBase consumerPlugin) { + this.consumerPlugin = consumerPlugin; + } + + private final int trialLicenseDurationInSeconds = 20; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.settingsBuilder() + .put(super.nodeSettings(nodeOrdinal)) + // this setting is only used in tests + .put(".trial_license_duration_in_seconds", trialLicenseDurationInSeconds) + .build(); + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(LicensePlugin.class, consumerPlugin.getClass()); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + @After + public void afterTest() throws Exception { + wipeAllLicenses(); + assertTrue(awaitBusy(() -> !clusterService().state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK))); + } + + public void testTrialLicenseAndSignedLicenseNotification() throws Exception { + logger.info("using " + consumerPlugin.getClass().getName() + " consumer plugin"); + logger.info(" --> trial license generated"); + // managerService should report feature to be enabled on all data nodes + assertLicenseeState(consumerPlugin.id(), LicenseState.ENABLED); + // consumer plugin service should return enabled on all data nodes + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.ENABLED, 2); + + logger.info(" --> check trial license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired trial license) + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.GRACE_PERIOD, trialLicenseDurationInSeconds * 2); + assertLicenseeState(consumerPlugin.id(), LicenseState.GRACE_PERIOD); + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.DISABLED, trialLicenseDurationInSeconds * 2); + assertLicenseeState(consumerPlugin.id(), LicenseState.DISABLED); + + logger.info(" --> put signed license"); + putLicense(TimeValue.timeValueSeconds(trialLicenseDurationInSeconds)); + + logger.info(" --> check signed license enabled notification"); + // consumer plugin should notify onEnabled on all data nodes (signed license) + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.ENABLED, 1); + assertLicenseeState(consumerPlugin.id(), LicenseState.ENABLED); + + logger.info(" --> check signed license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired signed license) + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.GRACE_PERIOD, trialLicenseDurationInSeconds * 2); + assertLicenseeState(consumerPlugin.id(), LicenseState.GRACE_PERIOD); + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.DISABLED, trialLicenseDurationInSeconds * 2); + assertLicenseeState(consumerPlugin.id(), LicenseState.DISABLED); + } + + public void testTrialLicenseNotification() throws Exception { + logger.info(" --> check onEnabled for trial license"); + // managerService should report feature to be enabled on all data nodes + assertLicenseeState(consumerPlugin.id(), LicenseState.ENABLED); + // consumer plugin service should return enabled on all data nodes + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.ENABLED, 1); + + logger.info(" --> check trial license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired signed license) + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.GRACE_PERIOD, trialLicenseDurationInSeconds); + assertLicenseeState(consumerPlugin.id(), LicenseState.GRACE_PERIOD); + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.DISABLED, trialLicenseDurationInSeconds); + assertLicenseeState(consumerPlugin.id(), LicenseState.DISABLED); + } + + public void testOverlappingTrialAndSignedLicenseNotification() throws Exception { + logger.info(" --> check onEnabled for trial license"); + // managerService should report feature to be enabled on all data nodes + assertLicenseeState(consumerPlugin.id(), LicenseState.ENABLED); + // consumer plugin service should return enabled on all data nodes + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.ENABLED, 1); + + logger.info(" --> put signed license while trial license is in effect"); + putLicense(TimeValue.timeValueSeconds(trialLicenseDurationInSeconds * 2)); + + logger.info(" --> check signed license enabled notification"); + // consumer plugin should notify onEnabled on all data nodes (signed license) + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.ENABLED, 1); + assertLicenseeState(consumerPlugin.id(), LicenseState.ENABLED); + + logger.info(" --> sleep for rest of trailLicense duration"); + Thread.sleep(trialLicenseDurationInSeconds * 1000l); + + logger.info(" --> check consumer is still enabled [signed license]"); + // consumer plugin should notify onEnabled on all data nodes (signed license) + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.ENABLED, 1); + assertLicenseeState(consumerPlugin.id(), LicenseState.ENABLED); + + logger.info(" --> check signed license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired signed license) + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.GRACE_PERIOD, trialLicenseDurationInSeconds * 2 * 2); + assertLicenseeState(consumerPlugin.id(), LicenseState.GRACE_PERIOD); + assertConsumerPluginNotification(consumerPluginServices(), LicenseState.DISABLED, trialLicenseDurationInSeconds * 2 * 2); + assertLicenseeState(consumerPlugin.id(), LicenseState.DISABLED); + } + + private List consumerPluginServices() { + final InternalTestCluster clients = internalCluster(); + List consumerPluginServices = new ArrayList<>(); + for (TestPluginServiceBase service : clients.getDataNodeInstances(consumerPlugin.service())) { + consumerPluginServices.add(service); + } + return consumerPluginServices; + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesIntegrationTestCase.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesIntegrationTestCase.java new file mode 100644 index 00000000000..4ca01a9a990 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesIntegrationTestCase.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.plugin.action.put.PutLicenseAction; +import org.elasticsearch.license.plugin.action.put.PutLicenseRequestBuilder; +import org.elasticsearch.license.plugin.action.put.PutLicenseResponse; +import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService; +import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationPluginService; +import org.elasticsearch.license.plugin.consumer.TestPluginServiceBase; +import org.elasticsearch.license.plugin.core.*; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.license.plugin.TestUtils.generateSignedLicense; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public abstract class AbstractLicensesIntegrationTestCase extends ESIntegTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.EMPTY; + } + + @Override + protected Collection> nodePlugins() { + return Collections.>singletonList(LicensePlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + @Override + protected Settings transportClientSettings() { + // Plugin should be loaded on the transport client as well + return nodeSettings(0); + } + + protected void wipeAllLicenses() throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName()); + clusterService.submitStateUpdateTask("delete licensing metadata", new ClusterStateUpdateTask() { + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + latch.countDown(); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); + mdBuilder.removeCustom(LicensesMetaData.TYPE); + return ClusterState.builder(currentState).metaData(mdBuilder).build(); + } + + @Override + public void onFailure(String source, @Nullable Throwable t) { + logger.error("error on metaData cleanup after test", t); + } + }); + latch.await(); + } + + protected void putLicense(TimeValue expiryDuration) throws Exception { + License license1 = generateSignedLicense(expiryDuration); + final PutLicenseResponse putLicenseResponse = new PutLicenseRequestBuilder(client().admin().cluster(), PutLicenseAction.INSTANCE).setLicense(license1).get(); + assertThat(putLicenseResponse.isAcknowledged(), equalTo(true)); + assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID)); + } + + protected void assertLicenseeState(final String id, final LicenseState state) throws InterruptedException { + assertTrue("LicensesManagerService for licensee " + id + " should have status " + state.name(), awaitBusy(() -> { + final InternalTestCluster clients = internalCluster(); + for (LicensesManagerService managerService : clients.getDataNodeInstances(LicensesManagerService.class)) { + if (!managerService.licenseesWithState(state).contains(id)) { + return false; + } + } + return true; + })); + } + + protected void assertLazyConsumerPluginNotification(final LicenseState state, int timeoutInSec) throws InterruptedException { + final List consumerPluginServices = consumerLazyPluginServices(); + assertConsumerPluginNotification(consumerPluginServices, state, timeoutInSec); + } + + protected void assertEagerConsumerPluginNotification(final LicenseState state, int timeoutInSec) throws InterruptedException { + final List consumerPluginServices = consumerEagerPluginServices(); + assertConsumerPluginNotification(consumerPluginServices, state, timeoutInSec); + } + + protected void assertConsumerPluginNotification(final List consumerPluginServices, final LicenseState state, int timeoutInSec) throws InterruptedException { + assertThat("At least one instance has to be present", consumerPluginServices.size(), greaterThan(0)); + boolean success = awaitBusy(() -> { + for (TestPluginServiceBase pluginService : consumerPluginServices) { + if (state != pluginService.state()) { + return false; + } + } + return true; + }, timeoutInSec + 1, TimeUnit.SECONDS); + logger.debug("Notification assertion complete"); + assertThat(consumerPluginServices.get(0).getClass().getName() + " should have status " + state.name(), success, equalTo(true)); + } + + private List consumerLazyPluginServices() { + final InternalTestCluster clients = internalCluster(); + List consumerPluginServices = new ArrayList<>(); + for (TestPluginServiceBase service : clients.getDataNodeInstances(LazyLicenseRegistrationPluginService.class)) { + consumerPluginServices.add(service); + } + return consumerPluginServices; + } + + private List consumerEagerPluginServices() { + final InternalTestCluster clients = internalCluster(); + List consumerPluginServices = new ArrayList<>(); + for (TestPluginServiceBase service : clients.getDataNodeInstances(EagerLicenseRegistrationPluginService.class)) { + consumerPluginServices.add(service); + } + return consumerPluginServices; + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesEagerConsumerPluginIntegrationTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesEagerConsumerPluginIntegrationTests.java new file mode 100644 index 00000000000..fc87ca63184 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesEagerConsumerPluginIntegrationTests.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.apache.lucene.util.LuceneTestCase.BadApple; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationConsumerPlugin; + +// test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") +public class LicensesEagerConsumerPluginIntegrationTests extends AbstractLicensesConsumerPluginIntegrationTestCase { + + public LicensesEagerConsumerPluginIntegrationTests() { + super(new EagerLicenseRegistrationConsumerPlugin(Settings.EMPTY)); + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesLazyConsumerPluginIntegrationTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesLazyConsumerPluginIntegrationTests.java new file mode 100644 index 00000000000..6204bbf9cae --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesLazyConsumerPluginIntegrationTests.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.apache.lucene.util.LuceneTestCase.BadApple; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationConsumerPlugin; + +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") +public class LicensesLazyConsumerPluginIntegrationTests extends AbstractLicensesConsumerPluginIntegrationTestCase { + + public LicensesLazyConsumerPluginIntegrationTests() { + super(new LazyLicenseRegistrationConsumerPlugin(Settings.EMPTY)); + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesPluginIntegrationTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesPluginIntegrationTests.java new file mode 100644 index 00000000000..7fc85a42a9d --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesPluginIntegrationTests.java @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.apache.lucene.util.LuceneTestCase.BadApple; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationConsumerPlugin; +import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService; +import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationConsumerPlugin; +import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationPluginService; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.junit.After; + +import java.util.Arrays; +import java.util.Collection; + +import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; + +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") +@ClusterScope(scope = TEST, numDataNodes = 2, numClientNodes = 0) +public class LicensesPluginIntegrationTests extends AbstractLicensesIntegrationTestCase { + private final boolean useEagerLicenseRegistrationPlugin = randomBoolean(); + + private final int trialLicenseDurationInSeconds = 10; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.settingsBuilder() + .put(super.nodeSettings(nodeOrdinal)) + // this setting is only used in tests + .put(".trial_license_duration_in_seconds", trialLicenseDurationInSeconds) + .build(); + } + + @Override + protected Collection> nodePlugins() { + if (useEagerLicenseRegistrationPlugin) { + return Arrays.asList(LicensePlugin.class, EagerLicenseRegistrationConsumerPlugin.class); + } else { + return Arrays.asList(LicensePlugin.class, LazyLicenseRegistrationConsumerPlugin.class); + } + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + @After + public void afterTest() throws Exception { + wipeAllLicenses(); + assertTrue(awaitBusy(() -> !clusterService().state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK))); + } + + public void testTrialLicenseAndSignedLicenseNotification() throws Exception { + logger.info("using " + ((useEagerLicenseRegistrationPlugin) ? "eager" : "lazy") + " consumer plugin"); + logger.info(" --> trial license generated"); + // managerService should report feature to be enabled on all data nodes + assertLicenseeState(getCurrentFeatureName(), LicenseState.ENABLED); + // consumer plugin service should return enabled on all data nodes + assertConsumerPluginEnabledNotification(2); + + logger.info(" --> check trial license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired trial license) + assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds * 2); + assertLicenseeState(getCurrentFeatureName(), LicenseState.GRACE_PERIOD); + + assertLicenseeState(getCurrentFeatureName(), LicenseState.DISABLED); + + logger.info(" --> put signed license"); + putLicense(TimeValue.timeValueSeconds(trialLicenseDurationInSeconds)); + + logger.info(" --> check signed license enabled notification"); + // consumer plugin should notify onEnabled on all data nodes (signed license) + assertConsumerPluginEnabledNotification(1); + assertLicenseeState(getCurrentFeatureName(), LicenseState.ENABLED); + + logger.info(" --> check signed license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired signed license) + assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds * 2); + assertLicenseeState(getCurrentFeatureName(), LicenseState.GRACE_PERIOD); + assertLicenseeState(getCurrentFeatureName(), LicenseState.DISABLED); + } + + public void testTrialLicenseNotification() throws Exception { + logger.info(" --> check onEnabled for trial license"); + // managerService should report feature to be enabled on all data nodes + assertLicenseeState(getCurrentFeatureName(), LicenseState.ENABLED); + // consumer plugin service should return enabled on all data nodes + assertConsumerPluginEnabledNotification(1); + + logger.info(" --> check trial license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired signed license) + assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds * 2); + assertLicenseeState(getCurrentFeatureName(), LicenseState.GRACE_PERIOD); + assertLicenseeState(getCurrentFeatureName(), LicenseState.DISABLED); + } + + public void testOverlappingTrialAndSignedLicenseNotification() throws Exception { + logger.info(" --> check onEnabled for trial license"); + // managerService should report feature to be enabled on all data nodes + assertLicenseeState(getCurrentFeatureName(), LicenseState.ENABLED); + // consumer plugin service should return enabled on all data nodes + assertConsumerPluginEnabledNotification(1); + + logger.info(" --> put signed license while trial license is in effect"); + putLicense(TimeValue.timeValueSeconds(trialLicenseDurationInSeconds * 2)); + + logger.info(" --> check signed license enabled notification"); + // consumer plugin should notify onEnabled on all data nodes (signed license) + assertConsumerPluginEnabledNotification(1); + assertLicenseeState(getCurrentFeatureName(), LicenseState.ENABLED); + + logger.info(" --> sleep for rest of trailLicense duration"); + Thread.sleep(trialLicenseDurationInSeconds * 1000l); + + logger.info(" --> check consumer is still enabled [signed license]"); + // consumer plugin should notify onEnabled on all data nodes (signed license) + assertConsumerPluginEnabledNotification(1); + assertLicenseeState(getCurrentFeatureName(), LicenseState.ENABLED); + + logger.info(" --> check signed license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired signed license) + assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds * 2 * 2); + assertLicenseeState(getCurrentFeatureName(), LicenseState.GRACE_PERIOD); + assertLicenseeState(getCurrentFeatureName(), LicenseState.DISABLED); + } + + private String getCurrentFeatureName() { + if (useEagerLicenseRegistrationPlugin) { + return EagerLicenseRegistrationPluginService.ID; + } else { + return LazyLicenseRegistrationPluginService.ID; + } + } + + private void assertConsumerPluginEnabledNotification(int timeoutInSec) throws InterruptedException { + if (useEagerLicenseRegistrationPlugin) { + assertEagerConsumerPluginNotification(LicenseState.ENABLED, timeoutInSec); + } else { + assertLazyConsumerPluginNotification(LicenseState.ENABLED, timeoutInSec); + } + } + + private void assertConsumerPluginDisabledNotification(int timeoutInSec) throws InterruptedException { + if (useEagerLicenseRegistrationPlugin) { + assertEagerConsumerPluginNotification(LicenseState.GRACE_PERIOD, timeoutInSec); + } else { + assertLazyConsumerPluginNotification(LicenseState.GRACE_PERIOD, timeoutInSec); + } + } + +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesPluginsIntegrationTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesPluginsIntegrationTests.java new file mode 100644 index 00000000000..ceb25d2921e --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesPluginsIntegrationTests.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.apache.lucene.util.LuceneTestCase.BadApple; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationConsumerPlugin; +import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService; +import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationConsumerPlugin; +import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationPluginService; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.junit.After; + +import java.util.Arrays; +import java.util.Collection; + +import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; + +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") +@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0) +public class LicensesPluginsIntegrationTests extends AbstractLicensesIntegrationTestCase { + private static final String ID_1 = EagerLicenseRegistrationPluginService.ID; + private static final String ID_2 = LazyLicenseRegistrationPluginService.ID; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.settingsBuilder() + .put(super.nodeSettings(nodeOrdinal)) + .build(); + } + + private Settings nodeSettingsWithConsumerPlugin(int trialLicenseDuration) { + return Settings.settingsBuilder() + .put(super.nodeSettings(0)) + // this setting is only used in tests + .put(".trial_license_duration_in_seconds", trialLicenseDuration) + .build(); + + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(LicensePlugin.class, EagerLicenseRegistrationConsumerPlugin.class, LazyLicenseRegistrationConsumerPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + @After + public void afterTest() throws Exception { + wipeAllLicenses(); + } + + public void testMultipleConsumerPlugins() throws Exception { + int nNodes = randomIntBetween(2, 3); + int trialLicenseDurationInSec = 20; + int signedLicenseDuration = 5; + startNodesWithConsumerPlugins(nNodes, trialLicenseDurationInSec); + + logger.info(" --> trial license generated"); + // managerService should report feature to be enabled on all data nodes + assertLicenseeState(ID_1, LicenseState.ENABLED); + assertLicenseeState(ID_2, LicenseState.ENABLED); + // consumer plugin service should return enabled on all data nodes + assertEagerConsumerPluginNotification(LicenseState.ENABLED, 1); + assertLazyConsumerPluginNotification(LicenseState.ENABLED, 1); + + logger.info(" --> check trial license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired trial license) + assertEagerConsumerPluginNotification(LicenseState.GRACE_PERIOD, trialLicenseDurationInSec * 2); + assertLazyConsumerPluginNotification(LicenseState.GRACE_PERIOD, trialLicenseDurationInSec * 2); + assertLicenseeState(ID_1, LicenseState.GRACE_PERIOD); + assertLicenseeState(ID_2, LicenseState.GRACE_PERIOD); + assertLicenseeState(ID_1, LicenseState.DISABLED); + assertLicenseeState(ID_2, LicenseState.DISABLED); + + logger.info(" --> put signed license"); + putLicense(TimeValue.timeValueSeconds(signedLicenseDuration)); + + logger.info(" --> check signed license enabled notification"); + // consumer plugin should notify onEnabled on all data nodes (signed license) + assertEagerConsumerPluginNotification(LicenseState.ENABLED, 1); + assertLazyConsumerPluginNotification(LicenseState.ENABLED, 1); + assertLicenseeState(ID_1, LicenseState.ENABLED); + assertLicenseeState(ID_2, LicenseState.ENABLED); + + logger.info(" --> check signed license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired signed license) + assertEagerConsumerPluginNotification(LicenseState.GRACE_PERIOD, signedLicenseDuration * 2); + assertLazyConsumerPluginNotification(LicenseState.GRACE_PERIOD, signedLicenseDuration * 2); + assertLicenseeState(ID_1, LicenseState.GRACE_PERIOD); + assertLicenseeState(ID_2, LicenseState.GRACE_PERIOD); + + assertEagerConsumerPluginNotification(LicenseState.DISABLED, 10 * 2); + assertLazyConsumerPluginNotification(LicenseState.DISABLED, 10 * 2); + assertLicenseeState(ID_1, LicenseState.DISABLED); + assertLicenseeState(ID_2, LicenseState.DISABLED); + } + + public void testRandomFeatureLicensesActions() throws Exception { + int nNodes = randomIntBetween(2, 3); + + startNodesWithConsumerPlugins(nNodes, 10); + + logger.info(" --> check license enabled notification"); + assertEagerConsumerPluginNotification(LicenseState.ENABLED, 1); + assertLazyConsumerPluginNotification(LicenseState.ENABLED, 1); + assertLicenseeState(ID_1, LicenseState.ENABLED); + assertLicenseeState(ID_2, LicenseState.ENABLED); + + logger.info(" --> check license expiry notification"); + // consumer plugin should notify onDisabled on all data nodes (expired signed license) + assertEagerConsumerPluginNotification(LicenseState.GRACE_PERIOD, 10 * 2); + assertLazyConsumerPluginNotification(LicenseState.GRACE_PERIOD, 10 * 2); + assertLicenseeState(ID_1, LicenseState.GRACE_PERIOD); + assertLicenseeState(ID_2, LicenseState.GRACE_PERIOD); + assertEagerConsumerPluginNotification(LicenseState.DISABLED, 10 * 2); + assertLazyConsumerPluginNotification(LicenseState.DISABLED, 10 * 2); + assertLicenseeState(ID_1, LicenseState.DISABLED); + assertLicenseeState(ID_2, LicenseState.DISABLED); + } + + private void startNodesWithConsumerPlugins(int nNodes, int trialLicenseDuration) { + for (int i = 0; i < nNodes; i++) { + internalCluster().startNode(nodeSettingsWithConsumerPlugin(trialLicenseDuration)); + } + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesServiceClusterTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesServiceClusterTests.java new file mode 100644 index 00000000000..f50e4e06fad --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesServiceClusterTests.java @@ -0,0 +1,232 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.client.ClusterAdminClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseAction; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseRequestBuilder; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseResponse; +import org.elasticsearch.license.plugin.action.get.GetLicenseAction; +import org.elasticsearch.license.plugin.action.get.GetLicenseRequestBuilder; +import org.elasticsearch.license.plugin.action.get.GetLicenseResponse; +import org.elasticsearch.license.plugin.action.put.PutLicenseAction; +import org.elasticsearch.license.plugin.action.put.PutLicenseRequestBuilder; +import org.elasticsearch.license.plugin.action.put.PutLicenseResponse; +import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationConsumerPlugin; +import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService; +import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationConsumerPlugin; +import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationPluginService; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.license.plugin.core.LicensesManagerService; +import org.elasticsearch.license.plugin.core.LicensesMetaData; +import org.elasticsearch.license.plugin.core.LicensesStatus; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.InternalTestCluster; + +import java.util.Arrays; +import java.util.Collection; + +import static org.elasticsearch.license.plugin.TestUtils.generateSignedLicense; +import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; + +@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0, transportClientRatio = 0) +public class LicensesServiceClusterTests extends AbstractLicensesIntegrationTestCase { + private final String[] PLUGINS = {EagerLicenseRegistrationPluginService.ID, LazyLicenseRegistrationPluginService.ID}; + + @Override + protected Settings transportClientSettings() { + return super.transportClientSettings(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return nodeSettingsBuilder(nodeOrdinal).build(); + } + + private Settings.Builder nodeSettingsBuilder(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugins.load_classpath_plugins", false) + .put("node.data", true) + .put("format", "json") + // this setting is only used in tests + .put(".trial_license_duration_in_seconds", 9) + // this setting is only used in tests + .put(".grace_duration_in_seconds", 9) + .put(Node.HTTP_ENABLED, true); + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(LicensePlugin.class, EagerLicenseRegistrationConsumerPlugin.class, LazyLicenseRegistrationConsumerPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + public void testClusterRestartWithLicense() throws Exception { + wipeAllLicenses(); + + int numNodes = randomIntBetween(1, 5); + logger.info("--> starting " + numNodes + " node(s)"); + for (int i = 0; i < numNodes; i++) { + internalCluster().startNode(); + } + ensureGreen(); + + logger.info("--> put signed license"); + License license = generateAndPutLicenses(); + getAndCheckLicense(license); + logger.info("--> restart all nodes"); + internalCluster().fullRestart(); + ensureYellow(); + + logger.info("--> get and check signed license"); + getAndCheckLicense(license); + + logger.info("--> remove licenses"); + removeLicense(); + assertNoLicense(); + logger.info("--> restart all nodes"); + internalCluster().fullRestart(); + ensureYellow(); + assertNoLicense(); + + wipeAllLicenses(); + } + + public void testClusterRestartWhileEnabled() throws Exception { + wipeAllLicenses(); + internalCluster().startNode(); + ensureGreen(); + assertEagerConsumerPluginNotification(LicenseState.ENABLED, 5); + assertLazyConsumerPluginNotification(LicenseState.ENABLED, 5); + logger.info("--> restart node"); + internalCluster().fullRestart(); + ensureYellow(); + logger.info("--> await node for enabled"); + assertEagerConsumerPluginNotification(LicenseState.ENABLED, 5); + assertLazyConsumerPluginNotification(LicenseState.ENABLED, 5); + } + + public void testClusterRestartWhileGrace() throws Exception { + wipeAllLicenses(); + internalCluster().startNode(); + ensureGreen(); + assertEagerConsumerPluginNotification(LicenseState.GRACE_PERIOD, 10); + assertLazyConsumerPluginNotification(LicenseState.GRACE_PERIOD, 10); + logger.info("--> restart node"); + internalCluster().fullRestart(); + ensureYellow(); + logger.info("--> await node for grace_period"); + assertEagerConsumerPluginNotification(LicenseState.GRACE_PERIOD, 5); + assertLazyConsumerPluginNotification(LicenseState.GRACE_PERIOD, 5); + } + + public void testClusterRestartWhileExpired() throws Exception { + wipeAllLicenses(); + internalCluster().startNode(); + ensureGreen(); + assertEagerConsumerPluginNotification(LicenseState.DISABLED, 20); + assertLazyConsumerPluginNotification(LicenseState.DISABLED, 20); + logger.info("--> restart node"); + internalCluster().fullRestart(); + ensureYellow(); + logger.info("--> await node for disabled"); + assertEagerConsumerPluginNotification(LicenseState.DISABLED, 5); + assertLazyConsumerPluginNotification(LicenseState.DISABLED, 5); + } + + public void testClusterNotRecovered() throws Exception { + logger.info("--> start one master out of two [recovery state]"); + internalCluster().startNode(nodeSettingsBuilder(0).put("discovery.zen.minimum_master_nodes", 2).put("node.master", true)); + logger.info("--> start second master out of two [recovered state]"); + internalCluster().startNode(nodeSettingsBuilder(1).put("discovery.zen.minimum_master_nodes", 2).put("node.master", true)); + assertLicenseesStateEnabled(); + assertConsumerPluginEnabledNotification(1); + } + + public void testAtMostOnceTrialLicenseGeneration() throws Exception { + wipeAllLicenses(); + logger.info("--> start one node [trial license should be generated & enabled]"); + internalCluster().startNode(nodeSettingsBuilder(0)); + assertLicenseesStateEnabled(); + assertConsumerPluginEnabledNotification(1); + + logger.info("--> start another node [trial license should be propagated from the old master not generated]"); + internalCluster().startNode(nodeSettings(1)); + assertLicenseesStateEnabled(); + assertConsumerPluginEnabledNotification(1); + + logger.info("--> check if multiple trial licenses are found for a id"); + LicensesMetaData licensesMetaData = clusterService().state().metaData().custom(LicensesMetaData.TYPE); + assertThat(licensesMetaData.getLicense(), not(LicensesMetaData.LICENSE_TOMBSTONE)); + + wipeAllLicenses(); + } + + private void removeLicense() throws Exception { + ClusterAdminClient cluster = internalCluster().client().admin().cluster(); + ensureGreen(); + License putLicenses = generateSignedLicense(TimeValue.timeValueMinutes(1)); + PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(cluster, PutLicenseAction.INSTANCE); + putLicenseRequestBuilder.setLicense(putLicenses); + DeleteLicenseResponse response = new DeleteLicenseRequestBuilder(cluster, DeleteLicenseAction.INSTANCE).get(); + assertThat(response.isAcknowledged(), equalTo(true)); + } + + private License generateAndPutLicenses() throws Exception { + ClusterAdminClient cluster = internalCluster().client().admin().cluster(); + ensureGreen(); + License putLicenses = generateSignedLicense(TimeValue.timeValueMinutes(1)); + PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(cluster, PutLicenseAction.INSTANCE); + putLicenseRequestBuilder.setLicense(putLicenses); + final PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get(); + assertThat(putLicenseResponse.isAcknowledged(), equalTo(true)); + assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID)); + return putLicenses; + } + + private void assertNoLicense() { + ClusterAdminClient cluster = internalCluster().client().admin().cluster(); + final GetLicenseResponse response = new GetLicenseRequestBuilder(cluster, GetLicenseAction.INSTANCE).get(); + assertThat(response.license(), nullValue()); + LicensesMetaData licensesMetaData = clusterService().state().metaData().custom(LicensesMetaData.TYPE); + assertThat(licensesMetaData, notNullValue()); + assertThat(licensesMetaData.getLicense(), equalTo(LicensesMetaData.LICENSE_TOMBSTONE)); + } + + private void getAndCheckLicense(License license) { + ClusterAdminClient cluster = internalCluster().client().admin().cluster(); + final GetLicenseResponse response = new GetLicenseRequestBuilder(cluster, GetLicenseAction.INSTANCE).get(); + assertThat(response.license(), equalTo(license)); + LicensesMetaData licensesMetaData = clusterService().state().metaData().custom(LicensesMetaData.TYPE); + assertThat(licensesMetaData, notNullValue()); + assertThat(licensesMetaData.getLicense(), not(LicensesMetaData.LICENSE_TOMBSTONE)); + } + + private void assertLicenseesStateEnabled() throws Exception { + for (String id : PLUGINS) { + assertLicenseeState(id, LicenseState.ENABLED); + } + } + + private void assertConsumerPluginEnabledNotification(int timeoutInSec) throws InterruptedException { + assertEagerConsumerPluginNotification(LicenseState.ENABLED, timeoutInSec); + assertLazyConsumerPluginNotification(LicenseState.ENABLED, timeoutInSec); + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesServiceNodeTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesServiceNodeTests.java new file mode 100644 index 00000000000..c9727efa93b --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesServiceNodeTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationConsumerPlugin; +import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; + +/** + */ +@ESIntegTestCase.ClusterScope(scope = TEST, numDataNodes = 10, numClientNodes = 0) +public class LicensesServiceNodeTests extends AbstractLicensesIntegrationTestCase { + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.settingsBuilder() + .put(super.nodeSettings(nodeOrdinal)) + .put(Node.HTTP_ENABLED, true) + .build(); + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(LicensePlugin.class, EagerLicenseRegistrationConsumerPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + public void testPluginStatus() throws Exception { + final Iterable testPluginServices = internalCluster().getDataNodeInstances(EagerLicenseRegistrationPluginService.class); + assertTrue(awaitBusy(() -> { + for (EagerLicenseRegistrationPluginService pluginService : testPluginServices) { + if (pluginService.state() != LicenseState.ENABLED) { + return false; + } + } + return true; + })); + + } + +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesTransportTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesTransportTests.java new file mode 100644 index 00000000000..9b8ea3294c3 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/LicensesTransportTests.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseAction; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseRequestBuilder; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseResponse; +import org.elasticsearch.license.plugin.action.get.GetLicenseAction; +import org.elasticsearch.license.plugin.action.get.GetLicenseRequestBuilder; +import org.elasticsearch.license.plugin.action.get.GetLicenseResponse; +import org.elasticsearch.license.plugin.action.put.PutLicenseAction; +import org.elasticsearch.license.plugin.action.put.PutLicenseRequestBuilder; +import org.elasticsearch.license.plugin.action.put.PutLicenseResponse; +import org.elasticsearch.license.plugin.core.LicensesStatus; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.junit.After; + +import static org.elasticsearch.license.plugin.TestUtils.dateMath; +import static org.elasticsearch.license.plugin.TestUtils.generateSignedLicense; +import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; + +@ClusterScope(scope = TEST, numDataNodes = 10) +public class LicensesTransportTests extends AbstractLicensesIntegrationTestCase { + @After + public void beforeTest() throws Exception { + wipeAllLicenses(); + } + + public void testEmptyGetLicense() throws Exception { + final ActionFuture getLicenseFuture = new GetLicenseRequestBuilder(client().admin().cluster(), GetLicenseAction.INSTANCE).execute(); + final GetLicenseResponse getLicenseResponse = getLicenseFuture.get(); + assertNotNull(getLicenseResponse.license()); + assertThat(getLicenseResponse.license().operationMode(), equalTo(License.OperationMode.TRIAL)); + } + + public void testPutLicense() throws Exception { + License signedLicense = generateSignedLicense(TimeValue.timeValueMinutes(2)); + + // put license + PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(client().admin().cluster(), PutLicenseAction.INSTANCE) + .setLicense(signedLicense); + PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get(); + assertThat(putLicenseResponse.isAcknowledged(), equalTo(true)); + assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID)); + + // get and check license + GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(client().admin().cluster(), GetLicenseAction.INSTANCE).get(); + assertThat(getLicenseResponse.license(), equalTo(signedLicense)); + } + + public void testPutLicenseFromString() throws Exception { + License signedLicense = generateSignedLicense(TimeValue.timeValueMinutes(2)); + String licenseString = TestUtils.dumpLicense(signedLicense); + + // put license source + PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(client().admin().cluster(), PutLicenseAction.INSTANCE) + .setLicense(licenseString); + PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get(); + assertThat(putLicenseResponse.isAcknowledged(), equalTo(true)); + assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID)); + + // get and check license + GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(client().admin().cluster(), GetLicenseAction.INSTANCE).get(); + assertThat(getLicenseResponse.license(), equalTo(signedLicense)); + } + + public void testPutInvalidLicense() throws Exception { + License signedLicense = generateSignedLicense(TimeValue.timeValueMinutes(2)); + + // modify content of signed license + License tamperedLicense = License.builder() + .fromLicenseSpec(signedLicense, signedLicense.signature()) + .expiryDate(signedLicense.expiryDate() + 10 * 24 * 60 * 60 * 1000l) + .validate() + .build(); + + PutLicenseRequestBuilder builder = new PutLicenseRequestBuilder(client().admin().cluster(), PutLicenseAction.INSTANCE); + builder.setLicense(tamperedLicense); + + // try to put license (should be invalid) + final PutLicenseResponse putLicenseResponse = builder.get(); + assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.INVALID)); + + // try to get invalid license + GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(client().admin().cluster(), GetLicenseAction.INSTANCE).get(); + assertThat(getLicenseResponse.license(), not(tamperedLicense)); + } + + public void testPutExpiredLicense() throws Exception { + License expiredLicense = generateSignedLicense(dateMath("now-10d/d", System.currentTimeMillis()), TimeValue.timeValueMinutes(2)); + License signedLicense = generateSignedLicense(TimeValue.timeValueMinutes(2)); + + PutLicenseRequestBuilder builder = new PutLicenseRequestBuilder(client().admin().cluster(), PutLicenseAction.INSTANCE); + builder.setLicense(signedLicense); + // put license should return valid (as there is one valid license) + PutLicenseResponse putLicenseResponse = builder.get(); + assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID)); + builder.setLicense(expiredLicense); + putLicenseResponse = builder.get(); + assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.EXPIRED)); + // get license should not return the expired license + GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(client().admin().cluster(), GetLicenseAction.INSTANCE).get(); + assertThat(getLicenseResponse.license(), equalTo(signedLicense)); + } + + public void testPutLicensesSimple() throws Exception { + License basicSignedLicense = generateSignedLicense("basic", TimeValue.timeValueMinutes(5)); + PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(client().admin().cluster(), PutLicenseAction.INSTANCE) + .setLicense(basicSignedLicense); + PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get(); + assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID)); + GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(client().admin().cluster(), GetLicenseAction.INSTANCE).get(); + assertThat(getLicenseResponse.license(), equalTo(basicSignedLicense)); + + License platinumSignedLicense = generateSignedLicense("platinum", TimeValue.timeValueMinutes(2)); + putLicenseRequestBuilder.setLicense(platinumSignedLicense); + putLicenseResponse = putLicenseRequestBuilder.get(); + assertThat(putLicenseResponse.isAcknowledged(), equalTo(true)); + assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID)); + getLicenseResponse = new GetLicenseRequestBuilder(client().admin().cluster(), GetLicenseAction.INSTANCE).get(); + assertThat(getLicenseResponse.license(), equalTo(platinumSignedLicense)); + } + + public void testRemoveLicensesSimple() throws Exception { + License goldLicense = generateSignedLicense("gold", TimeValue.timeValueMinutes(5)); + PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(client().admin().cluster(), PutLicenseAction.INSTANCE) + .setLicense(goldLicense); + PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get(); + assertThat(putLicenseResponse.isAcknowledged(), equalTo(true)); + assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID)); + GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(client().admin().cluster(), GetLicenseAction.INSTANCE).get(); + assertThat(getLicenseResponse.license(), equalTo(goldLicense)); + // delete all licenses + DeleteLicenseRequestBuilder deleteLicenseRequestBuilder = new DeleteLicenseRequestBuilder(client().admin().cluster(), DeleteLicenseAction.INSTANCE); + DeleteLicenseResponse deleteLicenseResponse = deleteLicenseRequestBuilder.get(); + assertThat(deleteLicenseResponse.isAcknowledged(), equalTo(true)); + // get licenses (expected no licenses) + getLicenseResponse = new GetLicenseRequestBuilder(client().admin().cluster(), GetLicenseAction.INSTANCE).get(); + assertNull(getLicenseResponse.license()); + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/PutLicenseResponseTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/PutLicenseResponseTests.java new file mode 100644 index 00000000000..5bec19a549c --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/PutLicenseResponseTests.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.license.plugin.action.put.PutLicenseResponse; +import org.elasticsearch.license.plugin.core.LicensesStatus; +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class PutLicenseResponseTests extends ESTestCase { + public void testSerialization() throws Exception { + boolean acknowledged = randomBoolean(); + LicensesStatus status = randomFrom(LicensesStatus.VALID, LicensesStatus.INVALID, LicensesStatus.EXPIRED); + int nFeatures = randomIntBetween(1, 5); + Map ackMessages = new HashMap<>(); + for (int i = 0; i < nFeatures; i++) { + String feature = randomAsciiOfLengthBetween(9, 15); + int nMessages = randomIntBetween(1, 5); + String[] messages = new String[nMessages]; + for (int j = 0; j < nMessages; j++) { + messages[j] = randomAsciiOfLengthBetween(10, 30); + } + ackMessages.put(feature, messages); + } + + PutLicenseResponse response = new PutLicenseResponse(acknowledged, status, "", ackMessages); + XContentBuilder contentBuilder = XContentFactory.jsonBuilder(); + contentBuilder.startObject(); + response.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS); + contentBuilder.endObject(); + + Map map = XContentHelper.convertToMap(contentBuilder.bytesStream().bytes(), false).v2(); + assertThat(map.containsKey("acknowledged"), equalTo(true)); + boolean actualAcknowledged = (boolean) map.get("acknowledged"); + assertThat(actualAcknowledged, equalTo(acknowledged)); + + assertThat(map.containsKey("license_status"), equalTo(true)); + String actualStatus = (String) map.get("license_status"); + assertThat(actualStatus, equalTo(status.name().toLowerCase(Locale.ROOT))); + + assertThat(map.containsKey("acknowledge"), equalTo(true)); + Map> actualAckMessages = (HashMap) map.get("acknowledge"); + assertTrue(actualAckMessages.containsKey("message")); + actualAckMessages.remove("message"); + assertThat(actualAckMessages.keySet(), equalTo(ackMessages.keySet())); + for (Map.Entry> entry : actualAckMessages.entrySet()) { + assertArrayEquals(entry.getValue().toArray(), ackMessages.get(entry.getKey())); + } + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/TestUtils.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/TestUtils.java new file mode 100644 index 00000000000..8450120ca14 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/TestUtils.java @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.block.ClusterBlock; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.joda.DateMathParser; +import org.elasticsearch.common.joda.FormatDateTimeFormatter; +import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.licensor.LicenseSigner; +import org.elasticsearch.license.plugin.action.put.PutLicenseRequest; +import org.elasticsearch.license.plugin.core.*; +import org.hamcrest.Matchers; +import org.junit.Assert; + +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.test.ESTestCase.*; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; + +public class TestUtils { + + private final static FormatDateTimeFormatter formatDateTimeFormatter = Joda.forPattern("yyyy-MM-dd"); + private final static DateMathParser dateMathParser = new DateMathParser(formatDateTimeFormatter); + + public static long dateMath(String time, final long now) { + return dateMathParser.parse(time, new Callable() { + @Override + public Long call() throws Exception { + return now; + } + }); + } + public static Path getTestPriKeyPath() throws Exception { + return getResourcePath("/private.key"); + } + + public static Path getTestPubKeyPath() throws Exception { + return getResourcePath("/public.key"); + } + + public static void isSame(Collection firstLicenses, Collection secondLicenses) { + // check if the effective licenses have the same feature set + assertThat(firstLicenses.size(), equalTo(secondLicenses.size())); + + List license1 = new ArrayList<>(firstLicenses); + + CollectionUtil.timSort(license1, new Comparator() { + @Override + public int compare(License o1, License o2) { + return Long.compare(o1.expiryDate(), o2.expiryDate()); + } + }); + List license2 = new ArrayList<>(secondLicenses); + CollectionUtil.timSort(license2, new Comparator() { + @Override + public int compare(License o1, License o2) { + return Long.compare(o1.expiryDate(), o2.expiryDate()); + } + }); + assertThat(license1, equalTo(license2)); + } + + public static String dumpLicense(License license) throws Exception { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.startObject(); + builder.startObject("license"); + license.toInnerXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + builder.endObject(); + return builder.string(); + } + + public static License generateSignedLicense(TimeValue expiryDuration) throws Exception { + return generateSignedLicense(null, -1, expiryDuration); + } + + public static License generateSignedLicense(String type, TimeValue expiryDuration) throws Exception { + return generateSignedLicense(type, -1, expiryDuration); + } + + public static License generateSignedLicense(long issueDate, TimeValue expiryDuration) throws Exception { + return generateSignedLicense(null, issueDate, expiryDuration); + } + + public static License generateSignedLicense(String type, long issueDate, TimeValue expiryDuration) throws Exception { + long issue = (issueDate != -1l) ? issueDate : System.currentTimeMillis(); + int version = randomIntBetween(License.VERSION_START, License.VERSION_CURRENT); + final String licenseType; + if (version < License.VERSION_NO_FEATURE_TYPE) { + licenseType = randomFrom("subscription", "internal", "development"); + } else { + licenseType = (type != null) ? type : randomFrom("basic", "silver", "dev", "gold", "platinum"); + } + final License.Builder builder = License.builder() + .uid(UUID.randomUUID().toString()) + .version(version) + .expiryDate(issue + expiryDuration.getMillis()) + .issueDate(issue) + .type(licenseType) + .issuedTo("customer") + .issuer("elasticsearch") + .maxNodes(5); + if (version == License.VERSION_START) { + builder.subscriptionType((type != null) ? type : randomFrom("dev", "gold", "platinum", "silver")); + builder.feature(randomAsciiOfLength(10)); + } + LicenseSigner signer = new LicenseSigner(getTestPriKeyPath(), getTestPubKeyPath()); + return signer.sign(builder.build()); + } + + private static Path getResourcePath(String resource) throws Exception { + return PathUtils.get(TestUtils.class.getResource(resource).toURI()); + } + + public static void awaitNoBlock(final Client client) throws InterruptedException { + boolean success = awaitBusy(() -> { + Set clusterBlocks = client.admin().cluster().prepareState().setLocal(true).execute().actionGet() + .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE); + return clusterBlocks.isEmpty(); + }); + assertThat("awaiting no block for too long", success, equalTo(true)); + } + + public static void awaitNoPendingTasks(final Client client) throws InterruptedException { + boolean success = awaitBusy(() -> client.admin().cluster().preparePendingClusterTasks().get().getPendingTasks().isEmpty()); + assertThat("awaiting no pending tasks for too long", success, equalTo(true)); + } + + public static void registerAndAckSignedLicenses(final LicensesService licensesService, License license, final LicensesStatus expectedStatus) { + PutLicenseRequest putLicenseRequest = new PutLicenseRequest().license(license); + final CountDownLatch latch = new CountDownLatch(1); + final AtomicReference status = new AtomicReference<>(); + licensesService.registerLicense(putLicenseRequest, new ActionListener() { + @Override + public void onResponse(LicensesService.LicensesUpdateResponse licensesUpdateResponse) { + status.set(licensesUpdateResponse.status()); + latch.countDown(); + } + + @Override + public void onFailure(Throwable e) { + latch.countDown(); + } + }); + try { + latch.await(); + } catch (InterruptedException e) { + Assert.fail(e.getMessage()); + } + assertThat(status.get(), equalTo(expectedStatus)); + } + + public static class AssertingLicensee implements Licensee { + public final ESLogger logger; + public final String id; + public final List licenseStates = new CopyOnWriteArrayList<>(); + public final AtomicInteger expirationMessagesCalled = new AtomicInteger(0); + public final List> acknowledgementRequested = new CopyOnWriteArrayList<>(); + + private String[] acknowledgmentMessages = new String[0]; + + public AssertingLicensee(String id, ESLogger logger) { + this.logger = logger; + this.id = id; + } + + public void setAcknowledgementMessages(String[] acknowledgementMessages) { + this.acknowledgmentMessages = acknowledgementMessages; + } + @Override + public String id() { + return id; + } + + @Override + public String[] expirationMessages() { + expirationMessagesCalled.incrementAndGet(); + return new String[0]; + } + + @Override + public String[] acknowledgmentMessages(License currentLicense, License newLicense) { + acknowledgementRequested.add(new Tuple<>(currentLicense, newLicense)); + return acknowledgmentMessages; + } + + @Override + public void onChange(Status status) { + assertNotNull(status); + licenseStates.add(status.getLicenseState()); + } + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/TrialLicenseTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/TrialLicenseTests.java new file mode 100644 index 00000000000..f1f20d3f7a3 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/TrialLicenseTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin; + +import org.elasticsearch.common.Base64; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.plugin.core.TrialLicense; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.UUID; + +import static org.elasticsearch.license.core.CryptUtils.encrypt; +import static org.hamcrest.Matchers.equalTo; + + +public class TrialLicenseTests extends ESTestCase { + public void testBasic() throws Exception { + long issueDate = System.currentTimeMillis(); + License.Builder specBuilder = License.builder() + .uid(UUID.randomUUID().toString()) + .issuedTo("customer") + .maxNodes(5) + .issueDate(issueDate) + .expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis()); + License trialLicense = TrialLicense.create(specBuilder); + assertThat(TrialLicense.verify(trialLicense), equalTo(true)); + } + + public void testTampered() throws Exception { + long issueDate = System.currentTimeMillis(); + License.Builder specBuilder = License.builder() + .uid(UUID.randomUUID().toString()) + .issuedTo("customer") + .maxNodes(5) + .issueDate(issueDate) + .expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis()); + License trialLicense = TrialLicense.create(specBuilder); + final String originalSignature = trialLicense.signature(); + License tamperedLicense = License.builder().fromLicenseSpec(trialLicense, originalSignature) + .expiryDate(System.currentTimeMillis() + TimeValue.timeValueHours(5).getMillis()) + .build(); + assertThat(TrialLicense.verify(trialLicense), equalTo(true)); + assertThat(TrialLicense.verify(tamperedLicense), equalTo(false)); + } + + public void testFrom1x() throws Exception { + long issueDate = System.currentTimeMillis(); + License.Builder specBuilder = License.builder() + .uid(UUID.randomUUID().toString()) + .issuedTo("customer") + .type("subscription") + .subscriptionType("trial") + .issuer("elasticsearch") + .feature("") + .version(License.VERSION_START) + .maxNodes(5) + .issueDate(issueDate) + .expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis()); + License pre20TrialLicense = specBuilder.build(); + License license = TrialLicense.create(License.builder().fromPre20LicenseSpec(pre20TrialLicense)); + assertThat(TrialLicense.verify(license), equalTo(true)); + } + + public void testTrialLicenseVerifyWithOlderVersion() throws Exception { + long issueDate = System.currentTimeMillis(); + License.Builder specBuilder = License.builder() + .issuedTo("customer") + .maxNodes(5) + .issueDate(issueDate) + .expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis()) + .feature("") + .subscriptionType("trial") + .version(1); + License trialLicenseV1 = createTrialLicense(specBuilder); + assertThat(TrialLicense.verify(trialLicenseV1), equalTo(true)); + } + + static License createTrialLicense(License.Builder specBuilder) { + License spec = specBuilder + .type("trial") + .issuer("elasticsearch") + .uid(UUID.randomUUID().toString()) + .build(); + final String signature; + try { + XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + spec.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true"))); + byte[] encrypt = encrypt(contentBuilder.bytes().toBytes()); + byte[] bytes = new byte[4 + 4 + encrypt.length]; + ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); + byteBuffer.putInt(-spec.version()) + .putInt(encrypt.length) + .put(encrypt); + signature = Base64.encodeBytes(bytes); + } catch (IOException e) { + throw new IllegalStateException(e); + } + return License.builder().fromLicenseSpec(spec, signature).build(); + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationConsumerPlugin.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationConsumerPlugin.java new file mode 100644 index 00000000000..bd003f279bf --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationConsumerPlugin.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.consumer; + +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; + +/** + * Registers licenses upon the start of the service lifecycle + * see {@link EagerLicenseRegistrationPluginService} + *

+ * License registration might happen before clusterService start() + */ +public class EagerLicenseRegistrationConsumerPlugin extends TestConsumerPluginBase { + + public final static String NAME = "test_consumer_plugin_1"; + + @Inject + public EagerLicenseRegistrationConsumerPlugin(Settings settings) { + super(settings); + } + + @Override + public Class service() { + return EagerLicenseRegistrationPluginService.class; + } + + @Override + protected String pluginName() { + return NAME; + } + + @Override + public String id() { + return EagerLicenseRegistrationPluginService.ID; + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationPluginService.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationPluginService.java new file mode 100644 index 00000000000..2afc647ad69 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationPluginService.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.consumer; + +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.Singleton; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.core.LicensesService; + +@Singleton +public class EagerLicenseRegistrationPluginService extends TestPluginServiceBase { + + public static String ID = "id1"; + + @Inject + public EagerLicenseRegistrationPluginService(Settings settings, LicensesService licensesClientService) { + super(true, settings, licensesClientService, null); + } + + @Override + public String id() { + return ID; + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationConsumerPlugin.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationConsumerPlugin.java new file mode 100644 index 00000000000..f8902f0da40 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationConsumerPlugin.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.consumer; + +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; + +/** + * Registers licenses only after cluster has recovered + * see {@link LazyLicenseRegistrationPluginService} + *

+ * License registration happens after clusterservice start() + */ +public class LazyLicenseRegistrationConsumerPlugin extends TestConsumerPluginBase { + + public static String NAME = "test_consumer_plugin_2"; + + @Inject + public LazyLicenseRegistrationConsumerPlugin(Settings settings) { + super(settings); + } + + @Override + public Class service() { + return LazyLicenseRegistrationPluginService.class; + } + + @Override + protected String pluginName() { + return NAME; + } + + @Override + public String id() { + return LazyLicenseRegistrationPluginService.ID; + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationPluginService.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationPluginService.java new file mode 100644 index 00000000000..72f2470e52b --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationPluginService.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.consumer; + +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.Singleton; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.core.LicensesService; + +@Singleton +public class LazyLicenseRegistrationPluginService extends TestPluginServiceBase { + + public static String ID = "id2"; + + @Inject + public LazyLicenseRegistrationPluginService(Settings settings, LicensesService licensesClientService, ClusterService clusterService) { + super(false, settings, licensesClientService, clusterService); + } + + @Override + public String id() { + return ID; + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/TestConsumerPluginBase.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/TestConsumerPluginBase.java new file mode 100644 index 00000000000..b2106920e02 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/TestConsumerPluginBase.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.consumer; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.component.LifecycleComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; + +import java.util.ArrayList; +import java.util.Collection; + +public abstract class TestConsumerPluginBase extends Plugin { + + private final boolean isEnabled; + + public TestConsumerPluginBase(Settings settings) { + if (DiscoveryNode.clientNode(settings)) { + // Enable plugin only on node clients + this.isEnabled = "node".equals(settings.get(Client.CLIENT_TYPE_SETTING)); + } else { + this.isEnabled = true; + } + } + + @Override + public String name() { + return pluginName(); + } + + @Override + public String description() { + return "test licensing consumer plugin"; + } + + + @Override + public Collection> nodeServices() { + Collection> services = new ArrayList<>(); + if (isEnabled) { + services.add(service()); + } + return services; + } + + public abstract Class service(); + + protected abstract String pluginName(); + + public abstract String id(); +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/TestPluginServiceBase.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/TestPluginServiceBase.java new file mode 100644 index 00000000000..94008744700 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/consumer/TestPluginServiceBase.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.consumer; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.license.plugin.core.Licensee; +import org.elasticsearch.license.plugin.core.LicensesService; + +import java.util.Collections; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +public abstract class TestPluginServiceBase extends AbstractLifecycleComponent implements ClusterStateListener, Licensee { + + private LicensesService licensesClientService; + private final ClusterService clusterService; + final boolean eagerLicenseRegistration; + public final AtomicBoolean registered = new AtomicBoolean(false); + private AtomicReference state = new AtomicReference<>(LicenseState.DISABLED); + + public TestPluginServiceBase(boolean eagerLicenseRegistration, Settings settings, LicensesService licensesClientService, ClusterService clusterService) { + super(settings); + this.eagerLicenseRegistration = eagerLicenseRegistration; + this.licensesClientService = licensesClientService; + int trialDurationInSec = settings.getAsInt(".trial_license_duration_in_seconds", -1); + if (trialDurationInSec != -1) { + licensesClientService.setTrialLicenseDuration(TimeValue.timeValueSeconds(trialDurationInSec)); + } + int graceDurationInSec = settings.getAsInt(".grace_duration_in_seconds", 5); + licensesClientService.setGracePeriodDuration(TimeValue.timeValueSeconds(graceDurationInSec)); + if (!eagerLicenseRegistration) { + this.clusterService = clusterService; + clusterService.add(this); + } else { + this.clusterService = null; + } + } + + // should be the same string used by the license Manger to generate + // signed license + public abstract String id(); + + // check if feature is enabled + public LicenseState state() { + return state.get(); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (!eagerLicenseRegistration && !event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + if (registered.compareAndSet(false, true)) { + logger.info("Registering to licensesService [lazy]"); + licensesClientService.register(this); + } + } + } + + protected void doStart() throws ElasticsearchException { + if (eagerLicenseRegistration) { + if (registered.compareAndSet(false, true)) { + logger.info("Registering to licensesService [eager]"); + licensesClientService.register(this); + } + } + } + + @Override + public String[] expirationMessages() { + return new String[0]; + } + + @Override + public String[] acknowledgmentMessages(License currentLicense, License newLicense) { + return new String[0]; + } + + @Override + public void onChange(Status status) { + this.state.set(status.getLicenseState()); + } + + @Override + protected void doStop() throws ElasticsearchException { + if (clusterService != null) { + clusterService.remove(this); + } + } + + @Override + protected void doClose() throws ElasticsearchException { + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesAcknowledgementTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesAcknowledgementTests.java new file mode 100644 index 00000000000..6beda195dce --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesAcknowledgementTests.java @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.plugin.TestUtils; +import org.elasticsearch.license.plugin.action.put.PutLicenseRequest; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.license.plugin.TestUtils.awaitNoBlock; +import static org.elasticsearch.license.plugin.TestUtils.awaitNoPendingTasks; +import static org.elasticsearch.license.plugin.TestUtils.generateSignedLicense; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +public class LicensesAcknowledgementTests extends ESSingleNodeTestCase { + static { + MetaData.registerPrototype(LicensesMetaData.TYPE, LicensesMetaData.PROTO); + } + + @Override + protected boolean resetNodeAfterTest() { + return true; + } + + public void testAcknowledgment() throws Exception { + final LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.start(); + String id = "testAcknowledgment"; + String[] acknowledgeMessages = new String[] {"message"}; + TestUtils.AssertingLicensee licensee = new TestUtils.AssertingLicensee(id, logger); + licensee.setAcknowledgementMessages(acknowledgeMessages); + awaitNoBlock(client()); + licensesService.register(licensee); + awaitNoPendingTasks(client()); + // try installing a signed license + License signedLicense = generateSignedLicense(TimeValue.timeValueHours(10)); + PutLicenseRequest putLicenseRequest = new PutLicenseRequest().license(signedLicense); + CountDownLatch latch = new CountDownLatch(1); + // ensure acknowledgement message was part of the response + licensesService.registerLicense(putLicenseRequest, new AssertingLicensesUpdateResponse(false, LicensesStatus.VALID, + Collections.singletonMap(id, acknowledgeMessages), latch)); + if (!latch.await(5, TimeUnit.SECONDS)) { + fail("waiting too long for a response to license registration"); + } + awaitNoPendingTasks(client()); + assertThat(licensee.acknowledgementRequested.size(), equalTo(1)); + assertThat(licensee.acknowledgementRequested.get(0).v2(), equalTo(signedLicense)); + assertThat(licensesService.getLicense(), not(signedLicense)); + + latch = new CountDownLatch(1); + // try installing a signed license with acknowledgement + putLicenseRequest = new PutLicenseRequest().license(signedLicense).acknowledge(true); + // ensure license was installed and no acknowledgment message was returned + licensee.setAcknowledgementMessages(new String[0]); + licensesService.registerLicense(putLicenseRequest, new AssertingLicensesUpdateResponse(true, LicensesStatus.VALID, Collections.emptyMap(), latch)); + if (!latch.await(5, TimeUnit.SECONDS)) { + fail("waiting too long for a response to license registration"); + } + awaitNoPendingTasks(client()); + assertThat(licensee.acknowledgementRequested.size(), equalTo(1)); + assertThat(licensee.acknowledgementRequested.get(0).v2(), equalTo(signedLicense)); + assertThat(licensesService.getLicense(), equalTo(signedLicense)); + licensesService.stop(); + } + + public void testAcknowledgementMultipleLicensee() throws Exception { + final LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.start(); + String id1 = "testAcknowledgementMultipleLicensee_1"; + String[] acknowledgeMessages1 = new String[] {"testAcknowledgementMultipleLicensee_1"}; + String id2 = "testAcknowledgementMultipleLicensee_2"; + String[] acknowledgeMessages2 = new String[] {"testAcknowledgementMultipleLicensee_2"}; + TestUtils.AssertingLicensee licensee1 = new TestUtils.AssertingLicensee(id1, logger); + licensee1.setAcknowledgementMessages(acknowledgeMessages1); + TestUtils.AssertingLicensee licensee2 = new TestUtils.AssertingLicensee(id2, logger); + licensee2.setAcknowledgementMessages(acknowledgeMessages2); + awaitNoBlock(client()); + licensesService.register(licensee1); + licensesService.register(licensee2); + awaitNoPendingTasks(client()); + // try installing a signed license + License signedLicense = generateSignedLicense(TimeValue.timeValueHours(10)); + PutLicenseRequest putLicenseRequest = new PutLicenseRequest().license(signedLicense); + CountDownLatch latch = new CountDownLatch(1); + // ensure acknowledgement message was part of the response + final HashMap expectedMessages = new HashMap<>(); + expectedMessages.put(id1, acknowledgeMessages1); + expectedMessages.put(id2, acknowledgeMessages2); + licensesService.registerLicense(putLicenseRequest, new AssertingLicensesUpdateResponse(false, LicensesStatus.VALID, + expectedMessages, latch)); + if (!latch.await(5, TimeUnit.SECONDS)) { + fail("waiting too long for a response to license registration"); + } + awaitNoPendingTasks(client()); + assertThat(licensee2.acknowledgementRequested.size(), equalTo(1)); + assertThat(licensee2.acknowledgementRequested.get(0).v2(), equalTo(signedLicense)); + assertThat(licensee1.acknowledgementRequested.size(), equalTo(1)); + assertThat(licensee1.acknowledgementRequested.get(0).v2(), equalTo(signedLicense)); + assertThat(licensesService.getLicense(), not(signedLicense)); + + latch = new CountDownLatch(1); + // try installing a signed license with acknowledgement + putLicenseRequest = new PutLicenseRequest().license(signedLicense).acknowledge(true); + // ensure license was installed and no acknowledgment message was returned + licensee1.setAcknowledgementMessages(new String[0]); + licensee2.setAcknowledgementMessages(new String[0]); + licensesService.registerLicense(putLicenseRequest, new AssertingLicensesUpdateResponse(true, LicensesStatus.VALID, Collections.emptyMap(), latch)); + if (!latch.await(5, TimeUnit.SECONDS)) { + fail("waiting too long for a response to license registration"); + } + awaitNoPendingTasks(client()); + assertThat(licensesService.getLicense(), equalTo(signedLicense)); + licensesService.stop(); + } + + private static class AssertingLicensesUpdateResponse implements ActionListener { + private final boolean expectedAcknowledgement; + private final LicensesStatus expectedStatus; + private final Map expectedAckMessages; + private final CountDownLatch latch; + + public AssertingLicensesUpdateResponse(boolean expectedAcknowledgement, LicensesStatus expectedStatus, Map expectedAckMessages, CountDownLatch latch) { + this.expectedAcknowledgement = expectedAcknowledgement; + this.expectedStatus = expectedStatus; + this.expectedAckMessages = expectedAckMessages; + this.latch = latch; + } + + @Override + public void onResponse(LicensesService.LicensesUpdateResponse licensesUpdateResponse) { + assertThat(licensesUpdateResponse.isAcknowledged(), equalTo(expectedAcknowledgement)); + assertThat(licensesUpdateResponse.status(), equalTo(expectedStatus)); + assertThat(licensesUpdateResponse.acknowledgeMessages().size(), equalTo(expectedAckMessages.size())); + for (Map.Entry expectedEntry : expectedAckMessages.entrySet()) { + Map actual = licensesUpdateResponse.acknowledgeMessages(); + assertThat(actual.containsKey(expectedEntry.getKey()), equalTo(true)); + String[] actualMessages = actual.get(expectedEntry.getKey()); + assertThat(actualMessages, equalTo(expectedEntry.getValue())); + } + latch.countDown(); + } + + @Override + public void onFailure(Throwable throwable) { + latch.countDown(); + } + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesExpirationCallbackTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesExpirationCallbackTests.java new file mode 100644 index 00000000000..f7d2da9bf2c --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesExpirationCallbackTests.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.plugin.TestUtils; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.Matchers.equalTo; + +public class LicensesExpirationCallbackTests extends ESSingleNodeTestCase { + static { + MetaData.registerPrototype(LicensesMetaData.TYPE, LicensesMetaData.PROTO); + } + + @Override + protected boolean resetNodeAfterTest() { + return true; + } + + public void testPostExpiration() throws Exception { + int postExpirySeconds = randomIntBetween(5, 10); + TimeValue postExpiryDuration = TimeValue.timeValueSeconds(postExpirySeconds); + TimeValue min = TimeValue.timeValueSeconds(postExpirySeconds - randomIntBetween(1, 3)); + TimeValue max = TimeValue.timeValueSeconds(postExpirySeconds + randomIntBetween(1, 10)); + + final LicensesService.ExpirationCallback.Post post = new LicensesService.ExpirationCallback.Post(min, max, TimeValue.timeValueMillis(10)) { + @Override + public void on(License license) { + } + }; + long now = System.currentTimeMillis(); + assertThat(post.matches(now - postExpiryDuration.millis(), now), equalTo(true)); + assertThat(post.matches(now + postExpiryDuration.getMillis(), now), equalTo(false)); + } + + public void testPostExpirationWithNullMax() throws Exception { + int postExpirySeconds = randomIntBetween(5, 10); + TimeValue postExpiryDuration = TimeValue.timeValueSeconds(postExpirySeconds); + TimeValue min = TimeValue.timeValueSeconds(postExpirySeconds - randomIntBetween(1, 3)); + + final LicensesService.ExpirationCallback.Post post = new LicensesService.ExpirationCallback.Post(min, null, TimeValue.timeValueMillis(10)) { + @Override + public void on(License license) { + } + }; + long now = System.currentTimeMillis(); + assertThat(post.matches(now - postExpiryDuration.millis(), now), equalTo(true)); + } + + public void testPreExpirationWithNullMin() throws Exception { + int expirySeconds = randomIntBetween(5, 10); + TimeValue expiryDuration = TimeValue.timeValueSeconds(expirySeconds); + TimeValue max = TimeValue.timeValueSeconds(expirySeconds + randomIntBetween(1, 10)); + + final LicensesService.ExpirationCallback.Pre pre = new LicensesService.ExpirationCallback.Pre(null, max, TimeValue.timeValueMillis(10)) { + @Override + public void on(License license) { + } + }; + long now = System.currentTimeMillis(); + assertThat(pre.matches(expiryDuration.millis() + now, now), equalTo(true)); + } + + public void testPreExpiration() throws Exception { + int expirySeconds = randomIntBetween(5, 10); + TimeValue expiryDuration = TimeValue.timeValueSeconds(expirySeconds); + TimeValue min = TimeValue.timeValueSeconds(expirySeconds - randomIntBetween(0, 3)); + TimeValue max = TimeValue.timeValueSeconds(expirySeconds + randomIntBetween(1, 10)); + + final LicensesService.ExpirationCallback.Pre pre = new LicensesService.ExpirationCallback.Pre(min, max, TimeValue.timeValueMillis(10)) { + @Override + public void on(License license) { + } + }; + long now = System.currentTimeMillis(); + assertThat(pre.matches(expiryDuration.millis() + now, now), equalTo(true)); + assertThat(pre.matches(now - expiryDuration.getMillis(), now), equalTo(false)); + } + + public void testPreExpirationNotification() throws Exception { + final LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.setTrialLicenseDuration(TimeValue.timeValueSeconds(5)); + AtomicInteger counter = new AtomicInteger(0); + licensesService.setExpirationCallbacks(Collections.singletonList( + preCallbackLatch(TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(2), TimeValue.timeValueMillis(400), counter)) // 2000, 1600, 1200 + ); + licensesService.start(); + TestUtils.AssertingLicensee licensee = new TestUtils.AssertingLicensee("testPreExpirationNotification", logger); + licensesService.register(licensee); + boolean success = awaitBusy(() -> (counter.get() == 3 || counter.get() == 2)); + assertThat("counter: actual: " + counter.get() + "vs expected: 3", success, equalTo(true)); + licensesService.stop(); + } + + public void testPostExpirationNotification() throws Exception { + final LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.setTrialLicenseDuration(TimeValue.timeValueSeconds(3)); + AtomicInteger counter = new AtomicInteger(0); + licensesService.setExpirationCallbacks(Collections.singletonList( + postCallbackLatch(TimeValue.timeValueMillis(700), TimeValue.timeValueSeconds(3), TimeValue.timeValueSeconds(1), counter)) // 700, 1700, 2700 + ); + licensesService.start(); + TestUtils.AssertingLicensee licensee = new TestUtils.AssertingLicensee("testPostExpirationNotification", logger); + licensesService.register(licensee); + // callback can be called only twice if the third notification is triggered with a delay + // causing the trigger time to be out of the post expiry callback window + boolean success = awaitBusy(() -> (counter.get() == 3 || counter.get() == 2)); + assertThat("counter: actual: " + counter.get() + "vs expected: 3", success, equalTo(true)); + licensesService.stop(); + } + + public void testMultipleExpirationNotification() throws Exception { + final LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.setTrialLicenseDuration(TimeValue.timeValueSeconds(4)); + AtomicInteger postCounter = new AtomicInteger(0); + AtomicInteger preCounter = new AtomicInteger(0); + licensesService.setExpirationCallbacks(Arrays.asList( + preCallbackLatch(TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(2), TimeValue.timeValueMillis(400), preCounter), // 2000, 1600, 1200 + postCallbackLatch(TimeValue.timeValueMillis(100), TimeValue.timeValueSeconds(2), TimeValue.timeValueMillis(400), postCounter)) // 100, 500, 900, 1300, 1700 + ); + licensesService.start(); + TestUtils.AssertingLicensee licensee = new TestUtils.AssertingLicensee("testMultipleExpirationNotification", logger); + licensesService.register(licensee); + // callback can be called one less than expected if the last notification is triggered + // with a delay, causing the trigger time to be out of the expiry callback window + boolean success = awaitBusy(() -> ((preCounter.get() == 3 || preCounter.get() == 2) + && (postCounter.get() == 5 || postCounter.get() == 4))); + assertThat("post count: actual: " + postCounter.get() + "vs expected: 5 " + + "pre count: actual: " + preCounter.get() + " vs expected: 3", success, equalTo(true)); + licensesService.stop(); + } + + private static LicensesService.ExpirationCallback preCallbackLatch(TimeValue min, TimeValue max, TimeValue frequency, final AtomicInteger count) { + return new LicensesService.ExpirationCallback.Pre(min, max, frequency) { + @Override + public void on(License license) { + count.incrementAndGet(); + } + }; + } + + private static LicensesService.ExpirationCallback postCallbackLatch(TimeValue min, TimeValue max, TimeValue frequency, final AtomicInteger count) { + return new LicensesService.ExpirationCallback.Post(min, max, frequency) { + @Override + public void on(License license) { + count.incrementAndGet(); + } + }; + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesExpiryNotificationTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesExpiryNotificationTests.java new file mode 100644 index 00000000000..7ba8b8375c5 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesExpiryNotificationTests.java @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.apache.lucene.util.LuceneTestCase.BadApple; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.license.plugin.TestUtils.AssertingLicensee; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.List; + +import static org.elasticsearch.license.plugin.TestUtils.awaitNoBlock; +import static org.elasticsearch.license.plugin.TestUtils.awaitNoPendingTasks; +import static org.elasticsearch.license.plugin.TestUtils.generateSignedLicense; +import static org.elasticsearch.license.plugin.TestUtils.registerAndAckSignedLicenses; +import static org.hamcrest.Matchers.equalTo; + +//test is just too slow, please fix it to not be sleep-based +@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") +public class LicensesExpiryNotificationTests extends ESSingleNodeTestCase { + static { + MetaData.registerPrototype(LicensesMetaData.TYPE, LicensesMetaData.PROTO); + } + + @Override + protected boolean resetNodeAfterTest() { + return true; + } + + public void testTrialLicenseEnforcement() throws Exception { + LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.setTrialLicenseDuration(TimeValue.timeValueSeconds(5)); + licensesService.setGracePeriodDuration(TimeValue.timeValueSeconds(3)); + licensesService.start(); + String id1 = "testTrialLicenseEnforcement"; + final AssertingLicensee licensee = new AssertingLicensee(id1, logger); + awaitNoBlock(client()); + licensesService.register(licensee); + awaitNoPendingTasks(client()); + boolean success = awaitBusy(() -> licensee.licenseStates.size() == 3); + // trail license: enable, grace, disabled + assertLicenseStates(licensee, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED); + assertTrue(dumpLicensingStates(licensee.licenseStates), success); + licensesService.stop(); + } + + public void testTrialLicenseEnforcementMultipleLicensees() throws Exception { + LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.setTrialLicenseDuration(TimeValue.timeValueSeconds(5)); + licensesService.setGracePeriodDuration(TimeValue.timeValueSeconds(3)); + licensesService.start(); + String id1 = "testTrialLicenseEnforcementMultipleLicensees_1"; + final AssertingLicensee licensee1 = new AssertingLicensee(id1, logger); + String id12 = "testTrialLicenseEnforcementMultipleLicensees_2"; + final AssertingLicensee licensee2 = new AssertingLicensee(id12, logger); + awaitNoBlock(client()); + licensesService.register(licensee1); + licensesService.register(licensee2); + awaitNoPendingTasks(client()); + boolean success = awaitBusy(() -> licensee1.licenseStates.size() == 3); + assertTrue(dumpLicensingStates(licensee1.licenseStates), success); + success = awaitBusy(() -> licensee2.licenseStates.size() == 3); + assertTrue(dumpLicensingStates(licensee2.licenseStates), success); + // trail license: enable, grace, disabled + assertLicenseStates(licensee1, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED); + assertLicenseStates(licensee2, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED); + licensesService.stop(); + } + + public void testTrialSignedLicenseEnforcement() throws Exception { + LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.setTrialLicenseDuration(TimeValue.timeValueSeconds(2)); + licensesService.setGracePeriodDuration(TimeValue.timeValueSeconds(3)); + licensesService.start(); + String id1 = "testTrialSignedLicenseEnforcement"; + final AssertingLicensee licensee = new AssertingLicensee(id1, logger); + awaitNoBlock(client()); + licensesService.register(licensee); + awaitNoPendingTasks(client()); + boolean success = awaitBusy(() -> licensee.licenseStates.size() == 1); + assertTrue(dumpLicensingStates(licensee.licenseStates), success); + registerAndAckSignedLicenses(licensesService, generateSignedLicense(TimeValue.timeValueSeconds(4)), LicensesStatus.VALID); + success = awaitBusy(() -> licensee.licenseStates.size() == 4); + // trial: enable, signed: enable, signed: grace, signed: disabled + assertLicenseStates(licensee, LicenseState.ENABLED, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED); + assertTrue(dumpLicensingStates(licensee.licenseStates), success); + licensesService.stop(); + } + + public void testSignedLicenseEnforcement() throws Exception { + LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.setTrialLicenseDuration(TimeValue.timeValueSeconds(4)); + licensesService.setGracePeriodDuration(TimeValue.timeValueSeconds(3)); + licensesService.start(); + String id1 = "testSignedLicenseEnforcement"; + final AssertingLicensee licensee = new AssertingLicensee(id1, logger); + awaitNoBlock(client()); + registerAndAckSignedLicenses(licensesService, generateSignedLicense(TimeValue.timeValueSeconds(2)), LicensesStatus.VALID); + licensesService.register(licensee); + awaitNoPendingTasks(client()); + boolean success = awaitBusy(() -> licensee.licenseStates.size() == 3); + // signed: enable, signed: grace, signed: disabled + assertLicenseStates(licensee, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED); + assertTrue(dumpLicensingStates(licensee.licenseStates), success); + licensesService.stop(); + } + + public void testSingedLicenseEnforcementMultipleLicensees() throws Exception { + LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.setTrialLicenseDuration(TimeValue.timeValueSeconds(4)); + licensesService.setGracePeriodDuration(TimeValue.timeValueSeconds(3)); + licensesService.start(); + String id1 = "testSingedLicenseEnforcementMultipleLicensees_1"; + final AssertingLicensee licensee1 = new AssertingLicensee(id1, logger); + String id12 = "testSingedLicenseEnforcementMultipleLicensees_2"; + final AssertingLicensee licensee2 = new AssertingLicensee(id12, logger); + awaitNoBlock(client()); + registerAndAckSignedLicenses(licensesService, generateSignedLicense(TimeValue.timeValueSeconds(2)), LicensesStatus.VALID); + licensesService.register(licensee1); + licensesService.register(licensee2); + awaitNoPendingTasks(client()); + boolean success = awaitBusy(() -> licensee1.licenseStates.size() == 3); + assertTrue(dumpLicensingStates(licensee1.licenseStates), success); + success = awaitBusy(() -> licensee2.licenseStates.size() == 3); + assertTrue(dumpLicensingStates(licensee2.licenseStates), success); + // signed license: enable, grace, disabled + assertLicenseStates(licensee1, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED); + assertLicenseStates(licensee2, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED); + licensesService.stop(); + } + + public void testMultipleSignedLicenseEnforcement() throws Exception { + // register with trial license and assert onEnable and onDisable notification + LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.setTrialLicenseDuration(TimeValue.timeValueSeconds(4)); + licensesService.setGracePeriodDuration(TimeValue.timeValueSeconds(1)); + licensesService.start(); + String id1 = "testMultipleSignedLicenseEnforcement"; + final AssertingLicensee licensee = new AssertingLicensee(id1, logger); + awaitNoBlock(client()); + licensesService.register(licensee); + awaitNoPendingTasks(client()); + // trial license enabled + boolean success = awaitBusy(() -> licensee.licenseStates.size() == 1); + assertTrue(dumpLicensingStates(licensee.licenseStates), success); + registerAndAckSignedLicenses(licensesService, generateSignedLicense("basic", TimeValue.timeValueSeconds(3)), LicensesStatus.VALID); + // signed license enabled + success = awaitBusy(() -> licensee.licenseStates.size() == 2); + assertTrue(dumpLicensingStates(licensee.licenseStates), success); + registerAndAckSignedLicenses(licensesService, generateSignedLicense("gold", TimeValue.timeValueSeconds(2)), LicensesStatus.VALID); + // second signed license enabled, grace and expired + success = awaitBusy(() ->licensee.licenseStates.size() == 5); + assertLicenseStates(licensee, LicenseState.ENABLED, LicenseState.ENABLED, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED); + assertTrue(dumpLicensingStates(licensee.licenseStates), success); + licensesService.stop(); + } + + public void testNonOverlappingMultipleLicensesEnforcement() throws Exception { + // register with trial license and assert onEnable and onDisable notification + LicensesService licensesService = getInstanceFromNode(LicensesService.class); + licensesService.setTrialLicenseDuration(TimeValue.timeValueSeconds(3)); + licensesService.setGracePeriodDuration(TimeValue.timeValueSeconds(1)); + licensesService.start(); + String id1 = "testNonOverlappingMultipleLicensesEnforcement"; + final AssertingLicensee licensee = new AssertingLicensee(id1, logger); + awaitNoBlock(client()); + licensesService.register(licensee); + // trial license: enabled, grace, disabled + boolean success = awaitBusy(() -> licensee.licenseStates.size() == 3); + + assertTrue(dumpLicensingStates(licensee.licenseStates), success); + // install license + registerAndAckSignedLicenses(licensesService, generateSignedLicense("basic", TimeValue.timeValueSeconds(2)), LicensesStatus.VALID); + // trial license: enabled, grace, disabled, signed license: enabled, grace, disabled + success = awaitBusy(() -> licensee.licenseStates.size() == 6); + assertLicenseStates(licensee, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED); + assertTrue(dumpLicensingStates(licensee.licenseStates), success); + licensesService.stop(); + } + + private void assertLicenseStates(AssertingLicensee licensee, LicenseState... states) { + StringBuilder msg = new StringBuilder(); + msg.append("Actual: "); + msg.append(dumpLicensingStates(licensee.licenseStates)); + msg.append(" Expected: "); + msg.append(dumpLicensingStates(states)); + assertThat(msg.toString(), licensee.licenseStates.size(), equalTo(states.length)); + for (int i = 0; i < states.length; i++) { + assertThat(msg.toString(), licensee.licenseStates.get(i), equalTo(states[i])); + } + } + + private String dumpLicensingStates(List states) { + return dumpLicensingStates(states.toArray(new LicenseState[states.size()])); + } + + private String dumpLicensingStates(LicenseState... states) { + StringBuilder sb = new StringBuilder(); + sb.append("["); + for (int i = 0; i < states.length; i++) { + sb.append(states[i].name()); + if (i != states.length - 1) { + sb.append(", "); + } + } + sb.append("]"); + return sb.toString(); + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesManagerServiceTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesManagerServiceTests.java new file mode 100644 index 00000000000..993b2f89d4a --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesManagerServiceTests.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.plugin.TestUtils; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseRequest; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.license.plugin.TestUtils.generateSignedLicense; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +public class LicensesManagerServiceTests extends ESSingleNodeTestCase { + static { + MetaData.registerPrototype(LicensesMetaData.TYPE, LicensesMetaData.PROTO); + } + + @Override + protected boolean resetNodeAfterTest() { + return true; + } + + public void testStoreAndGetLicenses() throws Exception { + LicensesService licensesService = getInstanceFromNode(LicensesService.class); + ClusterService clusterService = getInstanceFromNode(ClusterService.class); + License goldLicense = generateSignedLicense("gold", TimeValue.timeValueHours(1)); + TestUtils.registerAndAckSignedLicenses(licensesService, goldLicense, LicensesStatus.VALID); + License silverLicense = generateSignedLicense("silver", TimeValue.timeValueHours(2)); + TestUtils.registerAndAckSignedLicenses(licensesService, silverLicense, LicensesStatus.VALID); + License platinumLicense = generateSignedLicense("platinum", TimeValue.timeValueHours(1)); + TestUtils.registerAndAckSignedLicenses(licensesService, platinumLicense, LicensesStatus.VALID); + License basicLicense = generateSignedLicense("basic", TimeValue.timeValueHours(3)); + TestUtils.registerAndAckSignedLicenses(licensesService, basicLicense, LicensesStatus.VALID); + LicensesMetaData licensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE); + assertThat(licensesMetaData.getLicense(), equalTo(basicLicense)); + final License getLicenses = licensesService.getLicense(); + assertThat(getLicenses, equalTo(basicLicense)); + } + + public void testEffectiveLicenses() throws Exception { + final LicensesService licensesService = getInstanceFromNode(LicensesService.class); + final ClusterService clusterService = getInstanceFromNode(ClusterService.class); + License goldLicense = generateSignedLicense("gold", TimeValue.timeValueSeconds(5)); + // put gold license + TestUtils.registerAndAckSignedLicenses(licensesService, goldLicense, LicensesStatus.VALID); + LicensesMetaData licensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE); + assertThat(licensesService.getLicense(licensesMetaData), equalTo(goldLicense)); + + License platinumLicense = generateSignedLicense("platinum", TimeValue.timeValueSeconds(3)); + // put platinum license + TestUtils.registerAndAckSignedLicenses(licensesService, platinumLicense, LicensesStatus.VALID); + licensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE); + assertThat(licensesService.getLicense(licensesMetaData), equalTo(platinumLicense)); + + License basicLicense = generateSignedLicense("basic", TimeValue.timeValueSeconds(3)); + // put basic license + TestUtils.registerAndAckSignedLicenses(licensesService, basicLicense, LicensesStatus.VALID); + licensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE); + assertThat(licensesService.getLicense(licensesMetaData), equalTo(basicLicense)); + } + + public void testInvalidLicenseStorage() throws Exception { + LicensesService licensesService = getInstanceFromNode(LicensesService.class); + ClusterService clusterService = getInstanceFromNode(ClusterService.class); + License signedLicense = generateSignedLicense(TimeValue.timeValueMinutes(2)); + + // modify content of signed license + License tamperedLicense = License.builder() + .fromLicenseSpec(signedLicense, signedLicense.signature()) + .expiryDate(signedLicense.expiryDate() + 10 * 24 * 60 * 60 * 1000l) + .validate() + .build(); + + TestUtils.registerAndAckSignedLicenses(licensesService, tamperedLicense, LicensesStatus.INVALID); + + // ensure that the invalid license never made it to cluster state + LicensesMetaData licensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE); + if (licensesMetaData != null) { + assertThat(licensesMetaData.getLicense(), equalTo(LicensesMetaData.LICENSE_TOMBSTONE)); + } + } + + public void testRemoveLicenses() throws Exception { + LicensesService licensesService = getInstanceFromNode(LicensesService.class); + ClusterService clusterService = getInstanceFromNode(ClusterService.class); + + // generate a trial license for one feature + licensesService.register(new TestUtils.AssertingLicensee("", logger)); + + // generate signed licenses + License license = generateSignedLicense(TimeValue.timeValueHours(1)); + TestUtils.registerAndAckSignedLicenses(licensesService, license, LicensesStatus.VALID); + LicensesMetaData licensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE); + assertThat(licensesMetaData.getLicense(), not(LicensesMetaData.LICENSE_TOMBSTONE)); + + // remove signed licenses + removeAndAckSignedLicenses(licensesService); + licensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE); + assertThat(licensesMetaData.getLicense(), equalTo(LicensesMetaData.LICENSE_TOMBSTONE)); + } + + private void removeAndAckSignedLicenses(final LicensesService licensesService) { + final CountDownLatch latch = new CountDownLatch(1); + final AtomicBoolean success = new AtomicBoolean(false); + licensesService.removeLicense(new DeleteLicenseRequest(), new ActionListener() { + @Override + public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) { + if (clusterStateUpdateResponse.isAcknowledged()) { + success.set(true); + } + latch.countDown(); + } + + @Override + public void onFailure(Throwable throwable) { + latch.countDown(); + } + }); + try { + latch.await(); + } catch (InterruptedException e) { + fail(e.getMessage()); + } + assertThat("remove license(s) failed", success.get(), equalTo(true)); + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesMetaDataSerializationTests.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesMetaDataSerializationTests.java new file mode 100644 index 00000000000..e2bee38d75b --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesMetaDataSerializationTests.java @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.core; + +import org.elasticsearch.Version; +import org.elasticsearch.common.Base64; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.license.core.License; +import org.elasticsearch.license.plugin.TestUtils; +import org.elasticsearch.test.ESTestCase; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.UUID; + +import static org.elasticsearch.license.core.CryptUtils.encrypt; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class LicensesMetaDataSerializationTests extends ESTestCase { + public void testXContentSerializationOneSignedLicense() throws Exception { + License license = TestUtils.generateSignedLicense(TimeValue.timeValueHours(2)); + LicensesMetaData licensesMetaData = new LicensesMetaData(license); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject("licenses"); + licensesMetaData.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + byte[] serializedBytes = builder.bytes().toBytes(); + + LicensesMetaData licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(serializedBytes); + assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(license)); + } + + public void testXContentSerializationOneTrial() throws Exception { + long issueDate = System.currentTimeMillis(); + License.Builder specBuilder = License.builder() + .uid(UUID.randomUUID().toString()) + .issuedTo("customer") + .maxNodes(5) + .issueDate(issueDate) + .expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis()); + final License trialLicense = TrialLicense.create(specBuilder); + LicensesMetaData licensesMetaData = new LicensesMetaData(trialLicense); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject("licenses"); + licensesMetaData.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + byte[] serializedBytes = builder.bytes().toBytes(); + + LicensesMetaData licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(serializedBytes); + assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(trialLicense)); + } + + public void test1xLicensesMetaDataFromXContent() throws Exception { + License signedLicense = TestUtils.generateSignedLicense(TimeValue.timeValueHours(2)); + long issueDate = signedLicense.issueDate() - TimeValue.timeValueMillis(200).getMillis(); + License.Builder specBuilder = License.builder() + .uid(UUID.randomUUID().toString()) + .issuedTo("customer") + .maxNodes(5) + .issueDate(issueDate) + .expiryDate(issueDate + TimeValue.timeValueHours(2).getMillis()); + final License trialLicense = TrialLicense.create(specBuilder); + // trial license + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject("licenses"); + builder.startArray("trial_licenses"); + XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + trialLicense.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true"))); + builder.value(Base64.encodeBytes(encrypt(contentBuilder.bytes().toBytes()))); + builder.endArray(); + builder.startArray("signed_licenses"); + builder.endArray(); + builder.endObject(); + builder.endObject(); + LicensesMetaData licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes().toBytes()); + assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(trialLicense)); + + // signed license + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject("licenses"); + builder.startArray("trial_licenses"); + builder.endArray(); + builder.startArray("signed_licenses"); + signedLicense.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endArray(); + builder.endObject(); + builder.endObject(); + licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes().toBytes()); + assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(signedLicense)); + + // trial and signed license + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject("licenses"); + builder.startArray("trial_licenses"); + contentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + trialLicense.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true"))); + builder.value(Base64.encodeBytes(encrypt(contentBuilder.bytes().toBytes()))); + builder.endArray(); + builder.startArray("signed_licenses"); + signedLicense.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endArray(); + builder.endObject(); + builder.endObject(); + licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes().toBytes()); + assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(signedLicense)); + + // license with later issue date is selected + License signedLicenseIssuedLater = TestUtils.generateSignedLicense(TimeValue.timeValueHours(2)); + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject("licenses"); + builder.startArray("trial_licenses"); + contentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + trialLicense.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true"))); + builder.value(Base64.encodeBytes(encrypt(contentBuilder.bytes().toBytes()))); + builder.endArray(); + builder.startArray("signed_licenses"); + signedLicense.toXContent(builder, ToXContent.EMPTY_PARAMS); + signedLicenseIssuedLater.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endArray(); + builder.endObject(); + builder.endObject(); + licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes().toBytes()); + assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(signedLicenseIssuedLater)); + + } + + public void test1xLicensesMetaDataFromStream() throws Exception { + long issueDate = System.currentTimeMillis(); + License.Builder specBuilder = License.builder() + .uid(UUID.randomUUID().toString()) + .issuedTo("customer") + .maxNodes(5) + .issueDate(issueDate) + .expiryDate(issueDate + TimeValue.timeValueHours(1).getMillis()); + final License trialLicense = TrialLicense.create(specBuilder); + // trial license + BytesStreamOutput output = new BytesStreamOutput(); + output.writeVInt(0); + output.writeVInt(1); + XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + trialLicense.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true"))); + output.writeString(Base64.encodeBytes(encrypt(contentBuilder.bytes().toBytes()))); + byte[] bytes = output.bytes().toBytes(); + ByteBufferStreamInput input = new ByteBufferStreamInput(ByteBuffer.wrap(bytes)); + + input.setVersion(Version.V_2_0_0_beta1); + LicensesMetaData licensesMetaData = LicensesMetaData.PROTO.readFrom(input); + assertThat(licensesMetaData.getLicense(), equalTo(trialLicense)); + + // signed license + License signedLicense = TestUtils.generateSignedLicense(TimeValue.timeValueHours(2)); + output = new BytesStreamOutput(); + output.writeVInt(1); + signedLicense.writeTo(output); + output.writeVInt(0); + bytes = output.bytes().toBytes(); + input = new ByteBufferStreamInput(ByteBuffer.wrap(bytes)); + input.setVersion(Version.V_2_0_0_beta1); + licensesMetaData = LicensesMetaData.PROTO.readFrom(input); + assertThat(licensesMetaData.getLicense(), equalTo(signedLicense)); + } + + public void testLicenseTombstoneFromXContext() throws Exception { + final XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject("licenses"); + builder.nullField("license"); + builder.endObject(); + LicensesMetaData metaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes().toBytes()); + assertThat(metaDataFromXContent.getLicense(), equalTo(LicensesMetaData.LICENSE_TOMBSTONE)); + } + + private static LicensesMetaData getLicensesMetaDataFromXContent(byte[] bytes) throws Exception { + final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(bytes); + parser.nextToken(); // consume null + parser.nextToken(); // consume "licenses" + LicensesMetaData licensesMetaDataFromXContent = LicensesMetaData.PROTO.fromXContent(parser); + parser.nextToken(); // consume endObject + assertThat(parser.nextToken(), nullValue()); + return licensesMetaDataFromXContent; + } +} diff --git a/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/rest/LicensesRestIT.java b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/rest/LicensesRestIT.java new file mode 100644 index 00000000000..c8f828ecc86 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/java/org/elasticsearch/license/plugin/rest/LicensesRestIT.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.license.plugin.rest; + + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; + +import java.io.IOException; + +public class LicensesRestIT extends ESRestTestCase { + + public LicensesRestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ESRestTestCase.createParameters(0, 1); + } +} diff --git a/elasticsearch/license/plugin/src/test/resources/log4j.properties b/elasticsearch/license/plugin/src/test/resources/log4j.properties new file mode 100644 index 00000000000..bad42d74b70 --- /dev/null +++ b/elasticsearch/license/plugin/src/test/resources/log4j.properties @@ -0,0 +1,11 @@ +es.logger.level=DEBUG +log4j.rootLogger=${es.logger.level}, out + +log4j.logger.org.apache.http=INFO, out +log4j.additivity.org.apache.http=false + +log4j.logger.org.elasticsearch.license=TRACE + +log4j.appender.out=org.apache.log4j.ConsoleAppender +log4j.appender.out.layout=org.apache.log4j.PatternLayout +log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n diff --git a/elasticsearch/license/plugin/src/test/resources/private.key b/elasticsearch/license/plugin/src/test/resources/private.key new file mode 100644 index 0000000000000000000000000000000000000000..1f545803d875598d976b206250d5e95667c794ef GIT binary patch literal 1232 zcmV;>1TXv1%Bc_wbLO>E3}&#D<@-rDJ$(p047kR zkI3mLE>J{X5~_|GE5>Bxd81`fV@~0j8z81fwu78=uPV1?-b_fL$snD`NE}Zsv(l}S z+Dr52K=W~W#&Y(zE)IK9$=-71EtPe#X0XhttX&@1ia@?lvh&>PD=A#>rfdl2Ru)*H zKaKRTp&qI~gmO-)fq;gzrbngk1r$cUKd~25@6;rjPJf6eLtCHTaDU8Rbt#KGKlZ%g zPd`h6iZ4kWgcXo9P=@b~YoN9WQmtytp^dd%)MXVsW%n~=T9#TzSWvr*YCR25X&mo6 zJoJz>&ju@jBz@-J;se7XMiU5&x@g8>FTI*r^O6Xv2(VF_R@Vk3wQBmIE>&XqH^t=M z5_c~f_`fm)R~9ZqEv|Ma3$nQ}LvHLD`h~L2VKXBZwZnP`c6%Qwnwq8Z0!hd2+^*Gk z6R7QdYCwK?kavUU0Vd*Mq5K{M!Vk zzSLrHwA_oZsoml1+m@@Wyh8$ZA^E2_FpxTTRz7~A$x?V+XFW0!aa4^L&A*^Evlz{gN8ae8h9=J=b zC8F19FN`xNpxA!48b`dGRK3GbQMOho1>byaB*psVh{^^yqVL4J0~$j7viqxma>pYOhkH~PwpJT1j6^yHDMWGO{vw8PyI zIrYD&$F3e@0!346r3BY7JkLDD+7E5EE6oiV(gQ62K@eQRE&#pRq81Tut!V+1TCM~srn6-g`!g_`uozhhJEnN6u9$5|& zLC>$@xw*^CD*fb$ngUWU)yZ%TqX*wM!HU2(O2RRFnw&?^e*xOZWZ?n`No9lb?BLpV=@K}hkB uJ-}qCO)pTE(AWQ24*9x_mu179!iEY^J(*;5;=+=*kZC+fDbTdAaQBA diff --git a/qa/shield-core-rest-tests/src/test/java/org/elasticsearch/shield/RestIT.java b/elasticsearch/qa/shield-core-rest-tests/src/test/java/org/elasticsearch/shield/RestIT.java similarity index 100% rename from qa/shield-core-rest-tests/src/test/java/org/elasticsearch/shield/RestIT.java rename to elasticsearch/qa/shield-core-rest-tests/src/test/java/org/elasticsearch/shield/RestIT.java diff --git a/qa/shield-example-realm/build.gradle b/elasticsearch/qa/shield-example-realm/build.gradle similarity index 78% rename from qa/shield-example-realm/build.gradle rename to elasticsearch/qa/shield-example-realm/build.gradle index debd3aa8008..f90918cfdc1 100644 --- a/qa/shield-example-realm/build.gradle +++ b/elasticsearch/qa/shield-example-realm/build.gradle @@ -7,8 +7,8 @@ esplugin { } dependencies { - provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') - provided project(path: ':x-plugins:shield', configuration: 'runtime') + provided project(path: ':x-plugins:elasticsearch:license:plugin', configuration: 'runtime') + provided project(path: ':x-plugins:elasticsearch:shield', configuration: 'runtime') } compileJava.options.compilerArgs << "-Xlint:-rawtypes" @@ -16,8 +16,8 @@ compileJava.options.compilerArgs << "-Xlint:-rawtypes" integTest { cluster { - plugin 'license', project(':x-plugins:license:plugin') - plugin 'shield', project(':x-plugins:shield') + plugin 'license', project(':x-plugins:elasticsearch:license:plugin') + plugin 'shield', project(':x-plugins:elasticsearch:shield') // TODO: these should be settings? systemProperty 'es.shield.authc.realms.custom.order', '0' systemProperty 'es.shield.authc.realms.custom.type', 'custom' diff --git a/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java b/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java similarity index 100% rename from qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java rename to elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java diff --git a/qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomAuthenticationFailureHandler.java b/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomAuthenticationFailureHandler.java similarity index 100% rename from qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomAuthenticationFailureHandler.java rename to elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomAuthenticationFailureHandler.java diff --git a/qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealm.java b/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealm.java similarity index 100% rename from qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealm.java rename to elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealm.java diff --git a/qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealmFactory.java b/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealmFactory.java similarity index 100% rename from qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealmFactory.java rename to elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealmFactory.java diff --git a/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java b/elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java similarity index 100% rename from qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java rename to elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java diff --git a/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java b/elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java similarity index 100% rename from qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java rename to elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java diff --git a/qa/shield-tribe-node-tests/integration-tests.xml b/elasticsearch/qa/shield-tribe-node-tests/integration-tests.xml similarity index 100% rename from qa/shield-tribe-node-tests/integration-tests.xml rename to elasticsearch/qa/shield-tribe-node-tests/integration-tests.xml diff --git a/qa/shield-tribe-node-tests/rest-api-spec/test/tribe_node/10_basic.yaml b/elasticsearch/qa/shield-tribe-node-tests/rest-api-spec/test/tribe_node/10_basic.yaml similarity index 100% rename from qa/shield-tribe-node-tests/rest-api-spec/test/tribe_node/10_basic.yaml rename to elasticsearch/qa/shield-tribe-node-tests/rest-api-spec/test/tribe_node/10_basic.yaml diff --git a/qa/shield-tribe-node-tests/shield-roles.yml b/elasticsearch/qa/shield-tribe-node-tests/shield-roles.yml similarity index 100% rename from qa/shield-tribe-node-tests/shield-roles.yml rename to elasticsearch/qa/shield-tribe-node-tests/shield-roles.yml diff --git a/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/ant/HttpCondition.java b/elasticsearch/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/ant/HttpCondition.java similarity index 100% rename from qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/ant/HttpCondition.java rename to elasticsearch/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/ant/HttpCondition.java diff --git a/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/ant/HttpTask.java b/elasticsearch/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/ant/HttpTask.java similarity index 100% rename from qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/ant/HttpTask.java rename to elasticsearch/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/ant/HttpTask.java diff --git a/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/shield/RestIT.java b/elasticsearch/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/shield/RestIT.java similarity index 100% rename from qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/shield/RestIT.java rename to elasticsearch/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/shield/RestIT.java diff --git a/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/shield/TribeRestTestCase.java b/elasticsearch/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/shield/TribeRestTestCase.java similarity index 100% rename from qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/shield/TribeRestTestCase.java rename to elasticsearch/qa/shield-tribe-node-tests/src/test/java/org/elasticsearch/shield/TribeRestTestCase.java diff --git a/qa/smoke-test-found-license-with-shield-and-watcher/integration-tests.xml b/elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/integration-tests.xml similarity index 100% rename from qa/smoke-test-found-license-with-shield-and-watcher/integration-tests.xml rename to elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/integration-tests.xml diff --git a/qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/MarvelClusterInfoIT.java b/elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/MarvelClusterInfoIT.java similarity index 100% rename from qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/MarvelClusterInfoIT.java rename to elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/MarvelClusterInfoIT.java diff --git a/qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldIT.java b/elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldIT.java similarity index 100% rename from qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldIT.java rename to elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldIT.java diff --git a/qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldInsufficientRoleIT.java b/elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldInsufficientRoleIT.java similarity index 100% rename from qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldInsufficientRoleIT.java rename to elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldInsufficientRoleIT.java diff --git a/qa/smoke-test-found-license-with-shield-and-watcher/watcher-with-shield-roles.yml b/elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/watcher-with-shield-roles.yml similarity index 100% rename from qa/smoke-test-found-license-with-shield-and-watcher/watcher-with-shield-roles.yml rename to elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/watcher-with-shield-roles.yml diff --git a/qa/smoke-test-plugins-ssl/build.gradle b/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle similarity index 86% rename from qa/smoke-test-plugins-ssl/build.gradle rename to elasticsearch/qa/smoke-test-plugins-ssl/build.gradle index 30f361cd459..b273d7583db 100644 --- a/qa/smoke-test-plugins-ssl/build.gradle +++ b/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle @@ -3,7 +3,7 @@ import org.elasticsearch.gradle.LoggedExec apply plugin: 'elasticsearch.rest-test' dependencies { - testCompile project(path: ':x-plugins:shield', configuration: 'runtime') + testCompile project(path: ':x-plugins:elasticsearch:shield', configuration: 'runtime') } // location of keystore and files to generate it @@ -49,10 +49,10 @@ integTest { systemProperty 'es.shield.http.ssl', 'true' systemProperty 'es.shield.ssl.keystore.path', keystore.name systemProperty 'es.shield.ssl.keystore.password', 'keypass' - plugin 'licence', project(':x-plugins:license:plugin') - plugin 'shield', project(':x-plugins:shield') - plugin 'watcher', project(':x-plugins:watcher') - plugin 'marvel-agent', project(':x-plugins:marvel') + plugin 'licence', project(':x-plugins:elasticsearch:license:plugin') + plugin 'shield', project(':x-plugins:elasticsearch:shield') + plugin 'watcher', project(':x-plugins:elasticsearch:watcher') + plugin 'marvel-agent', project(':x-plugins:elasticsearch:marvel') // copy keystore into config/ extraConfigFile keystore.name, keystore diff --git a/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java b/elasticsearch/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java similarity index 100% rename from qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java rename to elasticsearch/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java diff --git a/qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml b/elasticsearch/qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml similarity index 100% rename from qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml rename to elasticsearch/qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml diff --git a/qa/smoke-test-plugins/build.gradle b/elasticsearch/qa/smoke-test-plugins/build.gradle similarity index 77% rename from qa/smoke-test-plugins/build.gradle rename to elasticsearch/qa/smoke-test-plugins/build.gradle index a3deb67646c..f575aa14447 100644 --- a/qa/smoke-test-plugins/build.gradle +++ b/elasticsearch/qa/smoke-test-plugins/build.gradle @@ -3,7 +3,7 @@ import org.elasticsearch.gradle.MavenFilteringHack apply plugin: 'elasticsearch.rest-test' dependencies { - testCompile project(path: ':x-plugins:shield', configuration: 'runtime') + testCompile project(path: ':x-plugins:elasticsearch:shield', configuration: 'runtime') } ext.pluginCount = 0 @@ -19,10 +19,10 @@ for (Project subproj : project.rootProject.subprojects) { project.pluginCount += 4 integTest { cluster { - plugin 'licence', project(':x-plugins:license:plugin') - plugin 'shield', project(':x-plugins:shield') - plugin 'watcher', project(':x-plugins:watcher') - plugin 'marvel-agent', project(':x-plugins:marvel') + plugin 'licence', project(':x-plugins:elasticsearch:license:plugin') + plugin 'shield', project(':x-plugins:elasticsearch:shield') + plugin 'watcher', project(':x-plugins:elasticsearch:watcher') + plugin 'marvel-agent', project(':x-plugins:elasticsearch:marvel') setupCommand 'setupDummyUser', 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' diff --git a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java b/elasticsearch/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java similarity index 100% rename from qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java rename to elasticsearch/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java diff --git a/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml b/elasticsearch/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml similarity index 100% rename from qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml rename to elasticsearch/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml diff --git a/qa/smoke-test-watcher-with-groovy/build.gradle b/elasticsearch/qa/smoke-test-watcher-with-groovy/build.gradle similarity index 53% rename from qa/smoke-test-watcher-with-groovy/build.gradle rename to elasticsearch/qa/smoke-test-watcher-with-groovy/build.gradle index bfbdef34180..3e583915218 100644 --- a/qa/smoke-test-watcher-with-groovy/build.gradle +++ b/elasticsearch/qa/smoke-test-watcher-with-groovy/build.gradle @@ -1,14 +1,14 @@ apply plugin: 'elasticsearch.rest-test' dependencies { - testCompile project(path: ':x-plugins:watcher', configuration: 'runtime') + testCompile project(path: ':x-plugins:elasticsearch:watcher', configuration: 'runtime') testCompile project(path: ':plugins:lang-groovy', configuration: 'runtime') } integTest { cluster { - plugin 'license', project(':x-plugins:license:plugin') - plugin 'watcher', project(':x-plugins:watcher') + plugin 'license', project(':x-plugins:elasticsearch:license:plugin') + plugin 'watcher', project(':x-plugins:elasticsearch:watcher') plugin 'groovy', project(':plugins:lang-groovy') systemProperty 'es.script.inline', 'on' } diff --git a/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java b/elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java similarity index 100% rename from qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java rename to elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java diff --git a/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherWithGroovyIT.java b/elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherWithGroovyIT.java similarity index 100% rename from qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherWithGroovyIT.java rename to elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherWithGroovyIT.java diff --git a/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/10_basic.yaml b/elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/10_basic.yaml similarity index 100% rename from qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/10_basic.yaml rename to elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/10_basic.yaml diff --git a/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/20_minimal_body.yaml b/elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/20_minimal_body.yaml similarity index 100% rename from qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/20_minimal_body.yaml rename to elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/20_minimal_body.yaml diff --git a/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/30_inline_watch.yaml b/elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/30_inline_watch.yaml similarity index 100% rename from qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/30_inline_watch.yaml rename to elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/resources/rest-api-spec/test/watcher_groovy/30_inline_watch.yaml diff --git a/qa/smoke-test-watcher-with-shield/build.gradle b/elasticsearch/qa/smoke-test-watcher-with-shield/build.gradle similarity index 73% rename from qa/smoke-test-watcher-with-shield/build.gradle rename to elasticsearch/qa/smoke-test-watcher-with-shield/build.gradle index 64db01497b7..fdabb511034 100644 --- a/qa/smoke-test-watcher-with-shield/build.gradle +++ b/elasticsearch/qa/smoke-test-watcher-with-shield/build.gradle @@ -1,14 +1,14 @@ apply plugin: 'elasticsearch.rest-test' dependencies { - testCompile project(path: ':x-plugins:shield', configuration: 'runtime') - testCompile project(path: ':x-plugins:watcher', configuration: 'runtime') + testCompile project(path: ':x-plugins:elasticsearch:shield', configuration: 'runtime') + testCompile project(path: ':x-plugins:elasticsearch:watcher', configuration: 'runtime') } // bring in watcher rest test suite task copyWatcherRestTests(type: Copy) { into project.sourceSets.test.output.resourcesDir - from project(':x-plugins:watcher').sourceSets.test.resources.srcDirs + from project(':x-plugins:elasticsearch:watcher').sourceSets.test.resources.srcDirs include 'rest-api-spec/test/**' } @@ -19,9 +19,9 @@ integTest { 'array_compare_watch/10_basic/Basic array_compare watch'].join(',') cluster { - plugin 'license', project(':x-plugins:license:plugin') - plugin 'shield', project(':x-plugins:shield') - plugin 'watcher', project(':x-plugins:watcher') + plugin 'license', project(':x-plugins:elasticsearch:license:plugin') + plugin 'shield', project(':x-plugins:elasticsearch:shield') + plugin 'watcher', project(':x-plugins:elasticsearch:watcher') extraConfigFile 'shield/roles.yml', 'roles.yml' setupCommand 'setupTestAdminUser', 'bin/shield/esusers', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'admin' diff --git a/qa/smoke-test-watcher-with-shield/roles.yml b/elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml similarity index 100% rename from qa/smoke-test-watcher-with-shield/roles.yml rename to elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml diff --git a/qa/smoke-test-watcher-with-shield/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldIT.java b/elasticsearch/qa/smoke-test-watcher-with-shield/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldIT.java similarity index 100% rename from qa/smoke-test-watcher-with-shield/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldIT.java rename to elasticsearch/qa/smoke-test-watcher-with-shield/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldIT.java diff --git a/qa/smoke-test-watcher-with-shield/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldInsufficientRoleIT.java b/elasticsearch/qa/smoke-test-watcher-with-shield/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldInsufficientRoleIT.java similarity index 100% rename from qa/smoke-test-watcher-with-shield/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldInsufficientRoleIT.java rename to elasticsearch/qa/smoke-test-watcher-with-shield/src/test/java/org/elasticsearch/smoketest/WatcherWithShieldInsufficientRoleIT.java diff --git a/shield/LICENSE.txt b/elasticsearch/shield/LICENSE.txt similarity index 100% rename from shield/LICENSE.txt rename to elasticsearch/shield/LICENSE.txt diff --git a/shield/NOTICE.txt b/elasticsearch/shield/NOTICE.txt similarity index 100% rename from shield/NOTICE.txt rename to elasticsearch/shield/NOTICE.txt diff --git a/shield/README.asciidoc b/elasticsearch/shield/README.asciidoc similarity index 100% rename from shield/README.asciidoc rename to elasticsearch/shield/README.asciidoc diff --git a/shield/TESTING.asciidoc b/elasticsearch/shield/TESTING.asciidoc similarity index 100% rename from shield/TESTING.asciidoc rename to elasticsearch/shield/TESTING.asciidoc diff --git a/shield/bin/shield/.in.bat b/elasticsearch/shield/bin/shield/.in.bat similarity index 100% rename from shield/bin/shield/.in.bat rename to elasticsearch/shield/bin/shield/.in.bat diff --git a/shield/bin/shield/esusers b/elasticsearch/shield/bin/shield/esusers similarity index 100% rename from shield/bin/shield/esusers rename to elasticsearch/shield/bin/shield/esusers diff --git a/shield/bin/shield/esusers.bat b/elasticsearch/shield/bin/shield/esusers.bat similarity index 100% rename from shield/bin/shield/esusers.bat rename to elasticsearch/shield/bin/shield/esusers.bat diff --git a/shield/bin/shield/syskeygen b/elasticsearch/shield/bin/shield/syskeygen similarity index 100% rename from shield/bin/shield/syskeygen rename to elasticsearch/shield/bin/shield/syskeygen diff --git a/shield/bin/shield/syskeygen.bat b/elasticsearch/shield/bin/shield/syskeygen.bat similarity index 100% rename from shield/bin/shield/syskeygen.bat rename to elasticsearch/shield/bin/shield/syskeygen.bat diff --git a/shield/build.gradle b/elasticsearch/shield/build.gradle similarity index 86% rename from shield/build.gradle rename to elasticsearch/shield/build.gradle index 38c142f9ba4..a92a96ce1e0 100644 --- a/shield/build.gradle +++ b/elasticsearch/shield/build.gradle @@ -7,8 +7,8 @@ esplugin { } dependencies { - provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') - compile project(':x-plugins:license:plugin-api') + provided project(path: ':x-plugins:elasticsearch:license:plugin', configuration: 'runtime') + compile project(':x-plugins:elasticsearch:license:plugin-api') compile 'dk.brics.automaton:automaton:1.11-8' compile 'com.unboundid:unboundid-ldapsdk:2.3.8' testCompile 'org.slf4j:slf4j-log4j12:1.6.2' diff --git a/shield/config/shield/logging.yml b/elasticsearch/shield/config/shield/logging.yml similarity index 100% rename from shield/config/shield/logging.yml rename to elasticsearch/shield/config/shield/logging.yml diff --git a/shield/config/shield/users b/elasticsearch/shield/config/shield/role_mapping.yml similarity index 100% rename from shield/config/shield/users rename to elasticsearch/shield/config/shield/role_mapping.yml diff --git a/shield/config/shield/roles.yml b/elasticsearch/shield/config/shield/roles.yml similarity index 100% rename from shield/config/shield/roles.yml rename to elasticsearch/shield/config/shield/roles.yml diff --git a/shield/config/shield/users_roles b/elasticsearch/shield/config/shield/users similarity index 100% rename from shield/config/shield/users_roles rename to elasticsearch/shield/config/shield/users diff --git a/elasticsearch/shield/config/shield/users_roles b/elasticsearch/shield/config/shield/users_roles new file mode 100644 index 00000000000..e69de29bb2d diff --git a/shield/dev-tools/esvm/.esvm-shield-config/role_mapping.yml b/elasticsearch/shield/dev-tools/esvm/.esvm-shield-config/role_mapping.yml similarity index 100% rename from shield/dev-tools/esvm/.esvm-shield-config/role_mapping.yml rename to elasticsearch/shield/dev-tools/esvm/.esvm-shield-config/role_mapping.yml diff --git a/shield/dev-tools/esvm/.esvm-shield-config/roles.yml b/elasticsearch/shield/dev-tools/esvm/.esvm-shield-config/roles.yml similarity index 100% rename from shield/dev-tools/esvm/.esvm-shield-config/roles.yml rename to elasticsearch/shield/dev-tools/esvm/.esvm-shield-config/roles.yml diff --git a/shield/dev-tools/esvm/.esvm-shield-config/system_key b/elasticsearch/shield/dev-tools/esvm/.esvm-shield-config/system_key similarity index 100% rename from shield/dev-tools/esvm/.esvm-shield-config/system_key rename to elasticsearch/shield/dev-tools/esvm/.esvm-shield-config/system_key diff --git a/shield/dev-tools/esvm/.esvm-shield-config/users b/elasticsearch/shield/dev-tools/esvm/.esvm-shield-config/users similarity index 100% rename from shield/dev-tools/esvm/.esvm-shield-config/users rename to elasticsearch/shield/dev-tools/esvm/.esvm-shield-config/users diff --git a/shield/dev-tools/esvm/.esvm-shield-config/users_roles b/elasticsearch/shield/dev-tools/esvm/.esvm-shield-config/users_roles similarity index 100% rename from shield/dev-tools/esvm/.esvm-shield-config/users_roles rename to elasticsearch/shield/dev-tools/esvm/.esvm-shield-config/users_roles diff --git a/shield/dev-tools/esvm/.esvmrc b/elasticsearch/shield/dev-tools/esvm/.esvmrc similarity index 100% rename from shield/dev-tools/esvm/.esvmrc rename to elasticsearch/shield/dev-tools/esvm/.esvmrc diff --git a/shield/dev-tools/esvm/readme.txt b/elasticsearch/shield/dev-tools/esvm/readme.txt similarity index 100% rename from shield/dev-tools/esvm/readme.txt rename to elasticsearch/shield/dev-tools/esvm/readme.txt diff --git a/shield/dev-tools/randomization.yml b/elasticsearch/shield/dev-tools/randomization.yml similarity index 100% rename from shield/dev-tools/randomization.yml rename to elasticsearch/shield/dev-tools/randomization.yml diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldBuild.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldBuild.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/ShieldBuild.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldBuild.java diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldDisabledModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldDisabledModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/ShieldDisabledModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldDisabledModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldLifecycleService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldLifecycleService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/ShieldLifecycleService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldLifecycleService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/ShieldModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java diff --git a/shield/src/main/java/org/elasticsearch/shield/ShieldSettingsFilter.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldSettingsFilter.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/ShieldSettingsFilter.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/ShieldSettingsFilter.java diff --git a/shield/src/main/java/org/elasticsearch/shield/User.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/User.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/User.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/User.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionFilter.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionFilter.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/ShieldActionFilter.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionFilter.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionMapper.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionMapper.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/ShieldActionMapper.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionMapper.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/ShieldActionModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheAction.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheAction.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheAction.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheAction.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequest.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequest.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequest.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequest.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequestBuilder.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequestBuilder.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequestBuilder.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequestBuilder.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheResponse.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheResponse.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheResponse.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheResponse.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/TransportClearRealmCacheAction.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/TransportClearRealmCacheAction.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/authc/cache/TransportClearRealmCacheAction.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/TransportClearRealmCacheAction.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/BulkRequestInterceptor.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/interceptor/FieldSecurityRequestInterceptor.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/FieldSecurityRequestInterceptor.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/interceptor/FieldSecurityRequestInterceptor.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/FieldSecurityRequestInterceptor.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/interceptor/RealtimeRequestInterceptor.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/RealtimeRequestInterceptor.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/interceptor/RealtimeRequestInterceptor.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/RealtimeRequestInterceptor.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/interceptor/RequestInterceptor.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/RequestInterceptor.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/interceptor/RequestInterceptor.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/RequestInterceptor.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/interceptor/SearchRequestInterceptor.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/SearchRequestInterceptor.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/interceptor/SearchRequestInterceptor.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/SearchRequestInterceptor.java diff --git a/shield/src/main/java/org/elasticsearch/shield/action/interceptor/UpdateRequestInterceptor.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/UpdateRequestInterceptor.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/action/interceptor/UpdateRequestInterceptor.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/action/interceptor/UpdateRequestInterceptor.java diff --git a/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrail.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrail.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/audit/AuditTrail.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrail.java diff --git a/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrailModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrailModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/audit/AuditTrailModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrailModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrailService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrailService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/audit/AuditTrailService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrailService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/audit/AuditUtil.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/AuditUtil.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/audit/AuditUtil.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/AuditUtil.java diff --git a/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditLevel.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditLevel.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditLevel.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditLevel.java diff --git a/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditTrail.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditTrail.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditTrail.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditTrail.java diff --git a/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditUserHolder.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditUserHolder.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditUserHolder.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditUserHolder.java diff --git a/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexNameResolver.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexNameResolver.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/audit/index/IndexNameResolver.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexNameResolver.java diff --git a/shield/src/main/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrail.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrail.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrail.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrail.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/AnonymousService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/AnonymousService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/AnonymousService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/AnonymousService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationFailureHandler.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationFailureHandler.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationFailureHandler.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationFailureHandler.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationToken.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationToken.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationToken.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationToken.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/DefaultAuthenticationFailureHandler.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/DefaultAuthenticationFailureHandler.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/DefaultAuthenticationFailureHandler.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/DefaultAuthenticationFailureHandler.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/InternalAuthenticationService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/InternalAuthenticationService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/InternalAuthenticationService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/InternalAuthenticationService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/Realm.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/Realm.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/Realm.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/Realm.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/RealmConfig.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/RealmConfig.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/RealmConfig.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/RealmConfig.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/Realms.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/Realms.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/Realms.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/Realms.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryGroupsResolver.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryGroupsResolver.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryGroupsResolver.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryGroupsResolver.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealm.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealm.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealm.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealm.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactory.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactory.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactory.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactory.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/esusers/ESUsersRealm.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/esusers/ESUsersRealm.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/esusers/ESUsersRealm.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/esusers/ESUsersRealm.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStore.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStore.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStore.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStore.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStore.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStore.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStore.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStore.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersTool.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersTool.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersTool.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersTool.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapRealm.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapRealm.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapRealm.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapRealm.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapSessionFactory.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapSessionFactory.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapSessionFactory.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapSessionFactory.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactory.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactory.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactory.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactory.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/ldap/SearchGroupsResolver.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/SearchGroupsResolver.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/ldap/SearchGroupsResolver.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/SearchGroupsResolver.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/ldap/UserAttributeGroupsResolver.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/UserAttributeGroupsResolver.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/ldap/UserAttributeGroupsResolver.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/UserAttributeGroupsResolver.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/AbstractLdapRealm.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/AbstractLdapRealm.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/AbstractLdapRealm.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/AbstractLdapRealm.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapSearchScope.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapSearchScope.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapSearchScope.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapSearchScope.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapSession.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapSession.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapSession.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapSession.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapUtils.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapUtils.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapUtils.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapUtils.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/SessionFactory.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/SessionFactory.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/SessionFactory.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/SessionFactory.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/pki/PkiRealm.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/pki/PkiRealm.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/pki/PkiRealm.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/pki/PkiRealm.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/pki/X509AuthenticationToken.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/pki/X509AuthenticationToken.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/pki/X509AuthenticationToken.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/pki/X509AuthenticationToken.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/support/BCrypt.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/BCrypt.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/support/BCrypt.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/BCrypt.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/support/CachingRealm.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/CachingRealm.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/support/CachingRealm.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/CachingRealm.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealm.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealm.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealm.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealm.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/support/CharArrays.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/CharArrays.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/support/CharArrays.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/CharArrays.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/support/DnRoleMapper.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/DnRoleMapper.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/support/DnRoleMapper.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/DnRoleMapper.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/support/Hasher.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/Hasher.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/support/Hasher.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/Hasher.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/support/RefreshListener.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/RefreshListener.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/support/RefreshListener.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/RefreshListener.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/support/SecuredString.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/SecuredString.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/support/SecuredString.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/SecuredString.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordRealm.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordRealm.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordRealm.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordRealm.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordToken.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordToken.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordToken.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordToken.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/Permission.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/Permission.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/Permission.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/Permission.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/Privilege.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/Privilege.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/Privilege.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/Privilege.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/SystemRole.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/SystemRole.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/SystemRole.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/SystemRole.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReader.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReader.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReader.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReader.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/FieldSubsetReader.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/FieldSubsetReader.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/FieldSubsetReader.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/FieldSubsetReader.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/IndicesAccessControl.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/IndicesAccessControl.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/IndicesAccessControl.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/IndicesAccessControl.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/OptOutQueryCache.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/OptOutQueryCache.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/OptOutQueryCache.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/OptOutQueryCache.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/RequestContext.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/RequestContext.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/RequestContext.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/RequestContext.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolver.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolver.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolver.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolver.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/store/RolesStore.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/store/RolesStore.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/authz/store/RolesStore.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/authz/store/RolesStore.java diff --git a/shield/src/main/java/org/elasticsearch/shield/client/ShieldAuthcClient.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/client/ShieldAuthcClient.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/client/ShieldAuthcClient.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/client/ShieldAuthcClient.java diff --git a/shield/src/main/java/org/elasticsearch/shield/client/ShieldClient.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/client/ShieldClient.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/client/ShieldClient.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/client/ShieldClient.java diff --git a/shield/src/main/java/org/elasticsearch/shield/crypto/CryptoModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/crypto/CryptoModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/crypto/CryptoModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/crypto/CryptoModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/crypto/CryptoService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/crypto/CryptoService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/crypto/CryptoService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/crypto/CryptoService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/crypto/tool/SystemKeyTool.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/crypto/tool/SystemKeyTool.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/crypto/tool/SystemKeyTool.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/crypto/tool/SystemKeyTool.java diff --git a/shield/src/main/java/org/elasticsearch/shield/license/LicenseModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/license/LicenseModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/license/LicenseModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/license/LicenseModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicenseState.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicenseState.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/license/ShieldLicenseState.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicenseState.java diff --git a/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicensee.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicensee.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/license/ShieldLicensee.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/license/ShieldLicensee.java diff --git a/shield/src/main/java/org/elasticsearch/shield/rest/RemoteHostHeader.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/rest/RemoteHostHeader.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/rest/RemoteHostHeader.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/rest/RemoteHostHeader.java diff --git a/shield/src/main/java/org/elasticsearch/shield/rest/ShieldRestFilter.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/rest/ShieldRestFilter.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/rest/ShieldRestFilter.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/rest/ShieldRestFilter.java diff --git a/shield/src/main/java/org/elasticsearch/shield/rest/ShieldRestModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/rest/ShieldRestModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/rest/ShieldRestModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/rest/ShieldRestModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/rest/action/RestShieldInfoAction.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/rest/action/RestShieldInfoAction.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/rest/action/RestShieldInfoAction.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/rest/action/RestShieldInfoAction.java diff --git a/shield/src/main/java/org/elasticsearch/shield/rest/action/authc/cache/RestClearRealmCacheAction.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/rest/action/authc/cache/RestClearRealmCacheAction.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/rest/action/authc/cache/RestClearRealmCacheAction.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/rest/action/authc/cache/RestClearRealmCacheAction.java diff --git a/shield/src/main/java/org/elasticsearch/shield/ssl/AbstractSSLService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/ssl/AbstractSSLService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/ssl/AbstractSSLService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/ssl/AbstractSSLService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/ssl/ClientSSLService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/ssl/ClientSSLService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/ssl/ClientSSLService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/ssl/ClientSSLService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/ssl/SSLModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/ssl/SSLModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/ssl/SSLModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/ssl/SSLModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/ssl/ServerSSLService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/ssl/ServerSSLService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/ssl/ServerSSLService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/ssl/ServerSSLService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/support/AbstractShieldModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/AbstractShieldModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/support/AbstractShieldModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/AbstractShieldModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/support/AutomatonPredicate.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/AutomatonPredicate.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/support/AutomatonPredicate.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/AutomatonPredicate.java diff --git a/shield/src/main/java/org/elasticsearch/shield/support/Automatons.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/Automatons.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/support/Automatons.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/Automatons.java diff --git a/shield/src/main/java/org/elasticsearch/shield/support/Exceptions.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/Exceptions.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/support/Exceptions.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/Exceptions.java diff --git a/shield/src/main/java/org/elasticsearch/shield/support/NoOpLogger.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/NoOpLogger.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/support/NoOpLogger.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/NoOpLogger.java diff --git a/shield/src/main/java/org/elasticsearch/shield/support/ShieldFiles.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/ShieldFiles.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/support/ShieldFiles.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/ShieldFiles.java diff --git a/shield/src/main/java/org/elasticsearch/shield/support/Validation.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/Validation.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/support/Validation.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/support/Validation.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/ClientTransportFilter.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/ClientTransportFilter.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/ClientTransportFilter.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/ClientTransportFilter.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/SSLClientAuth.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/SSLClientAuth.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/SSLClientAuth.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/SSLClientAuth.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/SSLExceptionHelper.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/SSLExceptionHelper.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/SSLExceptionHelper.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/SSLExceptionHelper.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/ServerTransportFilter.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/ServerTransportFilter.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/ServerTransportFilter.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/ServerTransportFilter.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/ShieldClientTransportService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/ShieldClientTransportService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/ShieldClientTransportService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/ShieldClientTransportService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/ShieldServerTransportService.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/ShieldServerTransportService.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/ShieldServerTransportService.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/ShieldServerTransportService.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/ShieldTransportModule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/ShieldTransportModule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/ShieldTransportModule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/ShieldTransportModule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/filter/IPFilter.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/filter/IPFilter.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/filter/IPFilter.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/filter/IPFilter.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRule.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRule.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRule.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRule.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/netty/HandshakeWaitingHandler.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/netty/HandshakeWaitingHandler.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/netty/HandshakeWaitingHandler.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/netty/HandshakeWaitingHandler.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandler.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandler.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandler.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandler.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransport.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransport.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransport.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransport.java diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransport.java b/elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransport.java similarity index 100% rename from shield/src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransport.java rename to elasticsearch/shield/src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransport.java diff --git a/shield/src/main/plugin-metadata/plugin-security.policy b/elasticsearch/shield/src/main/plugin-metadata/plugin-security.policy similarity index 100% rename from shield/src/main/plugin-metadata/plugin-security.policy rename to elasticsearch/shield/src/main/plugin-metadata/plugin-security.policy diff --git a/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-list.help b/elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-list.help similarity index 100% rename from shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-list.help rename to elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-list.help diff --git a/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-passwd.help b/elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-passwd.help similarity index 100% rename from shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-passwd.help rename to elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-passwd.help diff --git a/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-roles.help b/elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-roles.help similarity index 100% rename from shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-roles.help rename to elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-roles.help diff --git a/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-useradd.help b/elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-useradd.help similarity index 100% rename from shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-useradd.help rename to elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-useradd.help diff --git a/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-userdel.help b/elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-userdel.help similarity index 100% rename from shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-userdel.help rename to elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers-userdel.help diff --git a/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers.help b/elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers.help similarity index 100% rename from shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers.help rename to elasticsearch/shield/src/main/resources/org/elasticsearch/shield/authc/esusers/tool/esusers.help diff --git a/shield/src/main/resources/org/elasticsearch/shield/crypto/tool/syskey-generate.help b/elasticsearch/shield/src/main/resources/org/elasticsearch/shield/crypto/tool/syskey-generate.help similarity index 100% rename from shield/src/main/resources/org/elasticsearch/shield/crypto/tool/syskey-generate.help rename to elasticsearch/shield/src/main/resources/org/elasticsearch/shield/crypto/tool/syskey-generate.help diff --git a/shield/src/main/resources/shield-build.properties b/elasticsearch/shield/src/main/resources/shield-build.properties similarity index 100% rename from shield/src/main/resources/shield-build.properties rename to elasticsearch/shield/src/main/resources/shield-build.properties diff --git a/shield/src/main/resources/shield_audit_log.json b/elasticsearch/shield/src/main/resources/shield_audit_log.json similarity index 100% rename from shield/src/main/resources/shield_audit_log.json rename to elasticsearch/shield/src/main/resources/shield_audit_log.json diff --git a/shield/src/test/java/org/elasticsearch/bench/HasherBenchmark.java b/elasticsearch/shield/src/test/java/org/elasticsearch/bench/HasherBenchmark.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/bench/HasherBenchmark.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/bench/HasherBenchmark.java diff --git a/shield/src/test/java/org/elasticsearch/http/netty/NettyHttpMockUtil.java b/elasticsearch/shield/src/test/java/org/elasticsearch/http/netty/NettyHttpMockUtil.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/http/netty/NettyHttpMockUtil.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/http/netty/NettyHttpMockUtil.java diff --git a/shield/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java diff --git a/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/LicensingTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/ScrollIdSigningTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/ScrollIdSigningTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/ScrollIdSigningTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/ScrollIdSigningTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/SettingsFilterTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/SettingsFilterTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/SettingsFilterTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/SettingsFilterTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/ShieldCachePermissionTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/ShieldCachePermissionTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/ShieldCachePermissionTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/ShieldCachePermissionTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/ShieldClearScrollTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/ShieldClearScrollTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/ShieldClearScrollTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/ShieldClearScrollTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java diff --git a/shield/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java diff --git a/shield/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/ShieldPluginEnabledDisabledTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/ShieldPluginEnabledDisabledTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/ShieldPluginEnabledDisabledTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/ShieldPluginEnabledDisabledTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/ShieldPluginTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/ShieldPluginTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/ShieldPluginTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/ShieldPluginTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/UserTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/UserTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/UserTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/UserTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/VersionCompatibilityTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/VersionCompatibilityTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/VersionCompatibilityTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/VersionCompatibilityTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionFilterTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionFilterTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/action/ShieldActionFilterTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionFilterTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionMapperTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionMapperTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/action/ShieldActionMapperTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionMapperTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailServiceTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailServiceTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailServiceTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailServiceTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditLevelTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditLevelTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditLevelTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditLevelTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailEnabledTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailEnabledTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailEnabledTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailEnabledTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailUpdateMappingTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailUpdateMappingTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailUpdateMappingTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailUpdateMappingTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/index/RemoteIndexAuditTrailStartingTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/index/RemoteIndexAuditTrailStartingTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/audit/index/RemoteIndexAuditTrailStartingTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/index/RemoteIndexAuditTrailStartingTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/logfile/CapturingLogger.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/logfile/CapturingLogger.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/audit/logfile/CapturingLogger.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/logfile/CapturingLogger.java diff --git a/shield/src/test/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrailTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrailTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrailTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrailTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserHolderTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserHolderTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserHolderTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserHolderTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/AuthenticationModuleTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/AuthenticationModuleTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/AuthenticationModuleTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/AuthenticationModuleTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/InternalAuthenticationServiceTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/InternalAuthenticationServiceTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/InternalAuthenticationServiceTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/InternalAuthenticationServiceTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/RealmsTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/RealmsTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/RealmsTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/RealmsTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/RunAsIntegTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/RunAsIntegTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/RunAsIntegTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/RunAsIntegTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryGroupsResolverTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryGroupsResolverTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryGroupsResolverTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryGroupsResolverTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealmTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealmTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealmTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealmTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactoryTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactoryTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactoryTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactoryTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/esusers/ESUsersRealmTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/esusers/ESUsersRealmTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/esusers/ESUsersRealmTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/esusers/ESUsersRealmTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStoreTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStoreTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStoreTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStoreTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStoreTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStoreTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStoreTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStoreTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersToolTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersToolTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersToolTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersToolTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapRealmTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapRealmTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapRealmTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapRealmTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapSessionFactoryTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapSessionFactoryTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapSessionFactoryTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapSessionFactoryTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactoryTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactoryTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactoryTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactoryTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/ldap/OpenLdapTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/OpenLdapTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/ldap/OpenLdapTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/OpenLdapTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/ldap/SearchGroupsResolverTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/SearchGroupsResolverTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/ldap/SearchGroupsResolverTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/SearchGroupsResolverTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/ldap/UserAttributeGroupsResolverTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/UserAttributeGroupsResolverTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/ldap/UserAttributeGroupsResolverTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/UserAttributeGroupsResolverTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/LDAPServersTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/LDAPServersTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/LDAPServersTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/LDAPServersTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/LdapTestCase.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/LdapTestCase.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/LdapTestCase.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/LdapTestCase.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/SessionFactoryTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/SessionFactoryTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/SessionFactoryTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/SessionFactoryTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiAuthenticationTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiAuthenticationTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiAuthenticationTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiAuthenticationTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiOptionalClientAuthTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiOptionalClientAuthTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiOptionalClientAuthTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiOptionalClientAuthTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiRealmTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiRealmTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiRealmTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiRealmTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutClientAuthenticationTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutClientAuthenticationTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutClientAuthenticationTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutClientAuthenticationTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutSSLTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutSSLTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutSSLTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutSSLTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/support/BCryptTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/BCryptTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/support/BCryptTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/BCryptTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/support/DnRoleMapperTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/DnRoleMapperTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/support/DnRoleMapperTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/DnRoleMapperTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/support/HasherTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/HasherTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/support/HasherTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/HasherTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/support/SecuredStringTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/SecuredStringTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/support/SecuredStringTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/SecuredStringTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authc/support/UsernamePasswordTokenTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/UsernamePasswordTokenTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authc/support/UsernamePasswordTokenTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authc/support/UsernamePasswordTokenTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/AnalyzeTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/AnalyzeTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/AnalyzeTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/AnalyzeTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/PermissionTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/PermissionTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/PermissionTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/PermissionTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/PrivilegeTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/PrivilegeTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/PrivilegeTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/PrivilegeTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/SystemRoleTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/SystemRoleTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/SystemRoleTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/SystemRoleTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldSubsetReaderTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldSubsetReaderTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldSubsetReaderTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldSubsetReaderTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/IndicesPermissionTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/IndicesPermissionTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/IndicesPermissionTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/IndicesPermissionTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesResolverTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesResolverTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesResolverTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesResolverTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolverIntegrationTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolverIntegrationTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolverIntegrationTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolverIntegrationTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/crypto/InternalCryptoServiceTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/crypto/InternalCryptoServiceTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/crypto/InternalCryptoServiceTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/crypto/InternalCryptoServiceTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/crypto/tool/SystemKeyToolTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/crypto/tool/SystemKeyToolTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/crypto/tool/SystemKeyToolTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/crypto/tool/SystemKeyToolTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/license/ShieldLicenseStateTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/license/ShieldLicenseStateTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/license/ShieldLicenseStateTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/license/ShieldLicenseStateTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/rest/ShieldRestFilterTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/rest/ShieldRestFilterTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/rest/ShieldRestFilterTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/rest/ShieldRestFilterTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/ssl/ClientSSLServiceTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/ssl/ClientSSLServiceTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/ssl/ClientSSLServiceTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/ssl/ClientSSLServiceTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/ssl/SSLSettingsTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/ssl/SSLSettingsTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/ssl/SSLSettingsTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/ssl/SSLSettingsTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/ssl/ServerSSLServiceTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/ssl/ServerSSLServiceTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/ssl/ServerSSLServiceTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/ssl/ServerSSLServiceTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/support/AutomatonsTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/support/AutomatonsTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/support/AutomatonsTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/support/AutomatonsTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/support/ShieldFilesTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/support/ShieldFilesTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/support/ShieldFilesTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/support/ShieldFilesTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/support/ValidationTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/support/ValidationTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/support/ValidationTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/support/ValidationTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/test/ShieldAssertions.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/test/ShieldAssertions.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/test/ShieldAssertions.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/test/ShieldAssertions.java diff --git a/shield/src/test/java/org/elasticsearch/shield/test/ShieldTestUtils.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/test/ShieldTestUtils.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/test/ShieldTestUtils.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/test/ShieldTestUtils.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/ClientTransportFilterTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ClientTransportFilterTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/ClientTransportFilterTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ClientTransportFilterTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/ServerTransportFilterIntegrationTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ServerTransportFilterIntegrationTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/ServerTransportFilterIntegrationTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ServerTransportFilterIntegrationTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/ServerTransportFilterTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ServerTransportFilterTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/ServerTransportFilterTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ServerTransportFilterTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/TransportFilterTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/TransportFilterTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/TransportFilterTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/TransportFilterTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/filter/IPFilterTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/filter/IPFilterTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/filter/IPFilterTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/filter/IPFilterTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringIntegrationTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringIntegrationTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringIntegrationTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringIntegrationTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringUpdateTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringUpdateTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringUpdateTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringUpdateTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRuleTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRuleTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRuleTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRuleTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/netty/HandshakeWaitingHandlerTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/HandshakeWaitingHandlerTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/netty/HandshakeWaitingHandlerTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/HandshakeWaitingHandlerTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandlerTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandlerTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandlerTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandlerTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/netty/IPHostnameVerificationTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/IPHostnameVerificationTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/netty/IPHostnameVerificationTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/IPHostnameVerificationTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransportTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransportTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransportTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransportTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransportTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransportTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransportTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransportTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/netty/SslHostnameVerificationTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/SslHostnameVerificationTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/netty/SslHostnameVerificationTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/netty/SslHostnameVerificationTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslClientAuthTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslClientAuthTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslClientAuthTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslClientAuthTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslIntegrationTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslIntegrationTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslIntegrationTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslIntegrationTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslMultiPortTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslMultiPortTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslMultiPortTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/transport/ssl/SslMultiPortTests.java diff --git a/shield/src/test/java/org/elasticsearch/shield/tribe/TribeShieldLoadedTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/shield/tribe/TribeShieldLoadedTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/shield/tribe/TribeShieldLoadedTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/shield/tribe/TribeShieldLoadedTests.java diff --git a/shield/src/test/java/org/elasticsearch/test/ShieldIntegTestCase.java b/elasticsearch/shield/src/test/java/org/elasticsearch/test/ShieldIntegTestCase.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/test/ShieldIntegTestCase.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/test/ShieldIntegTestCase.java diff --git a/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java b/elasticsearch/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java diff --git a/shield/src/test/java/org/elasticsearch/test/ShieldTestsUtils.java b/elasticsearch/shield/src/test/java/org/elasticsearch/test/ShieldTestsUtils.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/test/ShieldTestsUtils.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/test/ShieldTestsUtils.java diff --git a/shield/src/test/java/org/elasticsearch/transport/KnownActionsTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/transport/KnownActionsTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/transport/KnownActionsTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/transport/KnownActionsTests.java diff --git a/shield/src/test/java/org/elasticsearch/transport/ShieldServerTransportServiceTests.java b/elasticsearch/shield/src/test/java/org/elasticsearch/transport/ShieldServerTransportServiceTests.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/transport/ShieldServerTransportServiceTests.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/transport/ShieldServerTransportServiceTests.java diff --git a/shield/src/test/java/org/elasticsearch/transport/netty/NettyMockUtil.java b/elasticsearch/shield/src/test/java/org/elasticsearch/transport/netty/NettyMockUtil.java similarity index 100% rename from shield/src/test/java/org/elasticsearch/transport/netty/NettyMockUtil.java rename to elasticsearch/shield/src/test/java/org/elasticsearch/transport/netty/NettyMockUtil.java diff --git a/shield/src/test/resources/log4j.properties b/elasticsearch/shield/src/test/resources/log4j.properties similarity index 100% rename from shield/src/test/resources/log4j.properties rename to elasticsearch/shield/src/test/resources/log4j.properties diff --git a/shield/src/test/resources/logstash-shield.conf b/elasticsearch/shield/src/test/resources/logstash-shield.conf similarity index 100% rename from shield/src/test/resources/logstash-shield.conf rename to elasticsearch/shield/src/test/resources/logstash-shield.conf diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/ad-schema.ldif b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/ad-schema.ldif similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/ad-schema.ldif rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/ad-schema.ldif diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/ad.ldif b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/ad.ldif similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/ad.ldif rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/ad.ldif diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/role_mapping.yml b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/role_mapping.yml similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/role_mapping.yml rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/activedirectory/role_mapping.yml diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/esusers/users b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/esusers/users similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/esusers/users rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/esusers/users diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/esusers/users_roles b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/esusers/users_roles similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/esusers/users_roles rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/esusers/users_roles diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldapWithGroupSearch.yaml b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldapWithGroupSearch.yaml similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldapWithGroupSearch.yaml rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldapWithGroupSearch.yaml diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldapWithRoleMapping.yaml b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldapWithRoleMapping.yaml similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldapWithRoleMapping.yaml rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldapWithRoleMapping.yaml diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldaptrust.jks b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldaptrust.jks similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldaptrust.jks rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/ldaptrust.jks diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/seven-seas.ldif b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/seven-seas.ldif similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/seven-seas.ldif rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/ldap/support/seven-seas.ldif diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/pki/role_mapping.yml b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/pki/role_mapping.yml similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/pki/role_mapping.yml rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/pki/role_mapping.yml diff --git a/shield/src/test/resources/org/elasticsearch/shield/authc/support/role_mapping.yml b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/support/role_mapping.yml similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authc/support/role_mapping.yml rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authc/support/role_mapping.yml diff --git a/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml diff --git a/shield/src/test/resources/org/elasticsearch/shield/authz/store/invalid_roles.yml b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authz/store/invalid_roles.yml similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authz/store/invalid_roles.yml rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authz/store/invalid_roles.yml diff --git a/shield/src/test/resources/org/elasticsearch/shield/authz/store/reserved_roles.yml b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authz/store/reserved_roles.yml similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authz/store/reserved_roles.yml rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authz/store/reserved_roles.yml diff --git a/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml diff --git a/shield/src/test/resources/org/elasticsearch/shield/plugin/roles.yml b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/plugin/roles.yml similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/plugin/roles.yml rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/plugin/roles.yml diff --git a/shield/src/test/resources/org/elasticsearch/shield/plugin/users b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/plugin/users similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/plugin/users rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/plugin/users diff --git a/shield/src/test/resources/org/elasticsearch/shield/plugin/users_roles b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/plugin/users_roles similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/plugin/users_roles rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/plugin/users_roles diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/README.asciidoc b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/README.asciidoc similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/README.asciidoc rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/README.asciidoc diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/activedir.crt b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/activedir.crt similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/activedir.crt rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/activedir.crt diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/openldap.crt b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/openldap.crt similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/openldap.crt rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/openldap.crt diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/openssl_config.cnf b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/openssl_config.cnf similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/openssl_config.cnf rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/openssl_config.cnf diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.cert b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.cert similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.cert rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.cert diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.jks b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.jks similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.jks rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.jks diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.p12 b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.p12 similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.p12 rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.p12 diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.pem b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.pem similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.pem rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.pem diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.cert b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.cert similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.cert rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.cert diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.p12 b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.p12 similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.p12 rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.p12 diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.pem b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.pem similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.pem rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.pem diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.cert b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.cert similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.cert rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.cert diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.jks b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.jks similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.jks rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.jks diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.p12 b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.p12 similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.p12 rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.p12 diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.pem b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.pem similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.pem rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.pem diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-different-passwords.jks b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-different-passwords.jks similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-different-passwords.jks rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-different-passwords.jks diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-ip-only.cert b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-ip-only.cert similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-ip-only.cert rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-ip-only.cert diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-ip-only.jks b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-ip-only.jks similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-ip-only.jks rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-ip-only.jks diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-no-subjaltname.cert b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-no-subjaltname.cert similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-no-subjaltname.cert rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-no-subjaltname.cert diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-no-subjaltname.jks b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-no-subjaltname.jks similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-no-subjaltname.jks rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-no-subjaltname.jks diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.p12 b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.p12 similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.p12 rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.p12 diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.pem b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.pem similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.pem rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.pem diff --git a/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/truststore-testnode-only.jks b/elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/truststore-testnode-only.jks similarity index 100% rename from shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/truststore-testnode-only.jks rename to elasticsearch/shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/truststore-testnode-only.jks diff --git a/shield/src/test/resources/org/elasticsearch/transport/actions b/elasticsearch/shield/src/test/resources/org/elasticsearch/transport/actions similarity index 100% rename from shield/src/test/resources/org/elasticsearch/transport/actions rename to elasticsearch/shield/src/test/resources/org/elasticsearch/transport/actions diff --git a/shield/src/test/resources/org/elasticsearch/transport/handlers b/elasticsearch/shield/src/test/resources/org/elasticsearch/transport/handlers similarity index 100% rename from shield/src/test/resources/org/elasticsearch/transport/handlers rename to elasticsearch/shield/src/test/resources/org/elasticsearch/transport/handlers diff --git a/shield/test-signatures.txt b/elasticsearch/shield/test-signatures.txt similarity index 100% rename from shield/test-signatures.txt rename to elasticsearch/shield/test-signatures.txt diff --git a/watcher/LICENSE.txt b/elasticsearch/watcher/LICENSE.txt similarity index 100% rename from watcher/LICENSE.txt rename to elasticsearch/watcher/LICENSE.txt diff --git a/watcher/NOTICE.txt b/elasticsearch/watcher/NOTICE.txt similarity index 100% rename from watcher/NOTICE.txt rename to elasticsearch/watcher/NOTICE.txt diff --git a/watcher/README.asciidoc b/elasticsearch/watcher/README.asciidoc similarity index 100% rename from watcher/README.asciidoc rename to elasticsearch/watcher/README.asciidoc diff --git a/watcher/bin/watcher/.in.bat b/elasticsearch/watcher/bin/watcher/.in.bat similarity index 100% rename from watcher/bin/watcher/.in.bat rename to elasticsearch/watcher/bin/watcher/.in.bat diff --git a/watcher/bin/watcher/croneval b/elasticsearch/watcher/bin/watcher/croneval similarity index 100% rename from watcher/bin/watcher/croneval rename to elasticsearch/watcher/bin/watcher/croneval diff --git a/watcher/bin/watcher/croneval.bat b/elasticsearch/watcher/bin/watcher/croneval.bat similarity index 100% rename from watcher/bin/watcher/croneval.bat rename to elasticsearch/watcher/bin/watcher/croneval.bat diff --git a/watcher/build.gradle b/elasticsearch/watcher/build.gradle similarity index 88% rename from watcher/build.gradle rename to elasticsearch/watcher/build.gradle index 7d2c5d7e4c3..441bd9f971d 100644 --- a/watcher/build.gradle +++ b/elasticsearch/watcher/build.gradle @@ -16,8 +16,8 @@ ext.versions = [ ext.compactProfile = 'full' dependencies { - provided project(path: ':x-plugins:license:plugin', configuration: 'runtime') - provided project(path: ':x-plugins:shield', configuration: 'runtime') + provided project(path: ':x-plugins:elasticsearch:license:plugin', configuration: 'runtime') + provided project(path: ':x-plugins:elasticsearch:shield', configuration: 'runtime') compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239' compile 'com.google.guava:guava:16.0.1' @@ -44,7 +44,7 @@ compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,- integTest { cluster { - plugin 'license', project(':x-plugins:license:plugin') + plugin 'license', project(':x-plugins:elasticsearch:license:plugin') } } diff --git a/watcher/dev-tools/randomization.yml b/elasticsearch/watcher/dev-tools/randomization.yml similarity index 100% rename from watcher/dev-tools/randomization.yml rename to elasticsearch/watcher/dev-tools/randomization.yml diff --git a/watcher/src/main/java/org/elasticsearch/watcher/WatcherBuild.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherBuild.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/WatcherBuild.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherBuild.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/WatcherLifeCycleService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherLifeCycleService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/WatcherLifeCycleService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherLifeCycleService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/WatcherMetaData.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherMetaData.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/WatcherMetaData.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherMetaData.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/WatcherModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/WatcherModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/WatcherPlugin.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherPlugin.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/WatcherPlugin.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherPlugin.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/WatcherService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/WatcherService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/WatcherState.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherState.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/WatcherState.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/WatcherState.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/Action.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/Action.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/Action.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/Action.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionBuilders.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionBuilders.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/ActionBuilders.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionBuilders.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/ActionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionRegistry.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionRegistry.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/ActionRegistry.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionRegistry.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionStatus.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionStatus.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/ActionStatus.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionStatus.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionWrapper.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionWrapper.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/ActionWrapper.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ActionWrapper.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/ExecutableAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ExecutableAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/ExecutableAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ExecutableAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/ExecutableActions.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ExecutableActions.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/ExecutableActions.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/ExecutableActions.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/WatcherActionModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/WatcherActionModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/WatcherActionModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/WatcherActionModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/DataAttachment.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/DataAttachment.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/DataAttachment.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/DataAttachment.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/EmailAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/EmailAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/EmailAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/EmailAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/EmailActionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/EmailActionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/EmailActionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/EmailActionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/ExecutableEmailAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/ExecutableEmailAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/ExecutableEmailAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/ExecutableEmailAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Account.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Account.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Account.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Account.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Accounts.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Accounts.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Accounts.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Accounts.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Attachment.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Attachment.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Attachment.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Attachment.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Authentication.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Authentication.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Authentication.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Authentication.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Email.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Email.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Email.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Email.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/EmailService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/EmailService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/EmailService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/EmailService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/EmailTemplate.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/EmailTemplate.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/EmailTemplate.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/EmailTemplate.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/HtmlSanitizer.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/HtmlSanitizer.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/HtmlSanitizer.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/HtmlSanitizer.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Inline.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Inline.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Inline.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Inline.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/InternalEmailService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/InternalEmailService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/InternalEmailService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/InternalEmailService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Profile.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Profile.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Profile.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/Profile.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/support/BodyPartSource.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/support/BodyPartSource.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/support/BodyPartSource.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/email/service/support/BodyPartSource.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/ExecutableHipChatAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/ExecutableHipChatAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/ExecutableHipChatAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/ExecutableHipChatAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccount.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccount.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccount.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccount.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccounts.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccounts.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccounts.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccounts.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessage.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessage.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessage.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessage.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatServer.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatServer.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatServer.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatServer.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccount.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccount.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccount.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccount.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/InternalHipChatService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/InternalHipChatService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/InternalHipChatService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/InternalHipChatService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/SentMessages.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/SentMessages.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/SentMessages.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/SentMessages.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccount.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccount.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccount.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccount.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/V1Account.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/V1Account.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/V1Account.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/hipchat/service/V1Account.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/index/ExecutableIndexAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/index/ExecutableIndexAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/index/ExecutableIndexAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/index/ExecutableIndexAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/index/IndexAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/index/IndexAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/index/IndexAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/index/IndexAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/index/IndexActionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/index/IndexActionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/index/IndexActionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/index/IndexActionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/ExecutableLoggingAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/ExecutableLoggingAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/logging/ExecutableLoggingAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/ExecutableLoggingAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingActionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingActionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingActionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingActionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingLevel.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingLevel.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingLevel.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/logging/LoggingLevel.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/ExecutableSlackAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/ExecutableSlackAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/ExecutableSlackAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/ExecutableSlackAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackActionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackActionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackActionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/SlackActionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/InternalSlackService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/InternalSlackService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/InternalSlackService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/InternalSlackService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SentMessages.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SentMessages.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SentMessages.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SentMessages.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackAccount.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackAccount.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackAccount.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackAccount.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackAccounts.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackAccounts.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackAccounts.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackAccounts.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/SlackService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/Attachment.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/Attachment.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/Attachment.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/Attachment.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/DynamicAttachments.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/DynamicAttachments.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/DynamicAttachments.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/DynamicAttachments.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/Field.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/Field.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/Field.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/Field.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/MessageElement.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/MessageElement.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/MessageElement.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/MessageElement.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessage.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessage.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessage.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessage.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageDefaults.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageDefaults.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageDefaults.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageDefaults.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/AckThrottler.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/AckThrottler.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/AckThrottler.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/AckThrottler.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/ActionThrottler.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/ActionThrottler.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/ActionThrottler.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/ActionThrottler.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/PeriodThrottler.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/PeriodThrottler.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/PeriodThrottler.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/PeriodThrottler.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/Throttler.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/Throttler.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/Throttler.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/throttler/Throttler.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/ExecutableWebhookAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/ExecutableWebhookAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/ExecutableWebhookAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/ExecutableWebhookAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/WebhookAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/WebhookAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/WebhookAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/WebhookAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/WebhookActionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/WebhookActionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/WebhookActionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/actions/webhook/WebhookActionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/client/WatchSourceBuilder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/client/WatchSourceBuilder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/client/WatchSourceBuilder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/client/WatchSourceBuilder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/client/WatchSourceBuilders.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/client/WatchSourceBuilders.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/client/WatchSourceBuilders.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/client/WatchSourceBuilders.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/client/WatcherClient.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/client/WatcherClient.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/client/WatcherClient.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/client/WatcherClient.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/client/WatcherClientModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/client/WatcherClientModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/client/WatcherClientModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/client/WatcherClientModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/Condition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/Condition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/Condition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/Condition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionBuilders.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionBuilders.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionBuilders.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionBuilders.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionRegistry.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionRegistry.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionRegistry.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/ConditionRegistry.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/ExecutableCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/ExecutableCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/ExecutableCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/ExecutableCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/always/AlwaysCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/always/AlwaysCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/always/AlwaysCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/always/AlwaysCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/always/AlwaysConditionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/always/AlwaysConditionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/always/AlwaysConditionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/always/AlwaysConditionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/always/ExecutableAlwaysCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/always/ExecutableAlwaysCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/always/ExecutableAlwaysCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/always/ExecutableAlwaysCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/AbstractExecutableCompareCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/AbstractExecutableCompareCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/compare/AbstractExecutableCompareCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/AbstractExecutableCompareCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/CompareCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/CompareCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/compare/CompareCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/CompareCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/CompareConditionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/CompareConditionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/compare/CompareConditionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/CompareConditionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/ExecutableCompareCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/ExecutableCompareCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/compare/ExecutableCompareCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/ExecutableCompareCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/LenientCompare.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/LenientCompare.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/compare/LenientCompare.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/LenientCompare.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ExecutableArrayCompareCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ExecutableArrayCompareCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ExecutableArrayCompareCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/compare/array/ExecutableArrayCompareCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/never/ExecutableNeverCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/never/ExecutableNeverCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/never/ExecutableNeverCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/never/ExecutableNeverCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/never/NeverCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/never/NeverCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/never/NeverCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/never/NeverCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/never/NeverConditionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/never/NeverConditionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/never/NeverConditionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/never/NeverConditionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/script/ExecutableScriptCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/script/ExecutableScriptCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/script/ExecutableScriptCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/script/ExecutableScriptCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/script/ScriptCondition.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/script/ScriptCondition.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/script/ScriptCondition.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/script/ScriptCondition.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/condition/script/ScriptConditionFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/script/ScriptConditionFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/condition/script/ScriptConditionFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/condition/script/ScriptConditionFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/ActionExecutionMode.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ActionExecutionMode.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/ActionExecutionMode.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ActionExecutionMode.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/AsyncTriggerListener.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/AsyncTriggerListener.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/AsyncTriggerListener.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/AsyncTriggerListener.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/CurrentExecutions.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/CurrentExecutions.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/CurrentExecutions.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/CurrentExecutions.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionPhase.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionPhase.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionPhase.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionPhase.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionState.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionState.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionState.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ExecutionState.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/InternalWatchExecutor.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/InternalWatchExecutor.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/InternalWatchExecutor.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/InternalWatchExecutor.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/ManualExecutionContext.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ManualExecutionContext.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/ManualExecutionContext.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/ManualExecutionContext.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/QueuedWatch.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/QueuedWatch.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/QueuedWatch.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/QueuedWatch.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/SyncTriggerListener.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/SyncTriggerListener.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/SyncTriggerListener.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/SyncTriggerListener.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredExecutionContext.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredExecutionContext.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredExecutionContext.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredExecutionContext.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatch.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatch.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatch.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatch.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionContext.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionContext.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionContext.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionContext.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionResult.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionResult.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionResult.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionResult.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionSnapshot.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionSnapshot.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionSnapshot.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionSnapshot.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutor.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutor.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutor.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutor.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/Wid.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/Wid.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/execution/Wid.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/execution/Wid.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/history/HistoryModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/history/WatchRecord.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/history/WatchRecord.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/history/WatchRecord.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/history/WatchRecord.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/ExecutableInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/ExecutableInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/ExecutableInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/ExecutableInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/Input.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/Input.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/Input.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/Input.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/InputBuilders.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/InputBuilders.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/InputBuilders.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/InputBuilders.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/InputFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/InputFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/InputFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/InputFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/InputModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/InputModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/InputModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/InputModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/InputRegistry.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/InputRegistry.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/InputRegistry.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/InputRegistry.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/chain/ChainInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/chain/ChainInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/chain/ChainInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/chain/ChainInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/chain/ChainInputFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/chain/ChainInputFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/chain/ChainInputFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/chain/ChainInputFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/chain/ExecutableChainInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/chain/ExecutableChainInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/chain/ExecutableChainInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/chain/ExecutableChainInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/http/ExecutableHttpInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/http/ExecutableHttpInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/http/ExecutableHttpInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/http/ExecutableHttpInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/http/HttpInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/http/HttpInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/http/HttpInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/http/HttpInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/http/HttpInputFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/http/HttpInputFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/http/HttpInputFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/http/HttpInputFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/none/ExecutableNoneInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/none/ExecutableNoneInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/none/ExecutableNoneInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/none/ExecutableNoneInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/none/NoneInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/none/NoneInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/none/NoneInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/none/NoneInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/none/NoneInputFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/none/NoneInputFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/none/NoneInputFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/none/NoneInputFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/search/ExecutableSearchInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/search/ExecutableSearchInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/search/ExecutableSearchInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/search/ExecutableSearchInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInputFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInputFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInputFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInputFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/simple/ExecutableSimpleInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/simple/ExecutableSimpleInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/simple/ExecutableSimpleInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/simple/ExecutableSimpleInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/simple/SimpleInput.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/simple/SimpleInput.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/simple/SimpleInput.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/simple/SimpleInput.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/input/simple/SimpleInputFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/simple/SimpleInputFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/input/simple/SimpleInputFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/input/simple/SimpleInputFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/license/LicenseModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/license/LicenseModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/license/LicenseModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/license/LicenseModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/license/WatcherLicensee.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/license/WatcherLicensee.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/license/WatcherLicensee.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/license/WatcherLicensee.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/WatcherRestHandler.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/WatcherRestHandler.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/WatcherRestHandler.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/WatcherRestHandler.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestAckWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestAckWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestAckWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestAckWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestActivateWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestActivateWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestActivateWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestActivateWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestDeleteWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestDeleteWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestDeleteWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestDeleteWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestExecuteWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestExecuteWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestExecuteWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestExecuteWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestGetWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestGetWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestGetWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestGetWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestHijackOperationAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestHijackOperationAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestHijackOperationAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestHijackOperationAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestPutWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestPutWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestPutWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestPutWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatchServiceAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatchServiceAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatchServiceAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatchServiceAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatcherInfoAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatcherInfoAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatcherInfoAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatcherInfoAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatcherStatsAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatcherStatsAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatcherStatsAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/rest/action/RestWatcherStatsAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/shield/ShieldIntegration.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/shield/ShieldIntegration.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/shield/ShieldIntegration.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/shield/ShieldIntegration.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/shield/ShieldSecretService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/shield/ShieldSecretService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/shield/ShieldSecretService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/shield/ShieldSecretService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherSettingsFilter.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherSettingsFilter.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherSettingsFilter.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherSettingsFilter.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherShieldModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherShieldModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherShieldModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherShieldModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherUserHolder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherUserHolder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherUserHolder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherUserHolder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/ArrayObjectIterator.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/ArrayObjectIterator.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/ArrayObjectIterator.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/ArrayObjectIterator.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/Exceptions.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/Exceptions.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/Exceptions.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/Exceptions.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/Integers.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/Integers.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/Integers.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/Integers.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/Script.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/Script.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/Script.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/Script.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/SearchRequestEquivalence.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/SearchRequestEquivalence.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/SearchRequestEquivalence.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/SearchRequestEquivalence.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/Strings.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/Strings.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/Strings.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/Strings.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/ThreadPoolSettingsBuilder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/ThreadPoolSettingsBuilder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/ThreadPoolSettingsBuilder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/ThreadPoolSettingsBuilder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/Variables.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/Variables.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/Variables.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/Variables.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherDateTimeUtils.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherDateTimeUtils.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/WatcherDateTimeUtils.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherDateTimeUtils.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistry.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistry.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistry.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistry.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherUtils.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherUtils.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/WatcherUtils.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherUtils.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/XContentFilterKeysUtils.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/XContentFilterKeysUtils.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/XContentFilterKeysUtils.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/XContentFilterKeysUtils.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/clock/Clock.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/clock/Clock.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/clock/Clock.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/clock/Clock.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/clock/ClockModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/clock/ClockModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/clock/ClockModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/clock/ClockModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/clock/HaltedClock.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/clock/HaltedClock.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/clock/HaltedClock.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/clock/HaltedClock.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/clock/SystemClock.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/clock/SystemClock.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/clock/SystemClock.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/clock/SystemClock.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/concurrent/FairKeyedLock.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/concurrent/FairKeyedLock.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/concurrent/FairKeyedLock.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/concurrent/FairKeyedLock.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClient.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClient.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClient.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClient.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClientModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClientModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClientModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClientModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpContentType.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpContentType.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpContentType.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpContentType.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpMethod.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpMethod.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpMethod.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpMethod.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpProxy.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpProxy.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpProxy.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpProxy.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpRequest.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpRequest.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpRequest.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpRequest.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpRequestTemplate.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpRequestTemplate.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpRequestTemplate.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpRequestTemplate.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpResponse.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpResponse.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpResponse.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpResponse.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/Scheme.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/Scheme.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/Scheme.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/Scheme.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/ApplicableHttpAuth.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/ApplicableHttpAuth.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/ApplicableHttpAuth.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/ApplicableHttpAuth.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuth.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuth.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuth.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuth.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuthFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuthFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuthFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuthFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuthRegistry.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuthRegistry.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuthRegistry.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/HttpAuthRegistry.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/ApplicableBasicAuth.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/ApplicableBasicAuth.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/ApplicableBasicAuth.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/ApplicableBasicAuth.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/BasicAuth.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/BasicAuth.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/BasicAuth.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/BasicAuth.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/BasicAuthFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/BasicAuthFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/BasicAuthFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/http/auth/basic/BasicAuthFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/init/InitializingModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/init/InitializingModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/init/InitializingModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/init/InitializingModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/init/InitializingService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/init/InitializingService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/init/InitializingService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/init/InitializingService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/init/proxy/ClientProxy.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/init/proxy/ClientProxy.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/init/proxy/ClientProxy.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/init/proxy/ClientProxy.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/init/proxy/ScriptServiceProxy.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/init/proxy/ScriptServiceProxy.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/init/proxy/ScriptServiceProxy.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/init/proxy/ScriptServiceProxy.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/secret/Secret.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/secret/Secret.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/secret/Secret.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/secret/Secret.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/secret/SecretModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/secret/SecretModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/secret/SecretModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/secret/SecretModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/secret/SecretService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/secret/SecretService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/secret/SecretService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/secret/SecretService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplate.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplate.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplate.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplate.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplateEngine.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplateEngine.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplateEngine.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplateEngine.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplateModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplateModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplateModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/TextTemplateModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTextTemplateEngine.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTextTemplateEngine.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTextTemplateEngine.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTextTemplateEngine.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/validation/Validation.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/validation/Validation.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/validation/Validation.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/validation/Validation.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/validation/WatcherSettingsValidation.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/validation/WatcherSettingsValidation.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/validation/WatcherSettingsValidation.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/validation/WatcherSettingsValidation.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/ObjectPath.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/ObjectPath.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/ObjectPath.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/ObjectPath.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherParams.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherParams.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherParams.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherParams.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentParser.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentParser.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentParser.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentParser.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentUtils.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentUtils.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentUtils.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/WatcherXContentUtils.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/XContentSource.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/XContentSource.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/XContentSource.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/support/xcontent/XContentSource.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/ExecutableTransform.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/ExecutableTransform.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/ExecutableTransform.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/ExecutableTransform.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/Transform.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/Transform.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/Transform.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/Transform.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformBuilders.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformBuilders.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/TransformBuilders.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformBuilders.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/TransformFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/TransformModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformRegistry.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformRegistry.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/TransformRegistry.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/TransformRegistry.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ChainTransform.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ChainTransform.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ChainTransform.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ChainTransform.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ChainTransformFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ChainTransformFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ChainTransformFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ChainTransformFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ExecutableChainTransform.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ExecutableChainTransform.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ExecutableChainTransform.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/chain/ExecutableChainTransform.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/script/ExecutableScriptTransform.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/script/ExecutableScriptTransform.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/script/ExecutableScriptTransform.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/script/ExecutableScriptTransform.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/script/ScriptTransform.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/script/ScriptTransform.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/script/ScriptTransform.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/script/ScriptTransform.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/script/ScriptTransformFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/script/ScriptTransformFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/script/ScriptTransformFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/script/ScriptTransformFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/search/ExecutableSearchTransform.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/search/ExecutableSearchTransform.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/search/ExecutableSearchTransform.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/search/ExecutableSearchTransform.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/search/SearchTransform.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/search/SearchTransform.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/search/SearchTransform.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/search/SearchTransform.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transform/search/SearchTransformFactory.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/search/SearchTransformFactory.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transform/search/SearchTransformFactory.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transform/search/SearchTransformFactory.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/WatcherTransportAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/WatcherTransportAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/WatcherTransportAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/WatcherTransportAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchRequest.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchRequest.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchRequest.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchRequest.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchRequestBuilder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchRequestBuilder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchRequestBuilder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchRequestBuilder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchResponse.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchResponse.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchResponse.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchResponse.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/TransportAckWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/TransportAckWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/TransportAckWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/TransportAckWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchRequest.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchRequest.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchRequest.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchRequest.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchRequestBuilder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchRequestBuilder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchRequestBuilder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchRequestBuilder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchResponse.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchResponse.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchResponse.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchResponse.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/TransportActivateWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/TransportActivateWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/TransportActivateWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/TransportActivateWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchRequest.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchRequest.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchRequest.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchRequest.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchResponse.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchResponse.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchResponse.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchResponse.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/TransportDeleteWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/TransportDeleteWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/TransportDeleteWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/TransportDeleteWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchRequest.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchRequest.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchRequest.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchRequest.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchRequestBuilder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchRequestBuilder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchRequestBuilder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchRequestBuilder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchResponse.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchResponse.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchResponse.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchResponse.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/TransportExecuteWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/TransportExecuteWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/TransportExecuteWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/TransportExecuteWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchRequest.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchRequest.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchRequest.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchRequest.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchRequestBuilder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchRequestBuilder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchRequestBuilder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchRequestBuilder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchResponse.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchResponse.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchResponse.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchResponse.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/TransportGetWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/TransportGetWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/TransportGetWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/TransportGetWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchRequest.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchRequest.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchRequest.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchRequest.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchRequestBuilder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchRequestBuilder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchRequestBuilder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchRequestBuilder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchResponse.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchResponse.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchResponse.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchResponse.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/TransportPutWatchAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/TransportPutWatchAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/TransportPutWatchAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/TransportPutWatchAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/TransportWatcherServiceAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/TransportWatcherServiceAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/TransportWatcherServiceAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/TransportWatcherServiceAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceRequest.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceRequest.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceRequest.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceRequest.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceRequestBuilder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceRequestBuilder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceRequestBuilder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceRequestBuilder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceResponse.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceResponse.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceResponse.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceResponse.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/TransportWatcherStatsAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/TransportWatcherStatsAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/TransportWatcherStatsAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/TransportWatcherStatsAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsAction.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsAction.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsAction.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsAction.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsRequest.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsRequest.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsRequest.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsRequest.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsRequestBuilder.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsRequestBuilder.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsRequestBuilder.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsRequestBuilder.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsResponse.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsResponse.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsResponse.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsResponse.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/AbstractTriggerEngine.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/AbstractTriggerEngine.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/AbstractTriggerEngine.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/AbstractTriggerEngine.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/Trigger.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/Trigger.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/Trigger.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/Trigger.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerBuilders.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerBuilders.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerBuilders.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerBuilders.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerEngine.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerEngine.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerEngine.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerEngine.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerEvent.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerEvent.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerEvent.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerEvent.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/TriggerService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTrigger.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTrigger.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTrigger.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTrigger.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEngine.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEngine.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEngine.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEngine.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEvent.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEvent.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEvent.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/manual/ManualTriggerEvent.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Cron.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Cron.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Cron.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Cron.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/CronSchedule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/CronSchedule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/CronSchedule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/CronSchedule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/CronnableSchedule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/CronnableSchedule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/CronnableSchedule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/CronnableSchedule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/DailySchedule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/DailySchedule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/DailySchedule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/DailySchedule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/HourlySchedule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/HourlySchedule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/HourlySchedule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/HourlySchedule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/IntervalSchedule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/IntervalSchedule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/IntervalSchedule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/IntervalSchedule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/MonthlySchedule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/MonthlySchedule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/MonthlySchedule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/MonthlySchedule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Schedule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Schedule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Schedule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Schedule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleRegistry.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleRegistry.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleRegistry.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleRegistry.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTrigger.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTrigger.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTrigger.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTrigger.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEngine.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEngine.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEngine.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEngine.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEvent.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEvent.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEvent.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEvent.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Schedules.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Schedules.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Schedules.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/Schedules.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/WeeklySchedule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/WeeklySchedule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/WeeklySchedule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/WeeklySchedule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/YearlySchedule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/YearlySchedule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/YearlySchedule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/YearlySchedule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleTriggerEngine.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleTriggerEngine.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleTriggerEngine.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleTriggerEngine.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/DayOfWeek.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/DayOfWeek.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/DayOfWeek.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/DayOfWeek.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/DayTimes.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/DayTimes.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/DayTimes.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/DayTimes.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/Month.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/Month.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/Month.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/Month.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/MonthTimes.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/MonthTimes.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/MonthTimes.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/MonthTimes.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/Times.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/Times.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/Times.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/Times.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/WeekTimes.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/WeekTimes.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/WeekTimes.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/WeekTimes.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/YearTimes.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/YearTimes.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/YearTimes.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/support/YearTimes.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/tool/CronEvalTool.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/tool/CronEvalTool.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/tool/CronEvalTool.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/tool/CronEvalTool.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/watch/Payload.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/Payload.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/watch/Payload.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/Payload.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/watch/Watch.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/Watch.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/watch/Watch.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/Watch.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchLockService.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchLockService.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/watch/WatchLockService.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchLockService.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchModule.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchModule.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/watch/WatchModule.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchModule.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStatus.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStatus.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStatus.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStatus.java diff --git a/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStore.java b/elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStore.java similarity index 100% rename from watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStore.java rename to elasticsearch/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStore.java diff --git a/watcher/src/main/plugin-metadata/plugin-security.policy b/elasticsearch/watcher/src/main/plugin-metadata/plugin-security.policy similarity index 100% rename from watcher/src/main/plugin-metadata/plugin-security.policy rename to elasticsearch/watcher/src/main/plugin-metadata/plugin-security.policy diff --git a/watcher/src/main/resources/org/elasticsearch/watcher/trigger/schedule/tool/croneval-eval.help b/elasticsearch/watcher/src/main/resources/org/elasticsearch/watcher/trigger/schedule/tool/croneval-eval.help similarity index 100% rename from watcher/src/main/resources/org/elasticsearch/watcher/trigger/schedule/tool/croneval-eval.help rename to elasticsearch/watcher/src/main/resources/org/elasticsearch/watcher/trigger/schedule/tool/croneval-eval.help diff --git a/watcher/src/main/resources/triggered_watches.json b/elasticsearch/watcher/src/main/resources/triggered_watches.json similarity index 100% rename from watcher/src/main/resources/triggered_watches.json rename to elasticsearch/watcher/src/main/resources/triggered_watches.json diff --git a/watcher/src/main/resources/watch_history.json b/elasticsearch/watcher/src/main/resources/watch_history.json similarity index 100% rename from watcher/src/main/resources/watch_history.json rename to elasticsearch/watcher/src/main/resources/watch_history.json diff --git a/watcher/src/main/resources/watcher-build.properties b/elasticsearch/watcher/src/main/resources/watcher-build.properties similarity index 100% rename from watcher/src/main/resources/watcher-build.properties rename to elasticsearch/watcher/src/main/resources/watcher-build.properties diff --git a/watcher/src/main/resources/watches.json b/elasticsearch/watcher/src/main/resources/watches.json similarity index 100% rename from watcher/src/main/resources/watches.json rename to elasticsearch/watcher/src/main/resources/watches.json diff --git a/watcher/src/test/java/org/elasticsearch/script/SleepScriptEngine.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/script/SleepScriptEngine.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/script/SleepScriptEngine.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/script/SleepScriptEngine.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/WatcherF.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/WatcherF.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/WatcherF.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/WatcherF.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/WatcherLifeCycleServiceTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/WatcherLifeCycleServiceTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/WatcherLifeCycleServiceTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/WatcherLifeCycleServiceTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginDisableTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginDisableTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginDisableTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginDisableTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/WatcherServiceTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/WatcherServiceTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/WatcherServiceTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/WatcherServiceTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/ActionErrorIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/ActionErrorIntegrationTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/ActionErrorIntegrationTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/ActionErrorIntegrationTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/TimeThrottleIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/TimeThrottleIntegrationTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/TimeThrottleIntegrationTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/TimeThrottleIntegrationTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/DataAttachmentTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/DataAttachmentTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/DataAttachmentTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/DataAttachmentTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionIntegrationTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionIntegrationTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionIntegrationTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailActionTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailSecretsIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailSecretsIntegrationTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailSecretsIntegrationTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailSecretsIntegrationTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/AccountTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/AccountTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/AccountTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/AccountTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/AccountsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/AccountsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/AccountsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/AccountsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/EmailTemplateTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/EmailTemplateTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/EmailTemplateTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/EmailTemplateTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/EmailTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/EmailTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/EmailTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/EmailTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/HtmlSanitizerTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/HtmlSanitizerTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/HtmlSanitizerTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/HtmlSanitizerTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/InternalEmailServiceTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/InternalEmailServiceTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/InternalEmailServiceTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/InternalEmailServiceTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/ManualPublicSmtpServersTester.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/ManualPublicSmtpServersTester.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/ManualPublicSmtpServersTester.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/ManualPublicSmtpServersTester.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/support/EmailServer.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/support/EmailServer.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/support/EmailServer.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/support/EmailServer.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionFactoryTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionFactoryTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionFactoryTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionFactoryTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/HipChatActionTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccountsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccountsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccountsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatAccountsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessageTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessageTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessageTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatMessageTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatServiceTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatServiceTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatServiceTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/HipChatServiceTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccountTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccountTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccountTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/IntegrationAccountTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/InternalHipChatServiceTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/InternalHipChatServiceTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/InternalHipChatServiceTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/InternalHipChatServiceTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccountTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccountTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccountTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/UserAccountTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/V1AccountTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/V1AccountTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/V1AccountTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/hipchat/service/V1AccountTests.java diff --git a/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionIntegrationTests.java new file mode 100644 index 00000000000..4fedd5b0d8a --- /dev/null +++ b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionIntegrationTests.java @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.watcher.actions.index; + +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.watcher.history.HistoryStore; +import org.elasticsearch.watcher.support.WatcherDateTimeUtils; +import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; +import org.elasticsearch.watcher.transport.actions.execute.ExecuteWatchResponse; +import org.elasticsearch.watcher.transport.actions.put.PutWatchResponse; +import org.elasticsearch.watcher.trigger.schedule.ScheduleTriggerEvent; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; +import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; +import static org.elasticsearch.watcher.actions.ActionBuilders.indexAction; +import static org.elasticsearch.watcher.client.WatchSourceBuilders.watchBuilder; +import static org.elasticsearch.watcher.input.InputBuilders.searchInput; +import static org.elasticsearch.watcher.input.InputBuilders.simpleInput; +import static org.elasticsearch.watcher.transform.TransformBuilders.scriptTransform; +import static org.elasticsearch.watcher.trigger.TriggerBuilders.schedule; +import static org.elasticsearch.watcher.trigger.schedule.Schedules.cron; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.is; + +/** + * + */ +@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") +public class IndexActionIntegrationTests extends AbstractWatcherIntegrationTestCase { + public void testSimple() throws Exception { + PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder() + .trigger(schedule(cron("0/1 * * * * ? 2020"))) + .input(simpleInput("foo", "bar")) + .addAction("index-buckets", indexAction("idx", "type").setExecutionTimeField("@timestamp"))) + .get(); + + assertThat(putWatchResponse.isCreated(), is(true)); + + DateTime now = timeWarped() ? timeWarp().clock().now(DateTimeZone.UTC) : DateTime.now(DateTimeZone.UTC); + + ExecuteWatchResponse executeWatchResponse = watcherClient().prepareExecuteWatch("_id") + .setTriggerEvent(new ScheduleTriggerEvent(now, now)) + .get(); + + assertThat(executeWatchResponse.getRecordSource().getValue("state"), is((Object) "executed")); + + flush("idx"); + refresh(); + + SearchResponse searchResponse = client().prepareSearch("idx").setTypes("type").get(); + assertThat(searchResponse.getHits().totalHits(), is(1L)); + SearchHit hit = searchResponse.getHits().getAt(0); + if (timeWarped()) { + assertThat(hit.getSource(), hasEntry("@timestamp", (Object) WatcherDateTimeUtils.formatDate(now))); + } else { + assertThat(hit.getSource(), hasKey("@timestamp")); + DateTime timestamp = WatcherDateTimeUtils.parseDate((String) hit.getSource().get("@timestamp")); + assertThat(timestamp.isEqual(now) || timestamp.isAfter(now), is(true)); + } + assertThat(hit.getSource(), hasEntry("foo", (Object) "bar")); + } + + public void testSimpleWithDocField() throws Exception { + PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder() + .trigger(schedule(cron("0/1 * * * * ? 2020"))) + .input(simpleInput("foo", "bar")) + .addAction("index-buckets", + scriptTransform("return [ '_doc' : ctx.payload ]"), + indexAction("idx", "type").setExecutionTimeField("@timestamp"))) + + .get(); + + assertThat(putWatchResponse.isCreated(), is(true)); + + DateTime now = timeWarped() ? timeWarp().clock().now(DateTimeZone.UTC) : DateTime.now(DateTimeZone.UTC); + + ExecuteWatchResponse executeWatchResponse = watcherClient().prepareExecuteWatch("_id") + .setTriggerEvent(new ScheduleTriggerEvent(now, now)) + .get(); + + assertThat(executeWatchResponse.getRecordSource().getValue("state"), is((Object) "executed")); + + flush("idx"); + refresh(); + + SearchResponse searchResponse = client().prepareSearch("idx").setTypes("type").get(); + assertThat(searchResponse.getHits().totalHits(), is(1L)); + SearchHit hit = searchResponse.getHits().getAt(0); + if (timeWarped()) { + assertThat(hit.getSource(), hasEntry("@timestamp", (Object) WatcherDateTimeUtils.formatDate(now))); + } else { + assertThat(hit.getSource(), hasKey("@timestamp")); + DateTime timestamp = WatcherDateTimeUtils.parseDate((String) hit.getSource().get("@timestamp")); + assertThat(timestamp.isEqual(now) || timestamp.isAfter(now), is(true)); + } + assertThat(hit.getSource(), hasEntry("foo", (Object) "bar")); + } + + public void testSimpleWithDocFieldWrongFieldType() throws Exception { + PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder() + .trigger(schedule(cron("0/1 * * * * ? 2020"))) + .input(simpleInput("foo", "bar")) + .addAction("index-buckets", + scriptTransform("return [ '_doc' : 1 ]"), + indexAction("idx", "type").setExecutionTimeField("@timestamp"))) + .get(); + + assertThat(putWatchResponse.isCreated(), is(true)); + + DateTime now = timeWarped() ? timeWarp().clock().now(DateTimeZone.UTC) : DateTime.now(DateTimeZone.UTC); + + ExecuteWatchResponse executeWatchResponse = watcherClient().prepareExecuteWatch("_id") + .setTriggerEvent(new ScheduleTriggerEvent(now, now)) + .setRecordExecution(true) + .get(); + + assertThat(executeWatchResponse.getRecordSource().getValue("state"), is((Object) "executed")); + + flush(); + refresh(); + + assertThat(client().admin().indices().prepareExists("idx").get().isExists(), is(false)); + + assertThat(docCount(HistoryStore.INDEX_PREFIX + "*", HistoryStore.DOC_TYPE, searchSource() + .query(matchQuery("result.actions.status", "failure"))), is(1L)); + + } + + public void testIndexAggsBucketsAsDocuments() throws Exception { + DateTime now = timeWarped() ? timeWarp().clock().now(DateTimeZone.UTC) : DateTime.now(DateTimeZone.UTC); + long bucketCount = randomIntBetween(2, 5); + for (int i = 0; i < bucketCount; i++) { + index("idx", "type", jsonBuilder().startObject() + .field("timestamp", now.minusDays(i)) + .endObject()); + } + + flush("idx"); + refresh(); + + PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder() + .trigger(schedule(cron("0/1 * * * * ? 2020"))) + .input(searchInput(new SearchRequest("idx") + .types("type") + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource() + .aggregation(dateHistogram("trend") + .field("timestamp") + .interval(DateHistogramInterval.DAY))))) + .addAction("index-buckets", + + // this transform takes the bucket list and assigns it to `_doc` + // this means each bucket will be indexed as a separate doc, + // so we expect to have the same number of documents as the number + // of buckets. + scriptTransform("return [ '_doc' : ctx.payload.aggregations.trend.buckets]"), + + indexAction("idx", "bucket").setExecutionTimeField("@timestamp"))) + + .get(); + + assertThat(putWatchResponse.isCreated(), is(true)); + + ExecuteWatchResponse executeWatchResponse = watcherClient().prepareExecuteWatch("_id") + .setTriggerEvent(new ScheduleTriggerEvent(now, now)) + .get(); + + assertThat(executeWatchResponse.getRecordSource().getValue("state"), is((Object) "executed")); + + flush("idx"); + refresh(); + + SearchResponse searchResponse = client().prepareSearch("idx").setTypes("bucket") + .addSort("key", SortOrder.DESC) + .get(); + assertThat(searchResponse.getHits().getTotalHits(), is(bucketCount)); + DateTime key = now.withMillisOfDay(0); + int i = 0; + for (SearchHit hit : searchResponse.getHits()) { + if (timeWarped()) { + assertThat(hit.getSource(), hasEntry("@timestamp", (Object) WatcherDateTimeUtils.formatDate(now))); + } else { + assertThat(hit.getSource(), hasKey("@timestamp")); + DateTime timestamp = WatcherDateTimeUtils.parseDate((String) hit.getSource().get("@timestamp")); + assertThat(timestamp.isEqual(now) || timestamp.isAfter(now), is(true)); + } + assertThat(hit.getSource(), hasEntry("key", (Object) key.getMillis())); + key = key.minusDays(1); + } + } +} diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/index/IndexActionTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/logging/LoggingActionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/logging/LoggingActionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/logging/LoggingActionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/logging/LoggingActionTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/SlackActionFactoryTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/SlackActionFactoryTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/slack/SlackActionFactoryTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/SlackActionFactoryTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/SlackActionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/SlackActionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/slack/SlackActionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/SlackActionTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/SlackAccountsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/SlackAccountsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/SlackAccountsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/SlackAccountsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/SlackServiceTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/SlackServiceTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/SlackServiceTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/SlackServiceTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageDefaultsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageDefaultsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageDefaultsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageDefaultsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/slack/service/message/SlackMessageTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/AckThrottlerTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/AckThrottlerTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/AckThrottlerTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/AckThrottlerTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/ActionThrottleTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/ActionThrottleTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/ActionThrottleTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/ActionThrottleTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/PeriodThrottlerTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/PeriodThrottlerTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/PeriodThrottlerTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/PeriodThrottlerTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/WatchThrottlerTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/WatchThrottlerTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/WatchThrottlerTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/throttler/WatchThrottlerTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookActionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookActionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookActionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookActionTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookHttpsIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookHttpsIntegrationTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookHttpsIntegrationTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookHttpsIntegrationTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookIntegrationTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookIntegrationTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/actions/webhook/WebhookIntegrationTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/condition/always/AlwaysConditionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/always/AlwaysConditionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/condition/always/AlwaysConditionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/always/AlwaysConditionTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/CompareConditionSearchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/CompareConditionSearchTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/condition/compare/CompareConditionSearchTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/CompareConditionSearchTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/CompareConditionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/CompareConditionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/condition/compare/CompareConditionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/CompareConditionTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionSearchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionSearchTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionSearchTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionSearchTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/compare/array/ArrayCompareConditionTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/condition/never/NeverConditionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/never/NeverConditionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/condition/never/NeverConditionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/never/NeverConditionTests.java diff --git a/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionSearchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionSearchTests.java new file mode 100644 index 00000000000..168796ec327 --- /dev/null +++ b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionSearchTests.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.watcher.condition.script; + +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.text.StringText; +import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.search.internal.InternalSearchHits; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.execution.WatchExecutionContext; +import org.elasticsearch.watcher.support.Script; +import org.elasticsearch.watcher.support.init.proxy.ScriptServiceProxy; +import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; +import org.elasticsearch.watcher.test.WatcherTestUtils; +import org.elasticsearch.watcher.watch.Payload; +import org.junit.After; +import org.junit.Before; + +import static org.elasticsearch.watcher.test.WatcherTestUtils.mockExecutionContext; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.when; + +/** + */ +@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") +public class ScriptConditionSearchTests extends AbstractWatcherIntegrationTestCase { + private ThreadPool tp = null; + private ScriptServiceProxy scriptService; + + @Before + public void init() throws Exception { + tp = new ThreadPool(ThreadPool.Names.SAME); + scriptService = WatcherTestUtils.getScriptServiceProxy(tp); + } + + @After + public void cleanup() { + tp.shutdownNow(); + } + + public void testExecuteWithAggs() throws Exception { + client().admin().indices().prepareCreate("my-index") + .addMapping("my-type", "_timestamp", "enabled=true") + .get(); + + client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:00").setSource("{}").get(); + client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:10").setSource("{}").get(); + client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:20").setSource("{}").get(); + client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:30").setSource("{}").get(); + refresh(); + + SearchResponse response = client().prepareSearch("my-index") + .addAggregation(AggregationBuilders.dateHistogram("rate").field("_timestamp").interval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC)) + .get(); + + ExecutableScriptCondition condition = new ExecutableScriptCondition(new ScriptCondition(Script.inline("ctx.payload.aggregations.rate.buckets[0]?.doc_count >= 5").build()), logger, scriptService); + + WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); + assertFalse(condition.execute(ctx).met()); + + client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:40").setSource("{}").get(); + refresh(); + + response = client().prepareSearch("my-index") + .addAggregation(AggregationBuilders.dateHistogram("rate").field("_timestamp").interval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC)) + .get(); + + ctx = mockExecutionContext("_name", new Payload.XContent(response)); + assertThat(condition.execute(ctx).met(), is(true)); + } + + public void testExecuteAccessHits() throws Exception { + ExecutableScriptCondition condition = new ExecutableScriptCondition(new ScriptCondition(Script.inline("ctx.payload.hits?.hits[0]?._score == 1.0").build()), logger, scriptService); + InternalSearchHit hit = new InternalSearchHit(0, "1", new StringText("type"), null); + hit.score(1f); + hit.shard(new SearchShardTarget("a", "a", 0)); + + InternalSearchResponse internalSearchResponse = new InternalSearchResponse(new InternalSearchHits(new InternalSearchHit[]{hit}, 1l, 1f), null, null, false, false); + SearchResponse response = new SearchResponse(internalSearchResponse, "", 3, 3, 500l, new ShardSearchFailure[0]); + + WatchExecutionContext ctx = mockExecutionContext("_watch_name", new Payload.XContent(response)); + assertThat(condition.execute(ctx).met(), is(true)); + hit.score(2f); + when(ctx.payload()).thenReturn(new Payload.XContent(response)); + assertThat(condition.execute(ctx).met(), is(false)); + } +} diff --git a/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionTests.java new file mode 100644 index 00000000000..f66a3de6ce2 --- /dev/null +++ b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/condition/script/ScriptConditionTests.java @@ -0,0 +1,219 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.watcher.condition.script; + + +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.script.ScriptException; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptService.ScriptType; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.condition.Condition; +import org.elasticsearch.watcher.execution.WatchExecutionContext; +import org.elasticsearch.watcher.support.Script; +import org.elasticsearch.watcher.support.init.proxy.ScriptServiceProxy; +import org.elasticsearch.watcher.watch.Payload; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; + +import static java.util.Collections.singletonMap; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.watcher.support.Exceptions.illegalArgument; +import static org.elasticsearch.watcher.test.WatcherTestUtils.getScriptServiceProxy; +import static org.elasticsearch.watcher.test.WatcherTestUtils.mockExecutionContext; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +/** + */ +@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") +public class ScriptConditionTests extends ESTestCase { + ThreadPool tp = null; + + @Before + public void init() { + tp = new ThreadPool(ThreadPool.Names.SAME); + } + + @After + public void cleanup() { + tp.shutdownNow(); + } + + public void testExecute() throws Exception { + ScriptServiceProxy scriptService = getScriptServiceProxy(tp); + ExecutableScriptCondition condition = new ExecutableScriptCondition(new ScriptCondition(Script.inline("ctx.payload.hits.total > 1").build()), logger, scriptService); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]); + WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); + assertFalse(condition.execute(ctx).met()); + } + + public void testExecuteMergedParams() throws Exception { + ScriptServiceProxy scriptService = getScriptServiceProxy(tp); + Script script = Script.inline("ctx.payload.hits.total > threshold").lang(ScriptService.DEFAULT_LANG).params(singletonMap("threshold", 1)).build(); + ExecutableScriptCondition executable = new ExecutableScriptCondition(new ScriptCondition(script), logger, scriptService); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]); + WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); + assertFalse(executable.execute(ctx).met()); + } + + public void testParserValid() throws Exception { + ScriptConditionFactory factory = new ScriptConditionFactory(Settings.settingsBuilder().build(), getScriptServiceProxy(tp)); + + XContentBuilder builder = createConditionContent("ctx.payload.hits.total > 1", null, ScriptType.INLINE); + + XContentParser parser = XContentFactory.xContent(builder.bytes()).createParser(builder.bytes()); + parser.nextToken(); + ScriptCondition condition = factory.parseCondition("_watch", parser); + ExecutableScriptCondition executable = factory.createExecutable(condition); + + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]); + WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); + + assertFalse(executable.execute(ctx).met()); + + + builder = createConditionContent("return true", null, ScriptType.INLINE); + parser = XContentFactory.xContent(builder.bytes()).createParser(builder.bytes()); + parser.nextToken(); + condition = factory.parseCondition("_watch", parser); + executable = factory.createExecutable(condition); + + ctx = mockExecutionContext("_name", new Payload.XContent(response)); + + assertTrue(executable.execute(ctx).met()); + } + + public void testParserInvalid() throws Exception { + ScriptConditionFactory factory = new ScriptConditionFactory(Settings.settingsBuilder().build(), getScriptServiceProxy(tp)); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject().endObject(); + XContentParser parser = XContentFactory.xContent(builder.bytes()).createParser(builder.bytes()); + parser.nextToken(); + try { + factory.parseCondition("_id", parser); + fail("expected a condition exception trying to parse an invalid condition XContent"); + } catch (ElasticsearchParseException e) { + // TODO add these when the test if fixed + // assertThat(e.getMessage(), is("ASDF")); + } + } + + public void testScriptConditionParserBadScript() throws Exception { + ScriptConditionFactory conditionParser = new ScriptConditionFactory(Settings.settingsBuilder().build(), getScriptServiceProxy(tp)); + ScriptType scriptType = randomFrom(ScriptType.values()); + String script; + switch (scriptType) { + case INDEXED: + case FILE: + script = "nonExisting_script"; + break; + case INLINE: + default: + script = "foo = = 1"; + } + XContentBuilder builder = createConditionContent(script, "groovy", scriptType); + XContentParser parser = XContentFactory.xContent(builder.bytes()).createParser(builder.bytes()); + parser.nextToken(); + ScriptCondition scriptCondition = conditionParser.parseCondition("_watch", parser); + try { + conditionParser.createExecutable(scriptCondition); + fail("expected a condition validation exception trying to create an executable with a bad or missing script"); + } catch (ScriptException e) { + // TODO add these when the test if fixed + // assertThat(e.getMessage(), is("ASDF")); + } + } + + public void testScriptConditionParser_badLang() throws Exception { + ScriptConditionFactory conditionParser = new ScriptConditionFactory(Settings.settingsBuilder().build(), getScriptServiceProxy(tp)); + ScriptType scriptType = ScriptType.INLINE; + String script = "return true"; + XContentBuilder builder = createConditionContent(script, "not_a_valid_lang", scriptType); + XContentParser parser = XContentFactory.xContent(builder.bytes()).createParser(builder.bytes()); + parser.nextToken(); + ScriptCondition scriptCondition = conditionParser.parseCondition("_watch", parser); + try { + conditionParser.createExecutable(scriptCondition); + fail("expected a condition validation exception trying to create an executable with an invalid language"); + } catch (ScriptException e) { + // TODO add these when the test if fixed + // assertThat(e.getMessage(), is("ASDF")); + } + } + + public void testScriptConditionThrowException() throws Exception { + ScriptServiceProxy scriptService = getScriptServiceProxy(tp); + ExecutableScriptCondition condition = new ExecutableScriptCondition(new ScriptCondition(Script.inline("assert false").build()), logger, scriptService); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]); + WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); + ScriptCondition.Result result = condition.execute(ctx); + assertThat(result, notNullValue()); + assertThat(result.status(), is(Condition.Result.Status.FAILURE)); + assertThat(result.reason(), notNullValue()); + assertThat(result.reason(), containsString("Assertion")); + } + + public void testScriptConditionReturnObject() throws Exception { + ScriptServiceProxy scriptService = getScriptServiceProxy(tp); + ExecutableScriptCondition condition = new ExecutableScriptCondition(new ScriptCondition(Script.inline("return new Object()").build()), logger, scriptService); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]); + WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); + ScriptCondition.Result result = condition.execute(ctx); + assertThat(result, notNullValue()); + assertThat(result.status(), is(Condition.Result.Status.FAILURE)); + assertThat(result.reason(), notNullValue()); + assertThat(result.reason(), containsString("ScriptException")); + } + + public void testScriptConditionAccessCtx() throws Exception { + ScriptServiceProxy scriptService = getScriptServiceProxy(tp); + ExecutableScriptCondition condition = new ExecutableScriptCondition(new ScriptCondition(Script.inline("ctx.trigger.scheduled_time.getMillis() < System.currentTimeMillis() ").build()), logger, scriptService); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]); + WatchExecutionContext ctx = mockExecutionContext("_name", new DateTime(DateTimeZone.UTC), new Payload.XContent(response)); + Thread.sleep(10); + assertThat(condition.execute(ctx).met(), is(true)); + } + + private static XContentBuilder createConditionContent(String script, String scriptLang, ScriptType scriptType) throws IOException { + XContentBuilder builder = jsonBuilder(); + if (scriptType == null) { + return builder.value(script); + } + builder.startObject(); + switch (scriptType) { + case INLINE: + builder.field("inline", script); + break; + case FILE: + builder.field("file", script); + break; + case INDEXED: + builder.field("id", script); + break; + default: + throw illegalArgument("unsupported script type [{}]", scriptType); + } + if (scriptLang != null) { + builder.field("lang", scriptLang); + } + return builder.endObject(); + } +} diff --git a/watcher/src/test/java/org/elasticsearch/watcher/execution/ExecutionServiceTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/execution/ExecutionServiceTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/execution/ExecutionServiceTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/execution/ExecutionServiceTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/execution/ManualExecutionTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/execution/ManualExecutionTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/execution/ManualExecutionTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/execution/ManualExecutionTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchStoreLifeCycleTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchStoreLifeCycleTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchStoreLifeCycleTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchStoreLifeCycleTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchStoreTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchStoreTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchStoreTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchStoreTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/execution/TriggeredWatchTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreSettingsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreSettingsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreSettingsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreSettingsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryStoreTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateEmailMappingsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateEmailMappingsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateEmailMappingsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateEmailMappingsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateHttpMappingsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateHttpMappingsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateHttpMappingsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateHttpMappingsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateIndexActionMappingsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateIndexActionMappingsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateIndexActionMappingsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateIndexActionMappingsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateSearchInputMappingsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateSearchInputMappingsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateSearchInputMappingsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateSearchInputMappingsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTimeMappingsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTimeMappingsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTimeMappingsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTimeMappingsTests.java diff --git a/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTransformMappingsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTransformMappingsTests.java new file mode 100644 index 00000000000..bac674d7821 --- /dev/null +++ b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateTransformMappingsTests.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.watcher.history; + +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.watcher.execution.ExecutionState; +import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; +import org.elasticsearch.watcher.transport.actions.put.PutWatchResponse; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; +import static org.elasticsearch.watcher.actions.ActionBuilders.loggingAction; +import static org.elasticsearch.watcher.client.WatchSourceBuilders.watchBuilder; +import static org.elasticsearch.watcher.condition.ConditionBuilders.alwaysCondition; +import static org.elasticsearch.watcher.input.InputBuilders.simpleInput; +import static org.elasticsearch.watcher.transform.TransformBuilders.scriptTransform; +import static org.elasticsearch.watcher.trigger.TriggerBuilders.schedule; +import static org.elasticsearch.watcher.trigger.schedule.Schedules.interval; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +/** + * This test makes sure that the http host and path fields in the watch_record action result are + * not analyzed so they can be used in aggregations + */ +@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") +public class HistoryTemplateTransformMappingsTests extends AbstractWatcherIntegrationTestCase { + @Override + protected boolean timeWarped() { + return true; // just to have better control over the triggers + } + + @Override + protected boolean enableShield() { + return false; // remove shield noise from this test + } + + public void testTransformFields() throws Exception { + String index = "the-index"; + String type = "the-type"; + createIndex(index); + index(index, type, "{}"); + flush(); + refresh(); + + PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id1").setSource(watchBuilder() + .trigger(schedule(interval("5s"))) + .input(simpleInput()) + .condition(alwaysCondition()) + .transform(scriptTransform("return [ 'key' : 'value1' ];")) + .addAction("logger", scriptTransform("return [ 'key' : 'value2' ];"), loggingAction("indexed"))) + .get(); + assertThat(putWatchResponse.isCreated(), is(true)); + timeWarp().scheduler().trigger("_id1"); + + // adding another watch which with a transform that should conflict with the preview watch. Since the + // mapping for the transform construct is disabled, there should be nor problems. + putWatchResponse = watcherClient().preparePutWatch("_id2").setSource(watchBuilder() + .trigger(schedule(interval("5s"))) + .input(simpleInput()) + .condition(alwaysCondition()) + .transform(scriptTransform("return [ 'key' : [ 'key1' : 'value1' ] ];")) + .addAction("logger", scriptTransform("return [ 'key' : [ 'key1' : 'value2' ] ];"), loggingAction("indexed"))) + .get(); + assertThat(putWatchResponse.isCreated(), is(true)); + timeWarp().scheduler().trigger("_id2"); + + flush(); + refresh(); + + assertWatchWithMinimumActionsCount("_id1", ExecutionState.EXECUTED, 1); + assertWatchWithMinimumActionsCount("_id2", ExecutionState.EXECUTED, 1); + + refresh(); + + assertBusy(new Runnable() { + @Override + public void run() { + GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings().get(); + assertThat(mappingsResponse, notNullValue()); + assertThat(mappingsResponse.getMappings().isEmpty(), is(false)); + for (ObjectObjectCursor> metadatas : mappingsResponse.getMappings()) { + if (!metadatas.key.startsWith(".watch_history")) { + continue; + } + MappingMetaData metadata = metadatas.value.get("watch_record"); + assertThat(metadata, notNullValue()); + try { + Map source = metadata.getSourceAsMap(); + logger.info("checking index [{}] with metadata:\n[{}]", metadatas.key, metadata.source().toString()); + assertThat(extractValue("properties.result.properties.transform.properties.payload.enabled", source), is((Object) false)); + assertThat(extractValue("properties.result.properties.actions.properties.transform.properties.payload.enabled", source), is((Object) false)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + }); + } +} diff --git a/watcher/src/test/java/org/elasticsearch/watcher/input/InputRegistryTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/InputRegistryTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/input/InputRegistryTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/InputRegistryTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/input/chain/ChainInputTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/chain/ChainInputTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/input/chain/ChainInputTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/chain/ChainInputTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/input/chain/ChainIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/chain/ChainIntegrationTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/input/chain/ChainIntegrationTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/chain/ChainIntegrationTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/input/http/HttpInputIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/http/HttpInputIntegrationTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/input/http/HttpInputIntegrationTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/http/HttpInputIntegrationTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/input/http/HttpInputTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/http/HttpInputTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/input/http/HttpInputTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/http/HttpInputTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/input/search/SearchInputTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/search/SearchInputTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/input/search/SearchInputTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/search/SearchInputTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/input/simple/SimpleInputTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/simple/SimpleInputTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/input/simple/SimpleInputTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/input/simple/SimpleInputTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/license/LicenseTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/license/LicenseTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/license/LicenseTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/license/LicenseTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/shield/BasicShieldTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/shield/BasicShieldTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/shield/BasicShieldTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/shield/BasicShieldTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/DynamicIndexNameIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/DynamicIndexNameIntegrationTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/DynamicIndexNameIntegrationTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/DynamicIndexNameIntegrationTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/FilterXContentTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/FilterXContentTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/FilterXContentTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/FilterXContentTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/VariablesTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/VariablesTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/VariablesTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/VariablesTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherDateTimeUtilsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherDateTimeUtilsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/WatcherDateTimeUtilsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherDateTimeUtilsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistryTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistryTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistryTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistryTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherUtilsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherUtilsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/WatcherUtilsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/WatcherUtilsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/clock/ClockMock.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/clock/ClockMock.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/clock/ClockMock.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/clock/ClockMock.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/clock/ClockTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/clock/ClockTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/clock/ClockTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/clock/ClockTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/concurrent/FairKeyedLockTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/concurrent/FairKeyedLockTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/concurrent/FairKeyedLockTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/concurrent/FairKeyedLockTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpClientTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpClientTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpClientTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpClientTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpConnectionTimeoutTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpConnectionTimeoutTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpConnectionTimeoutTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpConnectionTimeoutTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpReadTimeoutTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpReadTimeoutTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpReadTimeoutTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpReadTimeoutTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpRequestTemplateTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpRequestTemplateTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpRequestTemplateTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpRequestTemplateTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpResponseTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpResponseTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpResponseTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/http/HttpResponseTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/text/TextTemplateTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/text/TextTemplateTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/text/TextTemplateTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/text/TextTemplateTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheScriptEngineTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/text/xmustache/XMustacheTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/xcontent/MapPathTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/xcontent/MapPathTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/xcontent/MapPathTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/xcontent/MapPathTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/support/xcontent/XContentSourceTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/xcontent/XContentSourceTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/support/xcontent/XContentSourceTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/support/xcontent/XContentSourceTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/TimeWarpedWatcherPlugin.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/TimeWarpedWatcherPlugin.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/TimeWarpedWatcherPlugin.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/TimeWarpedWatcherPlugin.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/WatchExecutionContextMockBuilder.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/WatchExecutionContextMockBuilder.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/WatchExecutionContextMockBuilder.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/WatchExecutionContextMockBuilder.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/WatcherMatchers.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/WatcherMatchers.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/WatcherMatchers.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/WatcherMatchers.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/WatcherTestUtils.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/WatcherTestUtils.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/WatcherTestUtils.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/WatcherTestUtils.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/bench/ScheduleEngineTriggerBenchmark.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/bench/ScheduleEngineTriggerBenchmark.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/bench/ScheduleEngineTriggerBenchmark.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/bench/ScheduleEngineTriggerBenchmark.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherExecutorServiceBenchmark.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherExecutorServiceBenchmark.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherExecutorServiceBenchmark.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherExecutorServiceBenchmark.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BasicWatcherTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BasicWatcherTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/integration/BasicWatcherTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BasicWatcherTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BootStrapTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BootStrapTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/integration/BootStrapTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/BootStrapTests.java diff --git a/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/ExecutionVarsIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/ExecutionVarsIntegrationTests.java new file mode 100644 index 00000000000..404c1acc4bf --- /dev/null +++ b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/ExecutionVarsIntegrationTests.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.watcher.test.integration; + +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.util.Callback; +import org.elasticsearch.watcher.client.WatcherClient; +import org.elasticsearch.watcher.support.xcontent.ObjectPath; +import org.elasticsearch.watcher.support.xcontent.XContentSource; +import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; +import org.elasticsearch.watcher.transport.actions.execute.ExecuteWatchResponse; +import org.elasticsearch.watcher.transport.actions.put.PutWatchResponse; + +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.watcher.actions.ActionBuilders.loggingAction; +import static org.elasticsearch.watcher.client.WatchSourceBuilders.watchBuilder; +import static org.elasticsearch.watcher.condition.ConditionBuilders.scriptCondition; +import static org.elasticsearch.watcher.input.InputBuilders.simpleInput; +import static org.elasticsearch.watcher.transform.TransformBuilders.scriptTransform; +import static org.elasticsearch.watcher.trigger.TriggerBuilders.schedule; +import static org.elasticsearch.watcher.trigger.schedule.Schedules.cron; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +/** + */ +@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") +public class ExecutionVarsIntegrationTests extends AbstractWatcherIntegrationTestCase { + @Override + protected boolean timeWarped() { + return true; + } + + public void testVars() throws Exception { + WatcherClient watcherClient = watcherClient(); + + PutWatchResponse putWatchResponse = watcherClient.preparePutWatch("_id").setSource(watchBuilder() + .trigger(schedule(cron("0/1 * * * * ?"))) + .input(simpleInput("value", 5)) + .condition(scriptCondition("ctx.vars.condition_value = ctx.payload.value + 5; return ctx.vars.condition_value > 5;")) + .transform(scriptTransform("ctx.vars.watch_transform_value = ctx.vars.condition_value + 5; return ctx.payload;")) + .addAction( + "a1", + scriptTransform("ctx.vars.a1_transform_value = ctx.vars.watch_transform_value + 10; return ctx.payload;"), + loggingAction("condition_value={{ctx.vars.condition_value}}, watch_transform_value={{ctx.vars.watch_transform_value}}, a1_transform_value={{ctx.vars.a1_transform_value}}")) + .addAction( + "a2", + scriptTransform("ctx.vars.a2_transform_value = ctx.vars.watch_transform_value + 20; return ctx.payload;"), + loggingAction("condition_value={{ctx.vars.condition_value}}, watch_transform_value={{ctx.vars.watch_transform_value}}, a2_transform_value={{ctx.vars.a2_transform_value}}"))) + .get(); + + assertThat(putWatchResponse.isCreated(), is(true)); + + timeWarp().scheduler().trigger("_id"); + + flush(); + refresh(); + + SearchResponse searchResponse = searchWatchRecords(new Callback() { + @Override + public void handle(SearchRequestBuilder builder) { + // defaults to match all; + } + }); + + assertThat(searchResponse.getHits().getTotalHits(), is(1L)); + + Map source = searchResponse.getHits().getAt(0).getSource(); + + assertValue(source, "watch_id", is("_id")); + assertValue(source, "state", is("executed")); + + // we don't store the computed vars in history + assertValue(source, "vars", nullValue()); + + assertValue(source, "result.condition.status", is("success")); + assertValue(source, "result.transform.status", is("success")); + + List> actions = ObjectPath.eval("result.actions", source); + for (Map action : actions) { + String id = (String) action.get("id"); + switch (id) { + case "a1": + assertValue(action, "status", is("success")); + assertValue(action, "transform.status", is("success")); + assertValue(action, "logging.logged_text", is("condition_value=10, watch_transform_value=15, a1_transform_value=25")); + break; + case "a2": + assertValue(action, "status", is("success")); + assertValue(action, "transform.status", is("success")); + assertValue(action, "logging.logged_text", is("condition_value=10, watch_transform_value=15, a2_transform_value=35")); + break; + default: + fail("there should not be an action result for action with an id other than a1 or a2"); + } + } + } + + public void testVarsManual() throws Exception { + WatcherClient watcherClient = watcherClient(); + + PutWatchResponse putWatchResponse = watcherClient.preparePutWatch("_id").setSource(watchBuilder() + .trigger(schedule(cron("0/1 * * * * ? 2020"))) + .input(simpleInput("value", 5)) + .condition(scriptCondition("ctx.vars.condition_value = ctx.payload.value + 5; return ctx.vars.condition_value > 5;")) + .transform(scriptTransform("ctx.vars.watch_transform_value = ctx.vars.condition_value + 5; return ctx.payload;")) + .addAction( + "a1", + scriptTransform("ctx.vars.a1_transform_value = ctx.vars.watch_transform_value + 10; return ctx.payload;"), + loggingAction("condition_value={{ctx.vars.condition_value}}, watch_transform_value={{ctx.vars.watch_transform_value}}, a1_transform_value={{ctx.vars.a1_transform_value}}")) + .addAction( + "a2", + scriptTransform("ctx.vars.a2_transform_value = ctx.vars.watch_transform_value + 20; return ctx.payload;"), + loggingAction("condition_value={{ctx.vars.condition_value}}, watch_transform_value={{ctx.vars.watch_transform_value}}, a2_transform_value={{ctx.vars.a2_transform_value}}"))) + .get(); + + assertThat(putWatchResponse.isCreated(), is(true)); + + boolean debug = randomBoolean(); + + ExecuteWatchResponse executeWatchResponse = watcherClient + .prepareExecuteWatch("_id") + .setDebug(debug) + .get(); + assertThat(executeWatchResponse.getRecordId(), notNullValue()); + XContentSource source = executeWatchResponse.getRecordSource(); + + assertValue(source, "watch_id", is("_id")); + assertValue(source, "state", is("executed")); + + if (debug) { + assertValue(source, "vars.condition_value", is(10)); + assertValue(source, "vars.watch_transform_value", is(15)); + assertValue(source, "vars.a1_transform_value", is(25)); + assertValue(source, "vars.a2_transform_value", is(35)); + } + + assertValue(source, "result.condition.status", is("success")); + assertValue(source, "result.transform.status", is("success")); + + List> actions = source.getValue("result.actions"); + for (Map action : actions) { + String id = (String) action.get("id"); + switch (id) { + case "a1": + assertValue(action, "status", is("success")); + assertValue(action, "transform.status", is("success")); + assertValue(action, "logging.logged_text", is("condition_value=10, watch_transform_value=15, a1_transform_value=25")); + break; + case "a2": + assertValue(action, "status", is("success")); + assertValue(action, "transform.status", is("success")); + assertValue(action, "logging.logged_text", is("condition_value=10, watch_transform_value=15, a2_transform_value=35")); + break; + default: + fail("there should not be an action result for action with an id other than a1 or a2"); + } + } + } +} diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/HttpSecretsIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/HttpSecretsIntegrationTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/integration/HttpSecretsIntegrationTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/HttpSecretsIntegrationTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/NoMasterNodeTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/NoMasterNodeTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/integration/NoMasterNodeTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/NoMasterNodeTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/WatchMetadataTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/WatchMetadataTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/integration/WatchMetadataTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/WatchMetadataTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/integration/WatcherSettingsFilterTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/WatcherSettingsFilterTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/integration/WatcherSettingsFilterTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/integration/WatcherSettingsFilterTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/rest/WatcherRestIT.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/rest/WatcherRestIT.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/rest/WatcherRestIT.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/rest/WatcherRestIT.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/rest/WatcherRestTestCase.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/rest/WatcherRestTestCase.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/test/rest/WatcherRestTestCase.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/test/rest/WatcherRestTestCase.java diff --git a/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transform/TransformIntegrationTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transform/TransformIntegrationTests.java new file mode 100644 index 00000000000..f6be9b031da --- /dev/null +++ b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transform/TransformIntegrationTests.java @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.watcher.transform; + +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.watcher.support.Script; +import org.elasticsearch.watcher.test.AbstractWatcherIntegrationTestCase; +import org.elasticsearch.watcher.test.WatcherTestUtils; +import org.elasticsearch.watcher.transport.actions.put.PutWatchResponse; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.watcher.actions.ActionBuilders.indexAction; +import static org.elasticsearch.watcher.client.WatchSourceBuilders.watchBuilder; +import static org.elasticsearch.watcher.condition.ConditionBuilders.alwaysCondition; +import static org.elasticsearch.watcher.input.InputBuilders.searchInput; +import static org.elasticsearch.watcher.input.InputBuilders.simpleInput; +import static org.elasticsearch.watcher.transform.TransformBuilders.chainTransform; +import static org.elasticsearch.watcher.transform.TransformBuilders.scriptTransform; +import static org.elasticsearch.watcher.transform.TransformBuilders.searchTransform; +import static org.elasticsearch.watcher.trigger.TriggerBuilders.schedule; +import static org.elasticsearch.watcher.trigger.schedule.Schedules.interval; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; + +/** + */ +@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724") +public class TransformIntegrationTests extends AbstractWatcherIntegrationTestCase { + @Override + public Settings nodeSettings(int nodeOrdinal) { + Path configDir = createTempDir(); + Path scripts = configDir.resolve("scripts"); + try { + Files.createDirectories(scripts); + try (InputStream stream = TransformIntegrationTests.class.getResourceAsStream("/config/scripts/my-script.groovy"); + OutputStream output = Files.newOutputStream(scripts.resolve("my-script.groovy"))) { + Streams.copy(stream, output); + } + } catch (IOException ex) { + throw new RuntimeException(ex); + } + //Set path so ScriptService will pick up the test scripts + return settingsBuilder().put(super.nodeSettings(nodeOrdinal)).put("path.conf", configDir.toString()).build(); + } + + public void testScriptTransform() throws Exception { + final Script script; + if (randomBoolean()) { + logger.info("testing script transform with an inline script"); + script = Script.inline("return [key3 : ctx.payload.key1 + ctx.payload.key2]").lang("groovy").build(); + } else if (randomBoolean()) { + logger.info("testing script transform with an indexed script"); + client().preparePutIndexedScript("groovy", "_id", "{\"script\" : \"return [key3 : ctx.payload.key1 + ctx.payload.key2]\"}").get(); + script = Script.indexed("_id").lang("groovy").build(); + } else { + logger.info("testing script transform with a file script"); + script = Script.file("my-script").lang("groovy").build(); + } + + // put a watch that has watch level transform: + PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id1") + .setSource(watchBuilder() + .trigger(schedule(interval("5s"))) + .input(simpleInput(MapBuilder.newMapBuilder().put("key1", 10).put("key2", 10))) + .condition(alwaysCondition()) + .transform(scriptTransform(script)) + .addAction("_id", indexAction("output1", "type"))) + .get(); + assertThat(putWatchResponse.isCreated(), is(true)); + // put a watch that has a action level transform: + putWatchResponse = watcherClient().preparePutWatch("_id2") + .setSource(watchBuilder() + .trigger(schedule(interval("5s"))) + .input(simpleInput(MapBuilder.newMapBuilder().put("key1", 10).put("key2", 10))) + .condition(alwaysCondition()) + .addAction("_id", scriptTransform(script), indexAction("output2", "type"))) + .get(); + assertThat(putWatchResponse.isCreated(), is(true)); + + if (timeWarped()) { + timeWarp().scheduler().trigger("_id1"); + timeWarp().scheduler().trigger("_id2"); + refresh(); + } + + assertWatchWithMinimumPerformedActionsCount("_id1", 1, false); + assertWatchWithMinimumPerformedActionsCount("_id2", 1, false); + refresh(); + + SearchResponse response = client().prepareSearch("output1").get(); + assertNoFailures(response); + assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l)); + assertThat(response.getHits().getAt(0).sourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).sourceAsMap().get("key3").toString(), equalTo("20")); + + response = client().prepareSearch("output2").get(); + assertNoFailures(response); + assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l)); + assertThat(response.getHits().getAt(0).sourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).sourceAsMap().get("key3").toString(), equalTo("20")); + } + + public void testSearchTransform() throws Exception { + createIndex("my-condition-index", "my-payload-index"); + ensureGreen("my-condition-index", "my-payload-index"); + + index("my-payload-index", "payload", "mytestresult"); + refresh(); + + SearchRequest inputRequest = WatcherTestUtils.newInputSearchRequest("my-condition-index").source(searchSource().query(matchAllQuery())); + SearchRequest transformRequest = WatcherTestUtils.newInputSearchRequest("my-payload-index").source(searchSource().query(matchAllQuery())); + + PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id1") + .setSource(watchBuilder() + .trigger(schedule(interval("5s"))) + .input(searchInput(inputRequest)) + .transform(searchTransform(transformRequest)) + .addAction("_id", indexAction("output1", "result")) + ).get(); + assertThat(putWatchResponse.isCreated(), is(true)); + putWatchResponse = watcherClient().preparePutWatch("_id2") + .setSource(watchBuilder() + .trigger(schedule(interval("5s"))) + .input(searchInput(inputRequest)) + .addAction("_id", searchTransform(transformRequest), indexAction("output2", "result")) + ).get(); + assertThat(putWatchResponse.isCreated(), is(true)); + + if (timeWarped()) { + timeWarp().scheduler().trigger("_id1"); + timeWarp().scheduler().trigger("_id2"); + refresh(); + } + + assertWatchWithMinimumPerformedActionsCount("_id1", 1, false); + assertWatchWithMinimumPerformedActionsCount("_id2", 1, false); + refresh(); + + SearchResponse response = client().prepareSearch("output1").get(); + assertNoFailures(response); + assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l)); + assertThat(response.getHits().getAt(0).sourceAsString(), containsString("mytestresult")); + + response = client().prepareSearch("output2").get(); + assertNoFailures(response); + assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l)); + assertThat(response.getHits().getAt(0).sourceAsString(), containsString("mytestresult")); + } + + public void testChainTransform() throws Exception { + final Script script1 = Script.inline("return [key3 : ctx.payload.key1 + ctx.payload.key2]").lang("groovy").build(); + final Script script2 = Script.inline("return [key4 : ctx.payload.key3 + 10]").lang("groovy").build(); + // put a watch that has watch level transform: + PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id1") + .setSource(watchBuilder() + .trigger(schedule(interval("5s"))) + .input(simpleInput(MapBuilder.newMapBuilder().put("key1", 10).put("key2", 10))) + .condition(alwaysCondition()) + .transform(chainTransform(scriptTransform(script1), scriptTransform(script2))) + .addAction("_id", indexAction("output1", "type"))) + .get(); + assertThat(putWatchResponse.isCreated(), is(true)); + // put a watch that has a action level transform: + putWatchResponse = watcherClient().preparePutWatch("_id2") + .setSource(watchBuilder() + .trigger(schedule(interval("5s"))) + .input(simpleInput(MapBuilder.newMapBuilder().put("key1", 10).put("key2", 10))) + .condition(alwaysCondition()) + .addAction("_id", chainTransform(scriptTransform(script1), scriptTransform(script2)), indexAction("output2", "type"))) + .get(); + assertThat(putWatchResponse.isCreated(), is(true)); + + if (timeWarped()) { + timeWarp().scheduler().trigger("_id1"); + timeWarp().scheduler().trigger("_id2"); + refresh(); + } + + assertWatchWithMinimumPerformedActionsCount("_id1", 1, false); + assertWatchWithMinimumPerformedActionsCount("_id2", 1, false); + refresh(); + + SearchResponse response = client().prepareSearch("output1").get(); + assertNoFailures(response); + assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l)); + assertThat(response.getHits().getAt(0).sourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).sourceAsMap().get("key4").toString(), equalTo("30")); + + response = client().prepareSearch("output2").get(); + assertNoFailures(response); + assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l)); + assertThat(response.getHits().getAt(0).sourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).sourceAsMap().get("key4").toString(), equalTo("30")); + } + +} diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transform/chain/ChainTransformTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transform/chain/ChainTransformTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transform/chain/ChainTransformTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transform/chain/ChainTransformTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transform/script/ScriptTransformTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transform/script/ScriptTransformTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transform/script/ScriptTransformTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transform/script/ScriptTransformTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/ack/WatchAckTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/ack/WatchAckTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transport/action/ack/WatchAckTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/ack/WatchAckTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/activate/ActivateWatchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/activate/ActivateWatchTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transport/action/activate/ActivateWatchTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/activate/ActivateWatchTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/DeleteWatchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/DeleteWatchTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/DeleteWatchTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/DeleteWatchTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/ForceDeleteWatchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/ForceDeleteWatchTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/ForceDeleteWatchTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/delete/ForceDeleteWatchTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchWithDateMathTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchWithDateMathTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchWithDateMathTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/execute/ExecuteWatchWithDateMathTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/get/GetWatchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/get/GetWatchTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transport/action/get/GetWatchTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/get/GetWatchTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/put/PutWatchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/put/PutWatchTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transport/action/put/PutWatchTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/put/PutWatchTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/SlowWatchStatsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/SlowWatchStatsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/SlowWatchStatsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/SlowWatchStatsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/WatcherStatsTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/WatcherStatsTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/WatcherStatsTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/transport/action/stats/WatcherStatsTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/ScheduleTriggerEngineMock.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/ScheduleTriggerEngineMock.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/ScheduleTriggerEngineMock.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/ScheduleTriggerEngineMock.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/CronScheduleTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/CronScheduleTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/CronScheduleTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/CronScheduleTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/DailyScheduleTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/DailyScheduleTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/DailyScheduleTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/DailyScheduleTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/HourlyScheduleTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/HourlyScheduleTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/HourlyScheduleTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/HourlyScheduleTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/IntervalScheduleTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/IntervalScheduleTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/IntervalScheduleTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/IntervalScheduleTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/MonthlyScheduleTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/MonthlyScheduleTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/MonthlyScheduleTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/MonthlyScheduleTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleRegistryTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleRegistryTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleRegistryTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleRegistryTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTestCase.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTestCase.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTestCase.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTestCase.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEventTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEventTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEventTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/ScheduleTriggerEventTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/WeeklyScheduleTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/WeeklyScheduleTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/WeeklyScheduleTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/WeeklyScheduleTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/YearlyScheduleTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/YearlyScheduleTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/YearlyScheduleTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/YearlyScheduleTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/BaseTriggerEngineTestCase.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/BaseTriggerEngineTestCase.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/BaseTriggerEngineTestCase.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/BaseTriggerEngineTestCase.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleEngineTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleEngineTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleEngineTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/SchedulerScheduleEngineTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/tool/CronEvalToolTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/tool/CronEvalToolTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/tool/CronEvalToolTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/tool/CronEvalToolTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/tool/EvalCron.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/tool/EvalCron.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/tool/EvalCron.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/trigger/schedule/tool/EvalCron.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchLockServiceTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchLockServiceTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/watch/WatchLockServiceTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchLockServiceTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchStoreTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchStoreTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/watch/WatchStoreTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchStoreTests.java diff --git a/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchTests.java b/elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchTests.java similarity index 100% rename from watcher/src/test/java/org/elasticsearch/watcher/watch/WatchTests.java rename to elasticsearch/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchTests.java diff --git a/watcher/src/test/resources/config/scripts/my-script.groovy b/elasticsearch/watcher/src/test/resources/config/scripts/my-script.groovy similarity index 100% rename from watcher/src/test/resources/config/scripts/my-script.groovy rename to elasticsearch/watcher/src/test/resources/config/scripts/my-script.groovy diff --git a/watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode-different-passwords.jks b/elasticsearch/watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode-different-passwords.jks similarity index 100% rename from watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode-different-passwords.jks rename to elasticsearch/watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode-different-passwords.jks diff --git a/watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode.cert b/elasticsearch/watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode.cert similarity index 100% rename from watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode.cert rename to elasticsearch/watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode.cert diff --git a/watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode.jks b/elasticsearch/watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode.jks similarity index 100% rename from watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode.jks rename to elasticsearch/watcher/src/test/resources/org/elasticsearch/shield/keystore/testnode.jks diff --git a/watcher/src/test/resources/org/elasticsearch/shield/keystore/truststore-testnode-only.jks b/elasticsearch/watcher/src/test/resources/org/elasticsearch/shield/keystore/truststore-testnode-only.jks similarity index 100% rename from watcher/src/test/resources/org/elasticsearch/shield/keystore/truststore-testnode-only.jks rename to elasticsearch/watcher/src/test/resources/org/elasticsearch/shield/keystore/truststore-testnode-only.jks diff --git a/watcher/src/test/resources/org/elasticsearch/watcher/actions/email/service/logo.png b/elasticsearch/watcher/src/test/resources/org/elasticsearch/watcher/actions/email/service/logo.png similarity index 100% rename from watcher/src/test/resources/org/elasticsearch/watcher/actions/email/service/logo.png rename to elasticsearch/watcher/src/test/resources/org/elasticsearch/watcher/actions/email/service/logo.png diff --git a/watcher/src/test/resources/org/elasticsearch/watcher/input/search/config/scripts/test_disk_template.mustache b/elasticsearch/watcher/src/test/resources/org/elasticsearch/watcher/input/search/config/scripts/test_disk_template.mustache similarity index 100% rename from watcher/src/test/resources/org/elasticsearch/watcher/input/search/config/scripts/test_disk_template.mustache rename to elasticsearch/watcher/src/test/resources/org/elasticsearch/watcher/input/search/config/scripts/test_disk_template.mustache diff --git a/watcher/src/test/resources/org/elasticsearch/watcher/transform/search/config/scripts/test_disk_template.mustache b/elasticsearch/watcher/src/test/resources/org/elasticsearch/watcher/transform/search/config/scripts/test_disk_template.mustache similarity index 100% rename from watcher/src/test/resources/org/elasticsearch/watcher/transform/search/config/scripts/test_disk_template.mustache rename to elasticsearch/watcher/src/test/resources/org/elasticsearch/watcher/transform/search/config/scripts/test_disk_template.mustache diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.ack_watch.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.ack_watch.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.ack_watch.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.ack_watch.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.activate_watch.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.activate_watch.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.activate_watch.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.activate_watch.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.deactivate_watch.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.deactivate_watch.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.deactivate_watch.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.deactivate_watch.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.delete_watch.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.delete_watch.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.delete_watch.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.delete_watch.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.execute_watch.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.execute_watch.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.execute_watch.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.execute_watch.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.get_watch.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.get_watch.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.get_watch.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.get_watch.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.info.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.info.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.info.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.info.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.put_watch.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.put_watch.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.put_watch.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.put_watch.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.restart.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.restart.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.restart.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.restart.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.start.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.start.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.start.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.start.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.stats.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.stats.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.stats.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.stats.json diff --git a/watcher/src/test/resources/rest-api-spec/api/watcher.stop.json b/elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.stop.json similarity index 100% rename from watcher/src/test/resources/rest-api-spec/api/watcher.stop.json rename to elasticsearch/watcher/src/test/resources/rest-api-spec/api/watcher.stop.json diff --git a/watcher/src/test/resources/rest-api-spec/test/ack_watch/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/ack_watch/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/ack_watch/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/ack_watch/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/ack_watch/20_ack_individual_action.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/ack_watch/20_ack_individual_action.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/ack_watch/20_ack_individual_action.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/ack_watch/20_ack_individual_action.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/activate_watch/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/activate_watch/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/activate_watch/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/activate_watch/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/array_compare_watch/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/array_compare_watch/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/array_compare_watch/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/array_compare_watch/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/delete_watch/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/delete_watch/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/delete_watch/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/delete_watch/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/get_watch/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/get_watch/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/get_watch/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/get_watch/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/get_watch/20_missing.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/get_watch/20_missing.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/get_watch/20_missing.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/get_watch/20_missing.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/getting_started/10_monitor_cluster_health.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/getting_started/10_monitor_cluster_health.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/getting_started/10_monitor_cluster_health.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/getting_started/10_monitor_cluster_health.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/hijack/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/hijack/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/hijack/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/hijack/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/put_watch/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/put_watch/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/put_watch/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/put_watch/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/put_watch/20_put_watch_with_throttle_period.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/put_watch/20_put_watch_with_throttle_period.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/put_watch/20_put_watch_with_throttle_period.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/put_watch/20_put_watch_with_throttle_period.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/put_watch/30_put_watch_with_action_throttle_period.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/put_watch/30_put_watch_with_action_throttle_period.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/put_watch/30_put_watch_with_action_throttle_period.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/put_watch/30_put_watch_with_action_throttle_period.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/put_watch/40_put_watch_as_inactive.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/put_watch/40_put_watch_as_inactive.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/put_watch/40_put_watch_as_inactive.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/put_watch/40_put_watch_as_inactive.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/restart_watcher/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/restart_watcher/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/restart_watcher/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/restart_watcher/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/start_watcher/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/start_watcher/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/start_watcher/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/start_watcher/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/stats/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/stats/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/stats/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/stats/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/stop_watcher/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/stop_watcher/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/stop_watcher/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/stop_watcher/10_basic.yaml diff --git a/watcher/src/test/resources/rest-api-spec/test/watch_info/10_basic.yaml b/elasticsearch/watcher/src/test/resources/rest-api-spec/test/watch_info/10_basic.yaml similarity index 100% rename from watcher/src/test/resources/rest-api-spec/test/watch_info/10_basic.yaml rename to elasticsearch/watcher/src/test/resources/rest-api-spec/test/watch_info/10_basic.yaml diff --git a/x-dev-tools/RELEASE.md b/elasticsearch/x-dev-tools/RELEASE.md similarity index 100% rename from x-dev-tools/RELEASE.md rename to elasticsearch/x-dev-tools/RELEASE.md diff --git a/x-dev-tools/src/main/resources/ant/shield-overrides.xml b/elasticsearch/x-dev-tools/src/main/resources/ant/shield-overrides.xml similarity index 100% rename from x-dev-tools/src/main/resources/ant/shield-overrides.xml rename to elasticsearch/x-dev-tools/src/main/resources/ant/shield-overrides.xml diff --git a/x-dev-tools/src/main/resources/commercial-license-check/elasticsearch_license_header.txt b/elasticsearch/x-dev-tools/src/main/resources/commercial-license-check/elasticsearch_license_header.txt similarity index 100% rename from x-dev-tools/src/main/resources/commercial-license-check/elasticsearch_license_header.txt rename to elasticsearch/x-dev-tools/src/main/resources/commercial-license-check/elasticsearch_license_header.txt diff --git a/x-dev-tools/src/main/resources/commercial-license-check/license_header_definition.xml b/elasticsearch/x-dev-tools/src/main/resources/commercial-license-check/license_header_definition.xml similarity index 100% rename from x-dev-tools/src/main/resources/commercial-license-check/license_header_definition.xml rename to elasticsearch/x-dev-tools/src/main/resources/commercial-license-check/license_header_definition.xml diff --git a/shield/docs/private/indices_replace.asciidoc b/shield/docs/private/indices_replace.asciidoc deleted file mode 100644 index f4f5a79cfe3..00000000000 --- a/shield/docs/private/indices_replace.asciidoc +++ /dev/null @@ -1,19 +0,0 @@ -All the following scenario are run from a user authorized for: `test.*`: read - -[horizontal] -*Existing Indices*::*Action*::*Outcome (executed indices)* -`test1` `test2` `test3` `index1`::`GET _search`::`test1` `test2` `test3` -`test1` `test2` `test3` `index1`::`GET _search/*`::`test1` `test2` `test3` -`test1` `test2` `index1` `index2`::`GET _search/index*`::AuthorizationException -- empty cluster-::`GET _search`::IndexMissingException -- empty cluster-::`GET _search/*`::IndexMissingException -`index1` `index2`::`GET _search`::IndexMissingException -`index1` `index2`::`GET _search/*`::IndexMissingException -`test1` `test2` `index1`::`GET _search/test*,index1`::AuthorizationException -`test1` `test2` `index1`::`GET _search/missing`::AuthorizationException -`test1` `test2` `test3` `index1`::`GET _search/-test2`::`test1` `test3` -`test1` `test2` `test21` `test3` `index1`:: `GET _search/-test2*`::`test1` `test3` -`test1` `test2` `test3` `index1`::`GET msearch first item: all, second item: index1`:: AuthorizationException -`test1` `test2` `test3` `index1`::`GET msearch first item: all, second item: missing`:: AuthorizationException -`test1` `test2` `test3` `index1`::`GET msearch first item: all, second item: test4`:: 1st item:`test1` `test2` `test3`, 2nd item: IndexMissingException -`test1` `test2` `test3` `index1`::`GET msearch first item: all, second item: index*`:: IndexMissingException \ No newline at end of file diff --git a/shield/docs/private/ldap-testing.asciidoc b/shield/docs/private/ldap-testing.asciidoc deleted file mode 100644 index 32066b05fbd..00000000000 --- a/shield/docs/private/ldap-testing.asciidoc +++ /dev/null @@ -1,93 +0,0 @@ -== LDAP Configuration for INTERNAL only Test Servers - -We've two LDAP servers for testing: - -* Active Directory on Windows Server 2012 -* OpenLdap on Suse Enterprise Linux 10.x - -=== Configuration for OpenLdap - -Here is a configuration that works for openldap. This is using OpenSuse's method for creating ldap users that can -authenticate to the box. So it is probably close to a real-world scenario. For SSL the following truststore has both -public certificates in it: elasticsearch-shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks - -[source, yaml] ------------------------------------------------------------- -shield: - ssl.keystore: - path: "/path/to/elasticsearch-shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks" - password: testnode - authc.realms.openldap: - type: ldap - order: 0 - url: "ldaps://54.200.235.244:636" - user_dn_templates: [ "uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com" ] - group_search: - base_dn: "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com" - hostname_verification: false ------------------------------------------------------------- - -=== Configuration for Active Directory - -You could configure Active Directory the same way (with type ldap and user_dn_templates). But where is the fun in that! -Active directory has a simplified (non-standard) authentication workflow that helps us eliminate the templates. - -BUT this technique requires you use a DNS name for your active directory server. Do this adding the following to /etc/hosts: - -`54.213.145.20 ad.test.elasticsearch.com ForestDnsZones.ad.test.elasticsearch.com DomainDnsZones.ad.test.elasticsearch.com` - -[source, yaml] ------------------------------------------------------------- -shield: - authc.realms.ad: - type: active_directory - order: 0 - domain_name: ad.test.elasticsearch.com - ------------------------------------------------------------- - -The above configuration results in a plaintext LDAP connection. For SSL the following configuration is required: -[source, yaml] ------------------------------------------------------------- -shield: - ssl.keystore: - path: "/path/to/elasticsearch-shield/src/test/resources/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks" - password: testnode - authc.realms.ad: - type: active_directory - order: 0 - domain_name: ad.test.elasticsearch.com - url: "ldaps://ad.test.elasticsearch.com:636" - hostname_verification: false - ------------------------------------------------------------- - -=== Users & Groups - -Isn't LDAP fun?! No? Well that's why we've created super heros! - -|======================= -| CN (common name) | uid | group memberships -| Commander Kraken | kraken | Hydra -| Bruce Banner | hulk | Geniuses, SHIELD, Philanthropists, Avengers -| Clint Barton | hawkeye | SHIELD, Avengers -| Jarvis | jarvis | -| Natasha Romanoff | blackwidow | SHIELD, Avengers -| Nick Fury | fury | SHIELD, Avengers -| Phil Colson | phil | SHIELD -| Steve Rogers | cap | SHIELD, Avengers -| Thor | thor | SHIELD, Avengers, Gods, Philanthropists -| Tony Stark | ironman | Geniuses, Billionaries, Playboys, Philanthropists, SHIELD, Avengers -| Odin | odin | Gods -|======================= - -They aren't very good super-heros because they all share the same password: `NickFuryHeartsES`. You'll use the uid -for the username. - -=== Groups -If you want to map group names to es roles, you'll use the fully distinguished names of the groups. The DNs for groups in ad is -`CN={group name},CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com` -the DNs for groups in openldap is -`cn={group name},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com` - -Ping Cam Morris or Bill Hwang for more questions. diff --git a/shield/docs/public/configuring-auditing.asciidoc b/shield/docs/public/configuring-auditing.asciidoc deleted file mode 100644 index 0c13c9399b7..00000000000 --- a/shield/docs/public/configuring-auditing.asciidoc +++ /dev/null @@ -1,290 +0,0 @@ -[[configuring-auditing]] -== Configuring Auditing - -[IMPORTANT] -==== -Audit logs are **disabled** by default. To enable this functionality the following setting should be added to the -`elasticsearch.yml` file: - -[source,yaml] ----------------------------- -shield.audit.enabled: true ----------------------------- -==== - -The audit functionality was added to keep track of important events occurring in Elasticsearch, primarily around security -concerns. Keeping track and persisting these events is essential for any secured environment and potentially provides -evidence for suspicious/malicious activity on the Elasticsearch cluster. - -Shield provides two ways to output these events: in a dedicated `access.log` file stored on the host's file system, or -in an Elasticsearch index on the same or separate cluster. These options are not mutually exclusive. For example, both -options can be enabled through an entry in the `elasticsearch.yml` file: - -[source,yaml] ----------------------------- -shield.audit.outputs: [index, logfile] ----------------------------- - -It is expected that the `index` output type will be used in conjunction with the `logfile` output type. This is -because the `index` output type can lose messages if the target index is unavailable. For this reason, it is recommended -that, if auditing is enabled, then the `logfile` output type should be used as an official record of events. The `index` -output type can be enabled as a convenience to allow historical browsing of events. - -Please also note that, because audit events are batched together before being indexed, they may not appear immediately. -Please refer to the `shield.audit.index.flush_interval` setting below for instructions on how to modify the frequency -with which batched events are flushed. - -[float] -=== Log Entry Types - -Each audit related event that occurs is represented by a single log entry of a specific type (the type represents the -type of the event that occurred). Here are the possible log entry types: - -* `anonymous_access_denied` is logged when the request is denied due to missing authentication token. -* `authentication_failed` is logged when the authentication token cannot be matched to a known user. -* `authentication_failed []` is logged for every realm that fails to present a valid authentication token. - The value of __ is the realm type. -* `access_denied` is logged when an authenticated user attempts an action the user does not have the - <> to perform. -* `access_granted` is logged when an authenticated user attempts an action the user has the correct - privilege to perform. In TRACE level all system (internal) actions are logged as - well (in all other level they're not logged to avoid cluttering of the logs. -* `tampered_request` is logged when the request was detected to be tampered (typically relates to `search/scroll` requests when the scroll id is believed to be tampered) -* `connection_granted` is logged when an incoming tcp connection has passed the ip filtering for a specific profile -* `connection_denied` is logged when an incoming tcp connection did not pass the ip filtering for a specific profile - -To avoid needless proliferation of log entries, Shield enables you to control what entry types should be logged. This can -be done by setting the logging level. The following table lists the log entry types that will be logged for each of the -possible log levels: - -.Log Entry Types and Levels -[options="header"] -|====== -| Log Level | Entry Type -| `ERROR` | `authentication_failed`, `access_denied`, `tampered_request`, `connection_denied` -| `WARN` | `authentication_failed`, `access_denied`, `tampered_request`, `connection_denied`, `anonymous_access_denied` -| `INFO` | `authentication_failed`, `access_denied`, `tampered_request`, `connection_denied`, `anonymous_access_denied`, `access_granted` -| `DEBUG` | (doesn't output additional entry types beyond `INFO`, but extends the information emitted for each entry (see <> below) -| `TRACE` | `authentication_failed`, `access_denied`, `tampered_request`, `connection_denied`, `anonymous_access_denied`, `access_granted`, `connection_granted`, `authentication_failed []`. In addition, internal system requests (self-management requests triggered by Elasticsearch itself) will also be logged for `access_granted` entry type. -|====== - - -[float] -[[audit-log-entry-format]] -=== Log Entry Format - -As mentioned above, every log entry represents an event that occurred in the system. As such, each entry is associated with -a timestamp (at which the event occurred), the component/layer the event is associated with and the entry/event type. In -addition, every log entry (depending ot its type) carries addition information about the event. - -The format of a log entry is shown below: - -[source,txt] ----------------------------------------------------------------------------- -[] [] [] [] ----------------------------------------------------------------------------- - -Where: - -* `` - the timestamp of the entries (in the fomrat configured in `logging.yml` as shown above) -* `` - additional information about the local node that this log entry is printed from (the <> shows how this information can be controlled via settings) -* `` - the layer from which this entry relates to. Can be either `rest`, `transport` or `ip_filter` -* `` - the type of the entry as discussed above. Can be either `anonymous_access_denied`, `authentication_failed`, - `access_denied`, `access_granted`, `connection_granted`, `connection_denied`. -* `` - A comma-separated list of attribute carrying data relevant to the occurred event (formatted as `attr1=[val1], attr2=[val2],...`) - -[[audit-log-entry-local-node-info]] -.Local Node Info Settings -[options="header"] -|====== -| Name | Default | Description -| `shield.audit.logfile.prefix.emit_node_name` | true | When set to `true`, the local node's name will be emitted -| `shield.audit.logfile.prefix.emit_node_host_address` | false | When set to `true`, the local node's IP address will be emitted -| `shield.audit.logfile.prefix.emit_node_host_name` | false | When set to `true`, the local node's host name will be emitted -|====== - -The following tables describe the possible attributes each entry type can carry (the attributes that will be available depend on the configured log level): - -.`[rest] [anonymous_access_denied]` attributes -[options="header"] -|====== -| Attribute | Minimum Log Level | Description -| `origin_address` | WARN | The address the rest request origins from -| `uri` | WARN | The REST endpoint URI -| `request_body` | DEBUG | The body of the request -|====== - -.`[rest] [authentication_failed]` attributes -[options="header"] -|====== -| Attribute | Minimum Log Level | Description -| `origin_address` | ERROR | The address the rest request origins from -| `principal` | ERROR | The principal (username) that failed to authenticate -| `uri` | ERROR | The REST endpoint URI -| `request_body` | DEBUG | The body of the request -| `realm` | TRACE | The realm that failed to authenticate the user. NOTE: A separate entry will be printed for each of the consulted realms -|====== - -.`[transport] [anonymous_access_denied]` attributes -[options="header"] -|====== -| Attribute | Minimum Log Level | Description -| `origin_type` | WARN | The type of the origin the request originated from. Can be either `rest` (request was originated from a rest API request), `transport` (request received on the transport channel), `local_node` (the local node issued the request) -| `origin_address` | WARN | The address the request origins from -| `action` | WARN | The name of the action that was executed -| `request` | DEBUG | The type of the request that was executed -| `indices` | WARN | A comma-separated list of indices this request relates to (when applicable) -|====== - -.`[transport] [authentication_failed]` attributes -[options="header"] -|====== -| Attribute | Minimum Log Level | Description -| `origin_type` | ERROR | The type of the origin the request originated from. Can be either `rest` (request was originated from a rest API request), `transport` (request received on the transport channel), `local_node` (the local node issued the request) -| `origin_address` | ERROR | The address the request origins from -| `principal` | ERROR | The principal (username) that failed to authenticate -| `action` | ERROR | The name of the action that was executed -| `request` | DEBUG | The type of the request that was executed -| `indices` | ERROR | A comma-separated list of indices this request relates to (when applicable) -| `realm` | TRACE | The realm that failed to authenticate the user. NOTE: A separate entry will be printed for each of the consulted realms -|====== - -.`[transport] [access_granted]` attributes -[options="header"] -|====== -| Attribute | Minimum Log Level | Description -| `origin_type` | INFO | The type of the origin the request originated from. Can be either `rest` (request was originated from a rest API request), `transport` (request received on the transport channel), `local_node` (the local node issued the request) -| `origin_address` | INFO | The address the request origins from -| `principal` | INFO | The principal (username) that failed to authenticate -| `action` | INFO | The name of the action that was executed -| `request` | DEBUG | The type of the request that was executed -| `indices` | INFO | A comma-separated list of indices this request relates to (when applicable) -|====== - -.`[transport] [access_denied]` attributes -[options="header"] -|====== -| Attribute | Minimum Log Level | Description -| `origin_type` | ERROR | The type of the origin the request originated from. Can be either `rest` (request was originated from a rest API request), `transport` (request received on the transport channel), `local_node` (the local node issued the request) -| `origin_address` | ERROR | The address the request origins from -| `principal` | ERROR | The principal (username) that failed to authenticate -| `action` | ERROR | The name of the action that was executed -| `request` | DEBUG | The type of the request that was executed -| `indices` | ERROR | A comma-separated list of indices this request relates to (when applicable) -|====== - -.`[transport] [tampered_request]` attributes -[options="header"] -|====== -| Attribute | Minimum Log Level | Description -| `origin_type` | ERROR | The type of the origin the request originated from. Can be either `rest` (request was originated from a rest API request), `transport` (request received on the transport channel), `local_node` (the local node issued the request) -| `origin_address` | ERROR | The address the request origins from -| `principal` | ERROR | The principal (username) that failed to authenticate -| `action` | ERROR | The name of the action that was executed -| `request` | DEBUG | The type of the request that was executed -| `indices` | ERROR | A comma-separated list of indices this request relates to (when applicable) -|====== - -.`[ip_filter] [connection_granted]` attributes -[options="header"] -|====== -| Attribute | Minimum Log Level | Description -| `origin_address` | TRACE | The address the request origins from -| `transport_profile` | TRACE | The principal (username) that failed to authenticate -| `rule` | TRACE | The IP filtering rule that granted the request -|====== - -.`[ip_filter] [connection_denied]` attributes -[options="header"] -|====== -| Attribute | Minimum Log Level | Description -| `origin_address` | ERROR | The address the request origins from -| `transport_profile` | ERROR | The principal (username) that failed to authenticate -| `rule` | ERROR | The IP filtering rule that denied the request -|====== - - -[float] -=== Audit Logs Settings - -As mentioned above, the audit logs are configured in the `logging.yml` file located in `CONFIG_DIR/shield`. The following snippet shows the default logging configuration: - -[[logging-file]] - -.Default `logging.yml` File -[source,yaml] ----- -logger: - shield.audit.logfile: INFO, access_log - -additivity: - shield.audit.logfile: false - -appender: - - access_log: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}-access.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}] %m%n" ----- - -As can be seen above, by default audit information is appended to the `access.log` file located in the -standard Elasticsearch `logs` directory (typically located at `$ES_HOME/logs`). - -[float] -[[audit-index]] -=== Storing Audit Logs in an Elasticsearch Index - -It is possible to store audit logs in an Elasticsearch index. This index can be either on the same cluster, or on -a different cluster (see below). Several settings in `elasticsearch.yml` control this behavior. - -.`audit log indexing configuration` -[options="header"] -|====== -| Attribute | Default Setting | Description -| `shield.audit.outputs` | `logfile` | Must be set to *index* or *[index, logfile]* to enable -| `shield.audit.index.bulk_size` | `1000` | Controls how many audit events will be batched into a single write -| `shield.audit.index.flush_interval` | `1s` | Controls how often to flush buffered events into the index -| `shield.audit.index.rollover` | `daily` | Controls how often to roll over to a new index: hourly, daily, weekly, monthly. -| `shield.audit.index.events.include` | `anonymous_access_denied, authentication_failed, access_granted, access_denied, tampered_request, connection_granted, connection_denied`| The audit events to be indexed. Valid values are `anonymous_access_denied, authentication_failed, access_granted, access_denied, tampered_request, connection_granted, connection_denied`, `system_access_granted`. `_all` is a special value that includes all types. -| `shield.audit.index.events.exclude` | `system_access_granted` | The audit events to exclude from indexing. By default, `system_access_granted` events are excluded; enabling these events results in every internal node communication being indexed, which will make the index size much larger. -|====== - -.audit index settings -The settings for the index that the events are stored in, can also be configured. The index settings should be placed under -the `shield.audit.index.settings` namespace. For example, the following sets the number of shards and replicas to 1 for -the audit indices: - -[source,yaml] ----------------------------- -shield.audit.index.settings: - index: - number_of_shards: 1 - number_of_replicas: 1 ----------------------------- - -[float] -=== Forwarding Audit Logs to a Remote Cluster - -To have audit events stored into a remote Elasticsearch cluster, the additional following options are available. - -.`remote audit log indexing configuration` -[options="header"] -|====== -| Attribute | Default Setting | Description -| `shield.audit.index.client.hosts` | None | Comma separated list of host:port pairs. These hosts should be nodes in the cluster to which you want to index. -| `shield.audit.index.client.cluster.name` | None | The name of the remote cluster. -| `shield.audit.index.client.shield.user` | None | The username:password pair used to authenticate with the remote cluster. -|====== - -Additional settings may be passed to the remote client by placing them under the `shield.audit.index.client` namespace. -For example, to allow the remote client to discover all of the nodes in the remote cluster you could set -the *client.transport.sniff* option. - -[source,yaml] ----------------------------- -shield.audit.index.client.transport.sniff: true ----------------------------- diff --git a/shield/docs/public/configuring-clients-integrations.asciidoc b/shield/docs/public/configuring-clients-integrations.asciidoc deleted file mode 100644 index 1b7255adf12..00000000000 --- a/shield/docs/public/configuring-clients-integrations.asciidoc +++ /dev/null @@ -1,17 +0,0 @@ -[[configuring-clients-integrations]] -== Configuring Clients and Integrations - -You will need to update the configuration for several clients to work with the Shield security plugin. The jump list in -the right side bar lists the configuration information for the clients that support Shield. - -include::configuring-clients-integrations/java.asciidoc[] - -include::configuring-clients-integrations/http.asciidoc[] - -include::configuring-clients-integrations/hadoop.asciidoc[] - -include::configuring-clients-integrations/logstash.asciidoc[] - -include::configuring-clients-integrations/kibana.asciidoc[] - -include::configuring-clients-integrations/marvel.asciidoc[] \ No newline at end of file diff --git a/shield/docs/public/configuring-clients-integrations/hadoop.asciidoc b/shield/docs/public/configuring-clients-integrations/hadoop.asciidoc deleted file mode 100644 index 13b93105549..00000000000 --- a/shield/docs/public/configuring-clients-integrations/hadoop.asciidoc +++ /dev/null @@ -1,8 +0,0 @@ -[[hadoop]] -=== Using Elasticsearch for Apache Hadoop with Shield - -Elasticsearch for Apache Hadoop ("ES-Hadoop") is capable of using HTTP basic and PKI authentication and/or TLS/SSL when accessing an Elasticsearch cluster. For full details please refer to the ES-Hadoop documentation, in particular the `Security` section. - -For authentication purposes, select the user for your ES-Hadoop client (for maintenance purposes it is best to create a dedicated user). Then, assign that user to a role with the privileges required by your Hadoop/Spark/Storm job. Configure ES-Hadoop to use the user name and password through the `es.net.http.auth.user` and `es.net.http.auth.pass` properties. If PKI authentication is enabled, setup the appropriate `keystore` and `truststore` instead through `es.net.ssl.keystore.location` and `es.net.truststore.location` (and their respective `.pass` properties to specify the password). - -For secured transport, enable SSL/TLS through the `es.net.ssl` property by setting it to `true`. Depending on your SSL configuration (keystore, truststore, etc...) you might need to set other parameters as well - please refer to the http://www.elastic.co/guide/en/elasticsearch/hadoop/current/configuration.html[ES-Hadoop] documentation, specifically the `Configuration` and `Security` chapter. diff --git a/shield/docs/public/configuring-clients-integrations/http.asciidoc b/shield/docs/public/configuring-clients-integrations/http.asciidoc deleted file mode 100644 index b7c2aa7b60e..00000000000 --- a/shield/docs/public/configuring-clients-integrations/http.asciidoc +++ /dev/null @@ -1,58 +0,0 @@ -=== Using Elasticsearch HTTP/REST Clients with Shield - -Elasticsearch works with standard HTTP http://en.wikipedia.org/wiki/Basic_access_authentication[basic authentication] -headers to identify the requester. Since Elasticsearch is stateless, this header must be sent with every request: - -[source,shell] --------------------------------------------------- -Authorization: Basic <1> --------------------------------------------------- -<1> The `` is computed as `base64(USERNAME:PASSWORD)` - -[float] -==== Client examples - -This example uses `curl` without basic auth to create an index: - -[source,shell] -------------------------------------------------------------------------------- -curl -XPUT 'localhost:9200/idx' -------------------------------------------------------------------------------- - -[source,json] -------------------------------------------------------------------------------- -{ - "error": "AuthenticationException[Missing authentication token]", - "status": 401 -} -------------------------------------------------------------------------------- - -Since no user is associated with the request above, an authentication error is returned. Now we'll use `curl` with -basic auth to create an index as the `rdeniro` user: - -[source,shell] ---------------------------------------------------------- -curl --user rdeniro:taxidriver -XPUT 'localhost:9200/idx' ---------------------------------------------------------- - -[source,json] ---------------------------------------------------------- -{ - "acknowledged": true -} ---------------------------------------------------------- - -[float] -==== Client Libraries over HTTP - -For more information about how to use Shield with the language specific clients please refer to -https://github.com/elasticsearch/elasticsearch-ruby/tree/master/elasticsearch-transport#authentication[Ruby], -http://elasticsearch-py.readthedocs.org/en/master/#ssl-and-authentication[Python], -https://metacpan.org/pod/Search::Elasticsearch::Role::Cxn::HTTP#CONFIGURATION[Perl], -http://www.elastic.co/guide/en/elasticsearch/client/php-api/current/_security.html[PHP], -http://nest.azurewebsites.net/elasticsearch-net/security.html[.NET], -http://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/auth-reference.html[Javascript] - -//// -Groovy - TODO link -//// diff --git a/shield/docs/public/configuring-clients-integrations/java.asciidoc b/shield/docs/public/configuring-clients-integrations/java.asciidoc deleted file mode 100644 index 8c996d2be6b..00000000000 --- a/shield/docs/public/configuring-clients-integrations/java.asciidoc +++ /dev/null @@ -1,239 +0,0 @@ -=== Using Elasticsearch Java Clients with Shield - -Shield supports the Java http://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html[transport client] for Elasticsearch. The transport client uses the same transport protocol that the cluster nodes use for inter-node communication. It is very efficient as it does not have to marshall and unmarshall JSON requests like a typical REST client. - -NOTE: Using the Java Node Client with Shield is not recommended or supported. - -[float] -[[transport-client]] -==== Configuring the Transport Client to work with Shield - -To use the transport client with Shield, you need to: - -[[java-transport-client-role]] -. Configure a user with the privileges required to start the transport client. A default -`transport_client` role is defined in `roles.yml` that grants the `cluster:monitor/nodes/info` cluster permission. The transport client uses the node info API to fetch information about the nodes in the cluster. If the client is configured to use sniffing, you need to add the -`cluster:monitor/state` cluster permission to the `transport_client` role. - -. Add the Shield JAR file (`shield-2.1.0.jar`) to your CLASSPATH. You can download the Shield distribution and extract the JAR file manually or you can get it from the https://maven.elasticsearch.org/releases/org/elasticsearch/plugin/shield/{version}/shield-{version}.jar[Elasticsearch Maven repository]. -+ -If you are using Maven, you need to add the Shield JAR file as a dependency in your project's `pom.xml` file: -+ -[source,xml] --------------------------------------------------------------- - - - - - - elasticsearch-releases - https://maven.elasticsearch.org/releases - - true - - - false - - - ... - - ... - - - - - org.elasticsearch.plugin - shield - 2.1.0 - - ... - - ... - - --------------------------------------------------------------- -+ -If you are using Gradle, you need to add the Shield JAR file as a dependency in your `build.gradle` file: -+ -[source,groovy] --------------------------------------------------------------- -repositories { - /* ... Any other repositories ... */ - - // Add the Elasticsearch Maven Repository - maven { - url "https://maven.elasticsearch.org/releases" - } -} - -dependencies { - // Provide the Shield jar on the classpath for compilation and at runtime - // Note: Many projects can use the Shield jar as a runtime dependency - compile "org.elasticsearch.plugin:shield:2.1.0" - - /* ... */ -} --------------------------------------------------------------- - -. Set up the transport client. At a minimum, you must configure `shield.user` to include the name and password of your transport client user in your requests. The following snippet configures the user credentials globally--every request submitted with this client includes the `transport_client_user` credentials in its headers. -+ -[source,java] -------------------------------------------------------------------------------------------------- -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.shield.ShieldPlugin -... - -TransportClient client = TransportClient.builder() - .addPlugin(ShieldPlugin.class) - .settings(Settings.builder() - .put("cluster.name", "myClusterName") - .put("shield.user", "transport_client_user:changeme") - ... - .build()) - .addTransportAddress(new InetSocketTransportAddress("localhost", 9300)) - .addTransportAddress(new InetSocketTransportAddress("localhost", 9301)); -------------------------------------------------------------------------------------------------- -+ -WARNING: If you configure a transport client without SSL, passwords are sent in plaintext. -+ -You can also add an `Authorization` header to each request. If you've configured global authorization credentials, the `Authorization` header overrides the global authentication credentials. This is useful when an application has multiple users who access Elasticsearch using the same client. You can set the global token to a user that only has the `transport_client` role, and add the `transport_client` role to the individual users. -+ -For example, the following snippet adds the `Authorization` header to a search request: -+ -[source,java] --------------------------------------------------------------------------------------------------- -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.shield.authc.support.SecuredString; -import org.elasticsearch.shield.ShieldPlugin - -import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -... - -TransportClient client = TransportClient.builder() - .addPlugin(ShieldPlugin.class) - .settings(Settings.builder() - .put("cluster.name", "myClusterName") - .put("shield.user", "transport_client_user:changeme") - ... - .build()) - .build() - .addTransportAddress(new InetSocketTransportAddress("localhost", 9300)) - .addTransportAddress(new InetSocketTransportAddress("localhost", 9301)); - -String token = basicAuthHeaderValue("test_user", new SecuredString("changeme".toCharArray())); - -client.prepareSearch().putHeader("Authorization", token).get(); --------------------------------------------------------------------------------------------------- - -. Enable SSL to authenticate clients and encrypt communications. To enable SSL, you need to: - -.. Configure the client's keystore path and password. Client authentication requires every -client to have a certification signed by a trusted CA. -+ -NOTE: Client authentication is enabled by default. For information about disabling client authentication, see <>. -+ -[source,java] --------------------------------------------------------------------------------------------------- -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.shield.ShieldPlugin -... - -TransportClient client = TransportClient.builder() - .addPlugin(ShieldPlugin.class) - .settings(Settings.builder() - .put("cluster.name", "myClusterName") - .put("shield.user", "transport_client_user:changeme") - .put("shield.ssl.keystore.path", "/path/to/client.jks") (1) - .put("shield.ssl.keystore.password", "password") - ... - .build()); --------------------------------------------------------------------------------------------------- -+ -(1) The `client.jks` keystore must contain the client's signed certificate and the CA certificate. -+ -.. Enable the SSL transport by setting `shield.transport.ssl` to `true` in the client configuration. -+ -[source,java] --------------------------------------------------------------------------------------------------- -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.shield.ShieldPlugin -... - -TransportClient client = TransportClient.builder() - .addPlugin(ShieldPlugin.class) - .settings(Settings.builder() - .put("cluster.name", "myClusterName") - .put("shield.user", "transport_client_user:changeme") - .put("shield.ssl.keystore.path", "/path/to/client.jks") (1) - .put("shield.ssl.keystore.password", "password") - .put("shield.transport.ssl", "true") - ... - .build()) - .addTransportAddress(new InetSocketTransportAddress("localhost", 9300)) - .addTransportAddress(new InetSocketTransportAddress("localhost", 9301)); --------------------------------------------------------------------------------------------------- - -[float] -[[disabling-client-auth]] -===== Disabling Client Authentication - -If you want to disable client authentication, you can use a client-specific transport protocol. For more information, <>. - -If you are not using client authentication and sign the Elasticsearch node certificates with your own CA, you need to set the truststore path and password in the client configuration: - -[source,java] ------------------------------------------------------------------------------------------------------- -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.shield.ShieldPlugin -... - -TransportClient client = TransportClient.builder() - .addPlugin(ShieldPlugin.class) - .settings(Settings.builder() - .put("cluster.name", "myClusterName") - .put("shield.user", "test_user:changeme") - .put("shield.ssl.truststore.path", "/path/to/truststore.jks") (1) - .put("shield.ssl.truststore.password", "password") - .put("shield.transport.ssl", "true") - ... - .build()) - .addTransportAddress(new InetSocketTransportAddress("localhost", 9300)) - .addTransportAddress(new InetSocketTransportAddress("localhost", 9301)); ------------------------------------------------------------------------------------------------------- -(1) The `truststore.jks` truststore must contain the certificate of the CA that signed the Elasticsearch node certificates. - -NOTE: If you are using a public CA that is already trusted by the Java runtime, you to not need to set the `shield.ssl.truststore.path` and `shield.ssl.truststore.password`. - -[float] -[[connecting-anonymously]] -===== Connecting Anonymously added[1.1.0] - -To enable the transport client to connect anonymously, you must assign the anonymous user the privileges defined in the <> role. Anonymous access must also be enabled, of course. For more information, see <>. - -[float] -[[shield-client]] -==== Shield Client - -Shield exposes its own API through the `ShieldClient` class. At the moment, this API only exposes one operation, for clearing the realm caches. `ShieldClient` is a wrapper around the existing clients (any client class implementing `org.elasticsearch.client.Client`). - -The following example shows how you can clear Shield's realm caches using `ShieldClient`: - -[source,java] ------------------------------------------------------------------------------------------------------- -import static org.elasticsearch.node.NodeBuilder.*; -... - -Client client = ... // create the transport client - -ShieldClient shieldClient = new ShieldClient(client); -ClearRealmCacheResponse response = shieldClient.authc().prepareClearRealmCache() - .realms("ldap1", "ad1") (1) - .usernames("rdeniro") - .get(); ------------------------------------------------------------------------------------------------------- - -(1) Clears the `ldap1` and `ad1` realm caches for the `rdeniro` user. - - diff --git a/shield/docs/public/configuring-clients-integrations/kibana.asciidoc b/shield/docs/public/configuring-clients-integrations/kibana.asciidoc deleted file mode 100644 index e38c962d016..00000000000 --- a/shield/docs/public/configuring-clients-integrations/kibana.asciidoc +++ /dev/null @@ -1,316 +0,0 @@ - -[[kibana]] -=== Using Kibana with Shield - -Shield supports both Kibana 3 and Kibana 4.0 and later. To set things up, you need to update your Kibana configuration and define and assign roles for your Kibana users in Shield. If you're using Kibana 3, you also need to enable cross-origin resource sharing (CORS) in Elasticsearch. -If you're using Kibana 4, you need to configure credentials for the Kibana server. The following sections provide step-by-step instructions for using <> and <> with Shield. - -NOTE: With Shield installed, if you load a Kibana dashboard that accesses data in an index that you are not authorized to view, you get an error that indicates the index does not exist. Kibana and Shield do not currently provide a way to control which users can load which dashboards. - -[[using-kibana3-with-shield]] -[float] -==== Using Kibana 3 with Shield - -Kibana users have to authenticate when your cluster has Shield installed. You configure Shield roles for your Kibana users to control what data those users can access. In addition, you can encrypt communications between the browser and Elasticsearch. - -[[cors]] -To use Kibana 3 with Shield: - -. Configure Kibana to use credentials when communicating with Elasticsearch. To do this, set `withCredentials` to `true` in the `elasticsearch` property in Kibana's `config.js` file: -+ -[source,yaml] ------------------------------------- -elasticsearch: {server: "http://YOUR_ELASTICSEARCH_SERVER:9200", withCredentials: true} ------------------------------------- -+ -IMPORTANT: If SSL encryption is enabled in Shield, specify the HTTPS protocol in the Elasticsearch URL rather than HTTP. - -. Enable CORS in Elasticsearch and allow credentialed requests. To do this, set the following properties in `elasticsearch.yml` on each node in your cluster and restart the nodes: -+ -[source,yaml] ------------------------------------- -http.cors.enabled: true <1> -http.cors.allow-origin: "https://MYHOST:MYPORT" <2> -http.cors.allow-credentials: true <3> ------------------------------------- -<1> Enables CORS. For more information, see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-http.html[HTTP] in the Elasticsearch Reference. -<2> Specifies the webserver you are using for Kibana. Note that you must explicitly specify your server's protocol, hostname, and port--you cannot simply specify a wildcard `*` when using credentialed requests. -<3> Sends authentication headers to the browser. -+ -NOTE: If you are using a source build of Kibana 3, you might encounter authentication errors when trying to connect to Kibana 3 after deploying Shield and configuring the `http.cors.allow-credentials` property. If you do, simply clear your browser's cache and reconnect. - -. Derive Kibana 3 user roles from the default <> and add them to `roles.yml` to control which indices your Kibana users can access. Kibana users need access to the indices that they will be working with and the `kibana-int` index where their dashboards are stored. The default `kibana3` role grants read access to all indices and full access to the `kibana-int` index. -+ -IMPORTANT: We strongly recommend creating custom `kibana3` user roles -to limit access to specific indices according to your organization's goals and policies. You can define as many different roles for your Kibana users as you need. -+ -To constrain Kibana's access to specific indices, explicitly specify the index names in your role. When configuring a role for a Kibana user and granting access to a specific index, at a minimum the user needs the following privileges on the index: -+ --------------------------------------------------------------------------------- -indices:admin/mappings/fields/get -indices:admin/validate/query -indices:data/read/search -indices:data/read/msearch -indices:admin/get --------------------------------------------------------------------------------- -+ -For example, the following `kibana3_monitoring` role only allows users to build dashboards using data in the `logstash-*` indices. -+ -[source,yaml] --------------------------------------------------------------------------------- -kibana3_monitoring: - cluster: - - cluster:monitor/nodes/info <1> - indices: - 'logstash-*': - - indices:admin/mappings/fields/get - - indices:admin/validate/query - - indices:data/read/search - - indices:data/read/msearch - - indices:admin/get - '.kibana-int': <2> - - indices:data/read/get - - indices:data/read/search - - indices:data/write/delete - - indices:data/write/index - - create_index --------------------------------------------------------------------------------- -<1> Kibana 3 uses the cluster permission to access the `/_nodes` endpoint to check the node version. -<2> All Kibana users need access to the `kibana-int` index. - -. Assign the appropriate roles to your Kibana users or groups of users: - -** If you're using the default `esusers` realm, you can assign roles when you <>, or modify the role assignments with the <> command. For example, the following command creates a user named `jacknich` and assigns the `kibana3_monitoring` role: -+ -[source,console] --------------------------------------------------------------------------------- -esusers useradd jacknich -r kibana3_monitoring -p password --------------------------------------------------------------------------------- - -** If you are using an LDAP or Active Directory realm, you can either assign roles on a per user basis, or assign roles to groups of users. By default, role mappings are stored in <>. For example, the following snippet assigns the `kibana3_monitoring` role to the group named `admins` and the user named Jack Nicholson: -+ -[source,yaml] --------------------------------------------------------------------------------- -kibana3_monitoring: - - "cn=admins,dc=example,dc=com" - - "cn=Jack Nicholson,dc=example,dc=com" --------------------------------------------------------------------------------- - -. If you have <> in Shield and are using your own Certificate Authority (CA) to sign certificates for your nodes, configure your browser or operating system to trust your CA. When you access Kibana, your browser verifies that the certificate received from the Elasticsearch node is trusted before sending a request to the node. Establishing this trust requires that either your browser or operating system trust the CA that signed the node's certificate. Consult your local IT professional for information about the recommended procedure for adding trusted CAs in your organization. - -. Access Kibana 3 from your browser and verify that you can sign in as a user. For example, you could log in as the `jacknich` user created in step 4. - -[float] -[[using-kibana4-with-shield]] -==== Using Kibana 4 with Shield -Kibana users have to authenticate when your cluster has Shield installed. You configure Shield roles for your Kibana users to control what data those users can access. Kibana 4 runs a webserver that makes requests to Elasticsearch on the client's behalf, so you also need to configure credentials for the Kibana server so those requests can be authenticated. In addition, you can encrypt communications between the Kibana server and Elasticsearch. - -To use Kibana 4 with Shield: - -. Configure credentials for the Kibana server. The Kibana server needs access to the cluster monitoring APIs and the `.kibana` index. The server does _not_ need access to user indexes. The required privileges are specified in the <> provided in the default Shield <> file. - -.. Create a user account for the Kibana server and assign it the `kibana4_server` role. For example, if you're using the default `esusers` realm, you can create a `kibana-server` user with the <> command: -+ -[source,console] --------------------------------------------------------------------------------- -esusers useradd kibana4-server -r kibana4_server -p password --------------------------------------------------------------------------------- -+ -If you are using an LDAP, Active Directory, or PKI realm, you need to create a user for the -Kibana server and map the user's distinguished name to the `kibana4_server` role in the Shield <> file. By default, role mappings are stored in `config/shield/role_mapping.yml`. For example, the following snippet assigns the `kibana4_server` role to an LDAP or Active Directory user named `kibana-server`: -+ -[source,yaml] --------------------------------------------------------------------------------- -kibana4_server: - - "cn=kibana-server,cn=applications,dc=example,dc=com" --------------------------------------------------------------------------------- -+ -For PKI realms, you specify the user's common name, organizational unit, and organization: -+ -[source,yaml] --------------------------------------------------------------------------------- -kibana4_server: - - "cn=kibana-server,ou=example,o=com" --------------------------------------------------------------------------------- - -.. Specify the credentials for your Kibana server user in the Kibana configuration -file, `/config/kibana.yml`. -+ -[source,yaml] --------------------------------------------------------------------------------- -elasticsearch.username: kibana4-server -elasticsearch.password: password --------------------------------------------------------------------------------- - -[[kibana4-roles]] -. Derive Kibana 4 user roles from the default <> and add them to `roles.yml` to control which indices your Kibana users can access. Kibana users need access to the indices that they will be working with and the `.kibana` index where their saved searches, visualizations, and dashboards are stored. The default `kibana4` role grants read access to all indices and full access to the `.kibana` index. -+ -IMPORTANT: We strongly recommend creating custom `kibana4` user roles -to limit access to specific indices according to your organization's goals and policies. You can define as many different roles for your Kibana 4 users as you need. -+ -To constrain Kibana's access to specific indices, explicitly specify the index names in your role. When configuring a role for a Kibana user and granting access to a specific index, at a minimum the user needs the following privileges on the index: -+ --------------------------------------------------------------------------------- -indices:admin/mappings/fields/get -indices:admin/validate/query -indices:data/read/search -indices:data/read/msearch -indices:admin/get --------------------------------------------------------------------------------- -+ -For example, the following `kibana4_monitoring` role only allows users to discover and visualize data in the `logstash-*` indices. -+ -[source,yaml] --------------------------------------------------------------------------------- -kibana4_monitoring: - cluster: - - cluster:monitor/nodes/info - - cluster:monitor/health - indices: - 'logstash-*': - - indices:admin/mappings/fields/get - - indices:admin/validate/query - - indices:data/read/search - - indices:data/read/msearch - - indices:admin/get - '.kibana': <1> - - indices:admin/create - - indices:admin/exists - - indices:admin/mapping/put - - indices:admin/mappings/fields/get - - indices:admin/refresh - - indices:admin/validate/query - - indices:data/read/get - - indices:data/read/mget - - indices:data/read/search - - indices:data/write/delete - - indices:data/write/index - - indices:data/write/update --------------------------------------------------------------------------------- -<1> All Kibana users need access to the `.kibana` index. - -. Assign the appropriate roles to your Kibana users or groups of users: - -** If you're using the default `esusers` realm, you can assign roles when you <>, or modify the role assignments with the <> command. For example, the following command creates a user named `jacknich` and assigns the `kibana4_monitoring` role: -+ -[source,console] --------------------------------------------------------------------------------- -esusers useradd jacknich -r kibana4_monitoring -p password --------------------------------------------------------------------------------- - -** If you are using an LDAP or Active Directory realm, you can either assign roles on a per user basis, or assign roles to groups of users. By default, role mappings are stored in <>. For example, the following snippet assigns the `kibana4_monitoring` role to the group named `admins` and the user named Jack Nicholson: -+ -[source,yaml] --------------------------------------------------------------------------------- -kibana4_monitoring: - - "cn=admins,dc=example,dc=com" - - "cn=Jack Nicholson,dc=example,dc=com" --------------------------------------------------------------------------------- - -. If you have enabled SSL encryption in Shield, configure Kibana 4 to connect to Elasticsearch via HTTPS. To do this: - -.. Specify the HTTPS protocol in the `elasticsearch.url` setting in the Kibana configuration file, `kibana.yml`: -+ -[source,yaml] --------------------------------------------------------------------------------- -elasticsearch.url: "https://.com:9200" --------------------------------------------------------------------------------- - -.. If you are using your own CA to sign certificates for Elasticsearch, set the `elasticsearch.ssl.ca` setting in `kibana.yml` to specify the location of the PEM file. -+ -[source,yaml] --------------------------------------------------------------------------------- -elasticsearch.ssl.ca: /path/to/your/cacert.pem --------------------------------------------------------------------------------- - -. Configure Kibana 4 to encrypt communications between the browser and the Kibana server. To do this, configure the `server.ssl.key` and `server.ssl.cert` properties in `kibana.yml`: -+ -[source,yaml] --------------------------------------------------------------------------------- -server.ssl.key: /path/to/your/server.key -server.ssl.cert: /path/to/your/server.crt --------------------------------------------------------------------------------- -+ -Once you enable SSL encryption between the browser and the Kibana server, access Kibana via HTTPS. For example, `https://localhost:5601`. -+ -NOTE: Enabling browser encryption is required to prevent passing user credentials in the clear. - -. Restart Kibana and verify that you can sign in as a user. If you are running Kibana locally, -go to `localhost:5601` and enter the credentials for a user you've assigned a Kibana user role. For example, you could log in as the `jacknich` user created in step 3. -+ -NOTE: Sign in as a Kibana user--the Kibana server credentials should only be used internally by the Kibana server. The `kibana4_server` role doesn't grant permission to create the `.kibana` index or access user indices. - -[float] -[[default-roles]] -==== Default Roles for Kibana - -Default roles for Kibana 3 and Kibana 4 are provided in `roles.yml`. - -IMPORTANT: The default user roles grant read access to all indices. We strongly recommend deriving custom roles for your Kibana users that limit access to specific indices according to your organization's goals and policies. - -[[kibana3-user-role]] -.Kibana 3 User Role -[source,yaml] --------------------------------------------------------------------------------- -kibana3: - cluster: cluster:monitor/nodes/info - indices: - '*': indices:data/read/search, indices:data/read/get, indices:admin/get - 'kibana-int': indices:data/read/search, indices:data/read/get, indices:data/write/delete, indices:data/write/index, create_index --------------------------------------------------------------------------------- - -[[kibana4-user-role]] -.Kibana 4 User Role -[source,yaml] --------------------------------------------------------------------------------- -kibana4: - cluster: - - cluster:monitor/nodes/info - - cluster:monitor/health - indices: - '*': - - indices:admin/mappings/fields/get - - indices:admin/validate/query - - indices:data/read/search - - indices:data/read/msearch - - indices:admin/get - '.kibana': - - indices:admin/exists - - indices:admin/mapping/put - - indices:admin/mappings/fields/get - - indices:admin/refresh - - indices:admin/validate/query - - indices:data/read/get - - indices:data/read/mget - - indices:data/read/search - - indices:data/write/delete - - indices:data/write/index - - indices:data/write/update - - indices:admin/create --------------------------------------------------------------------------------- - - -[[kibana4-server-role]] -.Kibana 4 Server Role -[source,yaml] --------------------------------------------------------------------------------- -kibana4_server: - cluster: - - cluster:monitor/nodes/info - - cluster:monitor/health - indices: - '.kibana': - - indices:admin/create - - indices:admin/exists - - indices:admin/mapping/put - - indices:admin/mappings/fields/get - - indices:admin/refresh - - indices:admin/validate/query - - indices:data/read/get - - indices:data/read/mget - - indices:data/read/search - - indices:data/write/delete - - indices:data/write/index - - indices:data/write/update --------------------------------------------------------------------------------- - diff --git a/shield/docs/public/configuring-clients-integrations/logstash.asciidoc b/shield/docs/public/configuring-clients-integrations/logstash.asciidoc deleted file mode 100644 index b1ee1288e3a..00000000000 --- a/shield/docs/public/configuring-clients-integrations/logstash.asciidoc +++ /dev/null @@ -1,200 +0,0 @@ -[[logstash]] -=== Using Logstash with Shield - -IMPORTANT: Shield 2.0+ is compatible with Logstash 2.0 and above. - -Logstash provides Elasticsearch https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html[output], https://www.elastic.co/guide/en/logstash/current/plugins-inputs-elasticsearch.html[input] and https://www.elastic.co/guide/en/logstash/current/plugins-filters-elasticsearch.html[filter] plugins -used to index and retrieve documents through HTTP, transport or client node protocols. -All plugins support authentication and encryption over HTTP, while the output plugin additionally supports these -features over the transport protocol. - -NOTE: When using the `elasticsearch` output, only the `transport` and `http` protocol are supported (i.e. `node` protocol is unsupported) - -[float] -[[ls-user]] -==== Creating a user - -By default, the Shield plugin installs a dedicated user <> that enables the creation of indices with names -that match the `logstash-*` regular expression, along with privileges to read, scroll, index, update, and delete -documents on those indices: - -[source,yaml] --------------------------------------------------------------------------------------------- -logstash: - cluster: indices:admin/template/get, indices:admin/template/put - indices: - 'logstash-*': indices:data/write/bulk, indices:data/write/delete, indices:data/write/update, indices:data/read/search, indices:data/read/scroll, create_index --------------------------------------------------------------------------------------------- - -See the <> section for information on modifying roles. - -Create a user associated with the `logstash` role on the Elasticsearch cluster, using the <>: - -[source,shell] --------------------------------------------------- -esusers useradd -p -r logstash --------------------------------------------------- - -NOTE: When using the transport protocol, the logstash user requires the predefined `transport_client` role in addition to the `logstash` role shown above (`-r logstash,transport_client`). - -Once you've created the user, you are ready to configure Logstash. - -[float] -[[ls-http]] -==== Connecting with HTTP/HTTPS - -Logstash communicates with the Elasticsearch cluster through the REST APIs over HTTP. - -[float] -[[ls-http-auth]] -===== Authentication for HTTP protocol - -HTTP protocol supports both basic auth and client-certificate authentication through the use of Public Key Infrastructure (PKI). - -[float] -[[ls-http-auth-basic]] -===== Basic Authentication - -The  input, filter, and output plugins all support HTTP Basic Authentication. To use basic authentication when connecting to an instance of Elasticsearch with Shield, you configure the plugins to include username and password credentials with each request. For example, the following snippet configures credentials for the output plugin. The credentials are configured the same way for each plugin type. - -[source, shell] --------------------------------------------------- -input { ... } -output { - elasticsearch { - protocol => "http" - ... - user => ... # string - password => ... # string - } -} --------------------------------------------------- - -[float] -[[ls-http-auth-pki]] -===== PKI Authentication - -Elasticsearch Output supports the use of X.509 client-certificate to authenticate Logstash requests. To enable this you need to set up the following configuration parameters: - -[source, shell] --------------------------------------------------- -input { ... } -output { - elasticsearch { - ... - keystore => ... # string - keystore_password => ... # string - } -} --------------------------------------------------- - -[float] -[[ls-http-ssl]] -===== SSL/TLS Configuration for HTTPS - -To enable SSL/TLS encryption for HTTPS, use the following configuration block: - -[source, shell] --------------------------------------------------- -input { ... } -output { - elasticsearch { - ... - ssl => true - cacert => '/path/to/cert.pem' <1> - } -} --------------------------------------------------- -<1> The path to the `.pem` file in your filesystem that contains the Certificate Authority's certificate. - -[float] -[[ls-transport]] -==== Connecting with Transport protocol - -When using the `elasticsearch_java` plugins in Logstash, you can set the `protocol` option to `transport`. With `transport`, Logstash communicates with the Elasticsearch cluster through the same -protocol nodes use between each other. - -In order to unlock this option, it's necessary to install an additional plugin in Logstash using the following command: - -[source, shell] --------------------------------------------------- -bin/plugin install logstash-output-elasticsearch_java_shield --------------------------------------------------- - -[float] -[[ls-transport-auth]] -===== Authentication for Transport protocol - -Transport protocol supports both basic auth and client-certificate authentication through the use of Public Key Infrastructure (PKI). - -[float] -[[ls-transport-auth-basic]] -===== Basic Authentication - -To connect to an instance of Elasticsearch with Shield using basic auth, set up the username and password credentials with the following configuration parameters: - -[source, shell] --------------------------------------------------- -input { ... } -output { - elasticsearch_java { - protocol => "transport" - ... - user => ... # string - password => ... # string - } -} --------------------------------------------------- - -[float] -[[ls-transport-auth-pki]] -===== PKI Authentication - -To connect to an instance of Elasticsearch with Shield using client-certificate authentication you need to setup the keystore path which contain the client's certificate and the keystore password in the configuration: - -[source, shell] --------------------------------------------------- -input { ... } -output { - elasticsearch_java { - protocol => "transport" - ... - ssl => true - keystore => ... # string - keystore_password => ... # string - } -} --------------------------------------------------- - -[float] -[[ls-transport-conf]] -===== SSL Configuration for Transport protocols - -Specify the paths to the keystore and truststore `.jks` files with the following configuration parameters: - -[source, shell] --------------------------------------------------- -input { ... } -output { - elasticsearch_java { - protocol => "transport" - host => ... # string (optional) - cluster => ... # string (optional) - ... - ssl => true - keystore => ... # string - keystore_password => ... # string - truststore => ... # string - truststore_password => ... # string - } -} --------------------------------------------------- - -For more information on encryption and certificates, see the <> section: - -[float] -[[ls-failure]] -==== Failures - -Logstash raises an exception that halts the processing pipeline when the server's certificate does not validate over SSL -on any of the protocols discussed in this section. Same for the invalid user credentials. diff --git a/shield/docs/public/configuring-clients-integrations/marvel.asciidoc b/shield/docs/public/configuring-clients-integrations/marvel.asciidoc deleted file mode 100644 index 76b1f24f235..00000000000 --- a/shield/docs/public/configuring-clients-integrations/marvel.asciidoc +++ /dev/null @@ -1,115 +0,0 @@ -[[marvel]] -=== Using Marvel with Shield - -https://www.elastic.co/guide/en/marvel/current/introduction.html[Marvel] consists of two -components: a Marvel agent that you install on on each node in your cluster, and a Marvel application you install in https://www.elastic.co/guide/en/kibana/current/introduction.html[Kibana]. The Marvel agent collects and indexes metrics from Elasticsearch and you visualize the data through the Marvel dashboards in Kibana. The agent can index data on the same cluster, or send it to an external monitoring cluster. - -To use Marvel with Shield enabled, you need to <> and create at least one user for the Marvel app. If you are using an external monitoring cluster, you also need to configure a user for the Marvel agent and configure the agent to use the appropriate -credentials when communicating with the monitoring cluster. - -[float] -[[marvel-app-users]] -==== Setting Up Marvel App Users - -When Shield is enabled, Kibana users are prompted to log in when they access the UI. To use -the Marvel app, a user must have access to the Kibana indices and permission to read from the -Marvel indices. - -You set up Marvel app users on the cluster where the monitoring data is being stored. To grant -all of the necessary permissions, assign user the `kibana_user` and `marvel_user` roles defined -in `roles.yml`: - -* If you're using the default `esusers` realm, you can assign roles when you <>, or modify the role assignments with the <> command. For example, -the following command creates a user named `jacknich` and assigns the `marvel_user` role: -+ -[source,console] --------------------------------------------------------------------------------- -esusers useradd jacknich -r kibana_user,marvel_user -p password --------------------------------------------------------------------------------- - -* If you are using an LDAP or Active Directory realm, you can either assign roles on a per user -basis, or assign roles to groups of users. By default, role mappings are configured in -<>. For example, the following snippet assigns -the user named Jack Nicholson to the `kibana_user` and `marvel_user` roles: -+ -[source,yaml] --------------------------------------------------------------------------------- -kibana_user: - - "cn=Jack Nicholson,dc=example,dc=com" -marvel_user: - - "cn=Jack Nicholson,dc=example,dc=com" --------------------------------------------------------------------------------- - -[float] -[[configuring-marvel-agent-shield]] -==== Configuring Marvel Agent to Communicate with a Shield-Enabled Monitoring Cluster - -To configure the Marvel agent to communicate with a secured monitoring cluster: - -. Configure a user on the monitoring cluster who has the `marvel_agent` role, which is defined in -`roles.yml`. For example: -+ -[source,console] --------------------------------------------------------------------------------- -esusers useradd agent-user -r marvel_agent -p password --------------------------------------------------------------------------------- -+ -.Marvel Agent Role -[source,yaml] --------------------------------------------------- -marvel_agent: - cluster: indices:admin/template/get, indices:admin/template/put - indices: - '.marvel-*': indices:data/write/bulk, create_index --------------------------------------------------- - -. On each node in the cluster being monitored, configure a Marvel HTTP exporter -in `elasticsearch.yml` and resart Elasticsearch. In the exporter configuration, -you need to: -+ --- -.. Set the `type` to `http`. -.. Specify the location of the monitoring cluster in the `host` setting. -.. Provide the agent user credentials with the `username` and `password` settings. - -For example: - -[source,yaml] --------------------------------------------------- -marvel.agent.exporters: - id1: - type: http - host: ["http://es-mon1:9200", "http://es-mon2:9200"] - auth: - username: agent-user - password: password --------------------------------------------------- - -If SSL/TLS is enabled on the monitoring cluster: - -.. Specify the HTTPS protocol when setting the monitoring server host. -.. Specify a truststore that contains the CA certificate to use to verify the identities of the -nodes in the monitoring cluster. You need to set the `truststore.path` and -`truststore.password`. - -For example: - -[source,yaml] --------------------------------------------------- -marvel.agent.exporters: - id1: - type: http - host: ["https://es-mon1:9200", "https://es-mon2:9200"] - - auth: - username: agent-user - password: password - - ssl: - truststore.path: /path/to/file - truststore.password: password - id2: - type: local --------------------------------------------------- --- \ No newline at end of file diff --git a/shield/docs/public/configuring-rbac.asciidoc b/shield/docs/public/configuring-rbac.asciidoc deleted file mode 100644 index 3991afc6941..00000000000 --- a/shield/docs/public/configuring-rbac.asciidoc +++ /dev/null @@ -1,155 +0,0 @@ -[[configuring-rbac]] -== Configuring Role-based Access Control - -Shield introduces the concept of _action authorization_ to Elasticsearch. Action authorization restricts the actions -users can execute on the cluster. Shield implements authorization as Role Based Access Control (RBAC), where all -actions are restricted by default. Users are associated with roles that define a set of actions that are allowed -for those users. - -[[roles]] -[float] -=== Roles, Permissions and Privileges - -Privileges are actions or a set of actions that users may execute in Elasticsearch. For example, the ability to run a -query is a privilege. - -A permission is a set of privileges associated with one or more secured objects. For example, a permission could allow -querying or reading all documents of index `i1`. There are two types of secured objects in Elasticsearch - -cluster and indices. Cluster permissions grant access to cluster-wide administrative and monitoring actions. Index -permissions grant data access, including administrative and monitoring actions on specific indices in the cluster. - -A role is a named set of permissions. For example, you could define a role as a logging administrator. The logging -administrator is allowed to take all actions on indices named `logs-*`. - -As an administrator, you will need to define the roles that you want to use, then assign users to the roles. - -[[defining-roles]] -=== Defining Roles -Roles are defined in the role definition file `roles.yml` located in `CONFIG_DIR/shield`. -This is a YAML file where each entry defines the unique role name and the cluster and indices permissions associated -with it. - -[IMPORTANT] -============================== -The `roles.yml` file is managed locally by the node and is not managed globally by the cluster. This means that -with a typical multi-node cluster, the exact same changes need to be applied on each and every node in the cluster. - -A safer approach would be to apply the change on one of the nodes and have the `roles.yml` distributed/copied to -all other nodes in the cluster (either manually or using a configuration management system such as Puppet or Chef). -============================== - -The following snippet shows an example configuration: - -[source,yaml] ------------------------------------ -# All cluster rights -# All operations on all indices -admin: - cluster: all - indices: - '*': - privileges: all - -# Monitoring cluster privileges -# All operations on all indices -power_user: - cluster: monitor - indices: - '*': - privileges: all - -# Only read operations on indices -user: - indices: - '*': - privileges: read - -# Only read operations on indices named events_* -events_user: - indices: - 'events_*': - privileges: read ------------------------------------ - -[[valid-role-name]] -NOTE: A valid role name must be at least 1 character and no longer than 30 characters. It must begin with a letter - (`a-z`) or an underscore (`_`). Subsequent characters can be letters, underscores (`_`), digits (`0-9`) or any - of the following symbols `@`, `-`, `.` or `$` - - -The above example defines these roles: - -|======================= -| `admin` | Has full access (all privileges) on the cluster and full access on all indices in the cluster. -| `power_user` | Has monitoring-only access on the cluster, enabling the user to request cluster metrics, information, - and settings, without the ability to update settings. This user also has full access on all indices in - the cluster. -| `user` | Cannot update or monitor the cluster. Has read-only access to all indices in the cluster. -| `events_user` | Has read-only access to all indices with the `events_` prefix. -|======================= - -See the complete list of available <>. - - -=== Granting Privileges for Specific Actions - -The Shield security plugin enables access to specific actions in Elasticsearch. Access control using specific actions -provides a finer level of granularity than roles based on named privileges. - -The role in the following example allows access to document `GET` actions for a specific index and nothing else: - -.Example Role Using Action-level Access Control -[source,yaml] ---------------------------------------------------- -# Only GET read action on index named events_index -get_user: - indices: - 'events_index': - privileges: 'indices:data/read/get' ---------------------------------------------------- - -See the complete list of available <>. - -TIP: When specifying index names, you can use indices and aliases with their full names or regular expressions that - refer to multiple indices. - -* Wildcard (default) - simple wildcard matching where `*` is a placeholder for zero or more characters, `?` is a - placeholder for a single character and `\` may be used as an escape character. - -* Regular Expressions - A more powerful syntax for matching more complex patterns. This regular expression is based on - Lucene's regexp automaton syntax. To enable this syntax, it must be wrapped within a pair of forward slashes (`/`). - Any pattern starting with `/` and not ending with `/` is considered to be malformed. - -.Example Regular Expressions -[source,yaml] ------------------------------------------------------------------------------------- -"foo-bar": all # match the literal `foo-bar` -"foo-*": all # match anything beginning with "foo-" -"logstash-201?-*": all # ? matches any one character -"/.*-201[0-9]-.*/": all # use a regex to match anything containing 2010-2019 -"/foo": all # syntax error - missing final / ------------------------------------------------------------------------------------- - -TIP: Once the roles are defined, users can then be associated with any number of these roles. In -<> we'll learn more about authentication and see how users can be associated with the -configured roles. - -The privileges can also directly be set on an index expression. This notation is useful if no other security features -are configured. - -.Shorter privileges notation -[source,yaml] ---------------------------------------------------- -# Only GET read action on index named events_index -get_user: - indices: - 'events_index': 'indices:data/read/get' ---------------------------------------------------- - -include::granting-alias-privileges.asciidoc[] - -include::mapping-roles.asciidoc[] - -include::setting-up-field-and-document-level-security.asciidoc[] - -include::submitting-requests-for-other-users.asciidoc[] \ No newline at end of file diff --git a/shield/docs/public/example-deployments.asciidoc b/shield/docs/public/example-deployments.asciidoc deleted file mode 100644 index 12f01125446..00000000000 --- a/shield/docs/public/example-deployments.asciidoc +++ /dev/null @@ -1,6 +0,0 @@ -[[example-deployments]] -== Example Shield Deployments - -The examples in this section demonstrate how you might deploy Shield to secure an Elasticsearch cluster. - -include::example-deployments/e-commerce.asciidoc[] diff --git a/shield/docs/public/example-deployments/e-commerce.asciidoc b/shield/docs/public/example-deployments/e-commerce.asciidoc deleted file mode 100644 index 8469f210161..00000000000 --- a/shield/docs/public/example-deployments/e-commerce.asciidoc +++ /dev/null @@ -1,92 +0,0 @@ -[float] -[[example]] -=== E-commerce Example Using esusers - -The e-commerce store site in this example store has the following components: - -* A webshop application, which executes queries -* A nightly bulk import process, which reindexes the documents to ensure correct pricing for the following day -* A update mechanism that writes data concurrently during business hours on a per-document base -* A sales representative that needs to read sales-specific indices - -[float] -==== Defining the roles - -[source,yaml] --------------------------------------------------- -bulk: - indices: - 'products_*': write, manage, read - -updater: - indices: - 'products': index, delete, indices:admin/forcemerge - -webshop: - indices: - 'products': search, get - -monitoring: - cluster: monitor - indices: - '*': monitor - -sales_rep : - cluster : none - indices: - 'sales_*' : all - 'social_events' : data_access, monitor --------------------------------------------------- - -Let's step through each of the role definitions: - -* The `bulk` role definition has the privileges to create/delete all indices starting with `products_` as well as -indexing data into it. This set of privileges enables the user with this role to delete and repopulate a particular -index. - -* The `updater` role does not require any information about concrete indices. The only privileges required for updating -the `products` index are the `write` and `delete` privileges, as well as index optimization. - -* The `webshop` role is a read-only role that solely executes queries and GET requests. - -* The `monitoring` role extracts monitoring data for display on an internal screen of the web application. - -* The `sales_rep` role has write access on all indices starting with `sales` and read access to the `social_events` -index. - -[float] -==== Creating Users and Their Roles - -After creating the `roles.yml` file, you can use the `esusers` tool to create the needed users and the respective -user-to-role mapping. - -[source,shell] ------------------------------------------------------------ -bin/shield/esusers useradd webshop -r webshop,monitoring ------------------------------------------------------------ - -[source,shell] ------------------------------------------------------------ -bin/shield/esusers useradd bulk -r bulk ------------------------------------------------------------ - -[source,shell] ------------------------------------------------------------ -bin/shield/esusers useradd updater -r updater ------------------------------------------------------------ - -[source,shell] --------------------------------------------------------------------- -bin/shield/esusers useradd best_sales_guy_of_the_world -r sales_rep --------------------------------------------------------------------- - -[source,shell] ----------------------------------------------------------------------------- -bin/shield/esusers useradd second_best_sales_guy_of_the_world -r sales_rep ----------------------------------------------------------------------------- - -[float] -==== Modifying Your Application - -With the users and roles defined, you now need to modify your application. Each part of the application must -authenticate to Elasticsearch using the username and password you gave it in the previous steps. \ No newline at end of file diff --git a/shield/docs/public/getting-started.asciidoc b/shield/docs/public/getting-started.asciidoc deleted file mode 100644 index c1747c798d3..00000000000 --- a/shield/docs/public/getting-started.asciidoc +++ /dev/null @@ -1,56 +0,0 @@ -[[getting-started]] -== Getting Started with Shield - -This getting started guide walks you through installing Shield, setting up basic authentication, and getting started with role-based -access control. You can install Shield on nodes running Elasticsearch {version}. - -IMPORTANT: The Shield plugin must be installed on every node in the cluster. If you are installing -to a live cluster, you must stop all of the nodes, install Shield, and restart the nodes. You cannot -perform a rolling restart to install Shield. - -To install and run Shield: - -. Run `bin/plugin install` from `ES_HOME` to install the license plugin. -+ -[source,shell] ----------------------------------------------------------- -bin/plugin install license ----------------------------------------------------------- - -. Run `bin/plugin install` to install the Shield plugin. -+ -[source,shell] ----------------------------------------------------------- -bin/plugin install shield ----------------------------------------------------------- -+ -NOTE: If you are using a <> of Elasticsearch, you need to run the installation with superuser permissions. To perform an offline installation, <>. - -. Start Elasticsearch. -+ -[source,shell] ----------------------------------------------------------- -bin/elasticsearch ----------------------------------------------------------- - -. To verify that Shield is up and running, check the startup log entries. When Shield is operating -normally, the log indicates that the network transports are using Shield: -+ -[source,shell] ----------------- -[2014-10-09 13:47:38,841][INFO ][transport ] [Ezekiel Stane] Using [org.elasticsearch.shield.transport.ShieldServerTransportService] as transport service, overridden by [shield] -[2014-10-09 13:47:38,841][INFO ][transport ] [Ezekiel Stane] Using [org.elasticsearch.shield.transport.netty.ShieldNettyTransport] as transport, overridden by [shield] -[2014-10-09 13:47:38,842][INFO ][http ] [Ezekiel Stane] Using [org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport] as http transport, overridden by [shield] ----------------- - -Now you're ready to secure your cluster! Here are a few things -you might want to do to start with: - -* <> -* <> -* <> - -include::getting-started/enable-basic-auth.asciidoc[] -include::getting-started/enable-message-authentication.asciidoc[] -include::getting-started/enable-auditing.asciidoc[] -include::getting-started/moving-on.asciidoc[] diff --git a/shield/docs/public/getting-started/enable-auditing.asciidoc b/shield/docs/public/getting-started/enable-auditing.asciidoc deleted file mode 100644 index ba1a42cd799..00000000000 --- a/shield/docs/public/getting-started/enable-auditing.asciidoc +++ /dev/null @@ -1,17 +0,0 @@ -[[enable-auditing]] -=== Enable Auditing - -When you enable auditing, Shield stores a record of attempted and successful interactions with -your Elasticsearch cluster. You can use this information to keep track of who is doing what to -your cluster and identify potential security issues. - -To enable auditing, add the following setting to `elasticsearch.yml`: - -[source,yaml] ----------------------------- -shield.audit.enabled: true ----------------------------- - -By default, events are logged to a dedicated `elasticsearch-access.log` file in `ES_HOME/logs`. You can also store the events in an Elasticsearch index for easier analysis and control what events -are logged. For more information, see <>. - diff --git a/shield/docs/public/getting-started/enable-basic-auth.asciidoc b/shield/docs/public/getting-started/enable-basic-auth.asciidoc deleted file mode 100644 index 34a45e88d26..00000000000 --- a/shield/docs/public/getting-started/enable-basic-auth.asciidoc +++ /dev/null @@ -1,53 +0,0 @@ -[[enable-basic-auth]] -=== Control Access with Basic Authentication - -Shield makes it simple to password-protect your Elasticsearch cluster. Once Shield is installed, a username and password is required to communicate with the cluster. - -If you submit a request without a username and password, the request is rejected: - -[source,shell] ----------------------------------------------------------- -curl -XGET 'http://localhost:9200/' ----------------------------------------------------------- - -All you need to do to use basic authentication is set up users and assign them to one of the basic predefined roles: - -`admin` :: Can perform any cluster or index action. -`power_user` :: Can monitor the cluster and perform any index action. -`user` :: Can perform read actions on any index. - -To create a user and try out basic authentication: - -. Add a user called `es_admin` and assign the `admin` role. -+ -[source,shell] ----------------------------------------------------------- -bin/shield/esusers useradd es_admin -r admin ----------------------------------------------------------- - -. When prompted, enter a password for the new user. Passwords must be at least 6 characters long. - -. Submit a request using the newly-created user. -+ -[source,shell] ----------------------------------------------------------- -curl -u es_admin -XGET 'http://localhost:9200/' ----------------------------------------------------------- - -That's it! That's all it takes to set up the first layer of -security for your Elasticsearch cluster. However, Shield -offers much more that simple password protection. For example, -you can: - -* <> to verify that messages have not not been tampered with or corrupted in transit. -* <> to keep track of attempted and successful interactions with -your Elasticsearch cluster. - -And that's just the start. You can also: - -* <> for fine-grained access control. -* Integrate with <> or <>, or <> for authentication. -* Use <> to secure communications to and from nodes. -* Use <> to allow or deny requests from particular IP addresses or address ranges. - - diff --git a/shield/docs/public/getting-started/enable-message-authentication.asciidoc b/shield/docs/public/getting-started/enable-message-authentication.asciidoc deleted file mode 100644 index 83e407521ee..00000000000 --- a/shield/docs/public/getting-started/enable-message-authentication.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -[[enable-message-authentication]] -=== Enable Message Authentication - -Message authentication verifies that a message has not been tampered with or corrupted in transit. - -To enable message authentication: - -. Run the `syskeygen` tool from `ES_HOME` without any options: -+ -[source, shell] ----------------- -bin/shield/syskeygen ----------------- -+ -This creates a system key file in `CONFIG_DIR/shield/system_key`. - -. Copy the genererated system key to the rest of the nodes in the cluster. - -IMPORTANT: The system key is a symmetric key, so the same key must be on every node in the cluster. - -Now that you've enabled message authentication, you might also want to <> to keep track of attempted and successful interactions with your Elasticsearch cluster. diff --git a/shield/docs/public/getting-started/moving-on.asciidoc b/shield/docs/public/getting-started/moving-on.asciidoc deleted file mode 100644 index 500946f5e54..00000000000 --- a/shield/docs/public/getting-started/moving-on.asciidoc +++ /dev/null @@ -1,9 +0,0 @@ -[[moving-on]] -=== Moving On - -Now that you have Shield up and running, you might want to: - -* <> for fine-grained access control. -* Integrate with <> or <>, or <> for authentication. -* Use <> to secure communications to and from nodes. -* Use <> to allow or deny requests from particular IP addresses or address ranges. \ No newline at end of file diff --git a/shield/docs/public/granting-alias-privileges.asciidoc b/shield/docs/public/granting-alias-privileges.asciidoc deleted file mode 100644 index 797d50d47b8..00000000000 --- a/shield/docs/public/granting-alias-privileges.asciidoc +++ /dev/null @@ -1,101 +0,0 @@ -[[securing-aliases]] -=== Granting Privileges for Indices & Aliases - -Elasticsearch allows to execute operations against {ref}/indices-aliases.html[index aliases], -which are effectively virtual indices. An alias points to one or more indices, holds metadata and potentially a filter. -Shield treats aliases and indices the same. Privileges for indices actions are granted on specific indices or aliases. -In order for an indices action to be authorized by Shield, the user that executes it needs to have permissions for that -action on all the specific indices or aliases that the request relates to. - -Let's look at an example. Assuming we have an index called `2015`, an alias that points to it called `current_year`, -and a user with the following role: - -[source,yaml] --------------------------------------------------- -current_year_read: - indices: - '2015': read --------------------------------------------------- - -The user attempts to retrieve a document from `current_year`: - -[source,shell] -------------------------------------------------------------------------------- -curl -XGET 'localhost:9200/current_year/logs/1' -------------------------------------------------------------------------------- - -The above request gets rejected, although the user has read permissions on the concrete index that the `current_year` -alias points to. The correct permission would be as follows: - -[source,yaml] --------------------------------------------------- -current_year_read: - indices: - 'current_year': read --------------------------------------------------- - -[float] -==== Managing aliases - -Unlike creating indices, which requires `create_index` privilege, adding/removing/retrieving aliases requires -`manage_aliases` permission. Aliases can be added to an index directly as part of the index creation: - -[source,shell] -------------------------------------------------------------------------------- -curl -XPUT localhost:9200/2015 -d '{ - "aliases" : { - "current_year" : {} - } -}' -------------------------------------------------------------------------------- - -or via the dedicated aliases api if the index already exists: - -[source,shell] -------------------------------------------------------------------------------- -curl -XPOST 'http://localhost:9200/_aliases' -d ' -{ - "actions" : [ - { "add" : { "index" : "2015", "alias" : "current_year" } } - ] -}' -------------------------------------------------------------------------------- - -The above requests both require `manage_aliases` privilege on the alias name as well as the targeted index, as follows: - -[source,yaml] --------------------------------------------------- -admin: - indices: - '20*,current_year': create_index,manage_aliases --------------------------------------------------- - -Note also that the `manage` privilege includes both `create_index` and `manage_aliases` in addition to all of the other -management related privileges: - -[source,yaml] --------------------------------------------------- -admin: - indices: - '20*,current_year': manage --------------------------------------------------- - -The index aliases api allows also to delete aliases from existing indices, as follows. The privileges required for such -a request are the same as above. Both index and alias need the `manage_aliases` permission. - -[source,shell] -------------------------------------------------------------------------------- -curl -XPOST 'http://localhost:9200/_aliases' -d ' -{ - "actions" : [ - { "delete" : { "index" : "2015", "alias" : "current_year" } } - ] -}' -------------------------------------------------------------------------------- - -[float] -==== Filtered aliases - -Aliases can hold a filter, which allows to select a subset of documents that can be accessed out of all the documents that -the physical index contains. These filters are not always applied and should not be used -in place of <>. diff --git a/shield/docs/public/how-shield-works.asciidoc b/shield/docs/public/how-shield-works.asciidoc deleted file mode 100644 index f443909e686..00000000000 --- a/shield/docs/public/how-shield-works.asciidoc +++ /dev/null @@ -1,84 +0,0 @@ -[[how-shield-works]] -== How Shield Works - -Shield installs as a plugin into Elasticsearch. Once installed, the plugin intercepts inbound API calls in order to -enforce authentication and authorization. The plugin can also provide encryption using Secure Sockets Layer/Transport -Layer Security (SSL/TLS) for the network traffic to and from the Elasticsearch node. The plugin also uses the API -interception layer that enables authentication and authorization to provide audit logging capability. - - -[float] -=== User Authentication - -Shield defines a known set of users in order to authenticate users that make requests. These sets of users are defined -with an abstraction called a _realm_. A realm is a user database configured for the use of the Shield plugin. The -supported realms are _esusers_ and _LDAP_. - -In the _esusers_ realm, users exist exclusively within the Elasticsearch cluster. With the _esusers_ realm, the -administrator manages users with <>, and all the user operations occur within -the Elasticsearch cluster. Users authenticate with a username and password pair. - -In the _LDAP_ realm, the administrator manages users with the tools provided by the LDAP vendor. Elasticsearch -authenticates users by accessing the configured LDAP server. Users authenticate with a username and password pair. Shield -also enables mapping LDAP groups to roles in Shield (more on roles below). - -Your application can be a user in a Shield realm. Elasticsearch Clients authenticate to the cluster by providing a -username and password pair (a.k.a _Authentication Token_) with each request. To learn more on how different clients -can authenticate, see <>. - - -[float] -=== Authorization - -Shield's data model for action authorization consists of these elements: - -* _Secured Resource_, a resource against which security permissions are defined, including the cluster, an index/alias, - or a set of indices/aliases in the cluster - -* _Privilege_, one or more actions that a user may execute against a secured resource. This includes named groups of - actions (e.g. _read_), or a set specific actions (e.g. indices:/data/read/percolate) - -* _Permissions_, one or more privileges against a secured resource (e.g. _read on the "products" index_) - -* _Role_, named sets of permissions - -* _Users_, entities which may be assigned zero or more roles, authorizing them to perform the actions on the secure - resources described in the union of their roles - -A secure Elasticsearch cluster manages the privileges of users through <>. A role has a unique name and identifies -a set of permissions that translate to privileges on resources. A user can have an arbitrary number of roles. There are -two types of permissions: _cluster_ and _index_. The total set of permissions that a user has is defined by union of the -permissions in all its roles. - -Depending on the realm used, Shield provides the appropriate means to assign roles to users. - - -[float] -=== Node Authentication and Channel Encryption - -Nodes communicate to other nodes over port 9300. With Shield, you can use SSL/TLS to wrap this communication. When -SSL/TLS is enabled, the nodes validate each other's certificates, establishing trust between the nodes. This validation -prevents unauthenticated nodes from joining the cluster. Communications between nodes in the cluster are also encrypted -when SSL/TLS is in use. - -Users are responsible for generating and installing their own certificates. - -You can choose a variety of ciphers for encryption. See the <> -section for details. - -For more information on how to secure nodes see <>. - - -[float] -=== IP Filtering -Shield provides IP-based access control for Elasticsearch nodes. This access control allows you to restrict which -other servers, via their IP address, can connect to your Elasticsearch nodes and make requests. For example, you can -configure Shield to allow access to the cluster only from your application servers. The configuration provides for -whitelisting and blacklisting of subnets, specific IP addresses, and DNS domains. To read more about IP filtering see -<>. - - -[float] -=== Auditing -The <> in a secure Elasticsearch cluster logs particular events and activity on that -cluster. The events logged include authentication attempts, including granted and denied access. \ No newline at end of file diff --git a/shield/docs/public/index.asciidoc b/shield/docs/public/index.asciidoc deleted file mode 100644 index 59bb5f9652e..00000000000 --- a/shield/docs/public/index.asciidoc +++ /dev/null @@ -1,45 +0,0 @@ -[[shield]] - -= Shield Reference - -:ref: http://www.elastic.co/guide/en/elasticsearch/reference/2.0 -:ref-17: http://www.elastic.co/guide/en/elasticsearch/reference/1.7 - -:version: 2.1.0 - - -include::introduction.asciidoc[] - -include::getting-started.asciidoc[] - -include::how-shield-works.asciidoc[] - -include::installing-shield.asciidoc[] - -include::setting-up-authentication.asciidoc[] - -include::managing-users.asciidoc[] - -include::configuring-rbac.asciidoc[] - -include::configuring-auditing.asciidoc[] - -include::securing-communications.asciidoc[] - -include::configuring-clients-integrations.asciidoc[] - -include::managing-shield-licenses.asciidoc[] - -include::example-deployments.asciidoc[] - -include::reference.asciidoc[] - -include::limitations.asciidoc[] - -include::troubleshooting.asciidoc[] - -include::setting-up-certificate-authority.asciidoc[] - -include::release-notes.asciidoc[] - - diff --git a/shield/docs/public/installing-shield.asciidoc b/shield/docs/public/installing-shield.asciidoc deleted file mode 100644 index f00525bf26a..00000000000 --- a/shield/docs/public/installing-shield.asciidoc +++ /dev/null @@ -1,177 +0,0 @@ -[[installing-shield]] -== Installing Shield - -The <> steps through a basic Shield installation. This section provides some additional information about the installation prerequisites, deployment options, and the installation process for RPM/DEB package installations. - -IMPORTANT: The Shield plugin must be installed on every node in the cluster and every -node must be restarted after installation. Plan for a complete cluster restart before beginning the installation process. - -[float] -=== Shield Installation Prerequisites - -To install Shield {version}, you need: - -* Java 7 or later -* Elasticsearch {version} -* Elasticsearch License plugin {version} - -For information about installing the latest Oracle JDK, see http://www.oracle.com/technetwork/java/javase/downloads/index-jsp-138363.html[Java SE Downloads]. For information about installing Elasticsearch, see http://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html[Installation] in the Elasticsearch Reference. - -[float] -[[deb-rpm-install]] -=== Installing Shield on a DEB/RPM Package Installation - -If you use the DEB/RPM packages to install Elasticsearch, by default Elasticsearch is installed in -`/usr/share/elasticsearch` and the configuration files are stored in `/etc/elasticsearch`. (For the -complete list of default paths, see {ref}/setup-dir-layout.html#_deb_and_rpm[Directory Layout] in -the Elasticsearch Reference.) - -To install the Shield and License plugins on a DEB/RPM package installation, you need to run -`bin/plugin install` from the `/usr/share/elasticsearch` directory with superuser permissions. For example: - -[source,shell] ----------------------------------------------------------- -cd /usr/share/elasticsearch -sudo bin/plugin install license -sudo bin/plugin install shield ----------------------------------------------------------- - -[float] -[[offline-install]] -=== Installing Shield on Offline Machines - -Elasticsearch’s `bin/plugin` script requires direct Internet access to download and install the -License and Shield plugins. If your server doesn’t have Internet access, you can manually -download and install the plugins. - -To install Shield on a machine that doesn't have Internet access: - -. Manually download the appropriate License and Shield binaries: -** https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/license/{version}/license-{version}.zip[ -https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/license/{version}/license-{version}.zip] -** https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/shield/{version}/shield-{version}.zip[ -https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/shield/{version}/shield-{version}.zip] - -. Transfer the zip files to the offline machine. - -. Run `bin/plugin` with the `-u` option to install the plugins using the zip files. For example: -+ -[source,shell] ----------------------------------------------------------- -bin/plugin install file:///path/to/file/license-2.1.0.zip <1> -bin/plugin install file:///path/to/file/shield-2.1.0.zip ----------------------------------------------------------- -<1> Note that you must specify an absolute path to the zip file after the `file://` protocol. - -[float] -[[tribe-node]] -=== Installing Shield on Tribe Nodes - -Shield supports the {ref}/modules-tribe.html[Tribe Node], which acts as a federated client across multiple clusters. When using Tribe Nodes with Shield, you must have the same Shield configuration (users, roles, user-role mappings, SSL/TLS CA) on each cluster, and on the Tribe Node itself, where security checking is primarily done. This, of course, also means -that all clusters must be running Shield. - -To use a Tribe Node with Shield: - -. Configure the same privileges on all connected clusters. The Tribe Node has its own configuration and privileges, which need to grant access to actions and indices on all of the -connected clusters. Also, each cluster needs to grant access to indices belonging to other connected clusters as well. -+ -Let's look at an example: assuming we have two clusters, `cluster1` and `cluster2`, each one holding an index, `index1` -and `index2`. A search request that targets multiple clusters, as follows -+ -[source,shell] ------------------------------------------------------------ -curl -XGET tribe_node:9200/index1,index2/_search -u tribe_user:tribe_user ------------------------------------------------------------ -+ -requires `search` privileges for both `index1` and `index2` on the Tribe Node: -+ -[source,yaml] ------------------------------------------------------------ -tribe_user: - indices: - 'index*': search ------------------------------------------------------------ -+ -Also, the same privileges need to be granted on the connected clusters, meaning that `cluster1` has to grant access to -`index2` even though `index2` only exists on `cluster2`; the same requirement applies for `index1` on `cluster2`. This -applies to any indices action. As for cluster state read operations (e.g. cluster state api, get mapping api etc.), -they always get executed locally on the Tribe Node, to make sure that the merged cluster state gets returned; their -privileges are then required on the Tribe Node only. - -. Use the same system key on all clusters. For <> to properly work across multiple clusters, the Tribe Node -and all of the connected clusters need to share the same system key. - -. Enable encryption globally. Encrypted communication via SSL/TLS can only be enabled globally, meaning that either all of the connected clusters and the Tribe Node have SSL enabled, or none of them have. - -. Use the same certification authority on all clusters. When using encrypted communication, for simplicity, we recommend all of the connected clusters and the Tribe Node use -the same certification authority to generate their certificates. - -[float] -==== Tribe Node Example - -Let's see a complete example on how to use the Tribe Node with shield and the configuration required. First of all the -Shield and License plugins need to be installed and enabled on all clusters and on the Tribe Node. - -The system key needs to be generated on one node, as described in <>, -and then copied over to all of the other nodes in each cluster and the Tribe Node itself. - -Each cluster can have its own users with `admin` privileges that don't need to be present in the Tribe Node too. In fact, -administration tasks (e.g. create index) cannot be performed through the Tribe Node but need to be sent directly to the -corresponding cluster. The users that need to be created on Tribe Node are those that allow to get back data merged from -the different clusters through the Tribe Node itself. Let's for instance create as follows a `tribe_user` user, with -role `user`, that has `read` privileges on any index. - -[source,shell] ------------------------------------------------------------ -./bin/shield/esusers useradd tribe_user -p tribe_user -r user ------------------------------------------------------------ - -The above command needs to be executed on each cluster, since the same user needs to be present on the Tribe Node as well -as on every connected cluster. - -The following is the configuration required on the Tribe Node, that needs to be added to `elasticsearch.yml`. -Elasticsearch allows to list specific settings per cluster. We disable multicast discovery and configure the proper unicast discovery hosts for each cluster, -as well as their cluster names: - -[source,yaml] ------------------------------------------------------------ -tribe: - t1: - cluster.name: tribe1 - discovery.zen.ping.multicast.enabled: false - discovery.zen.ping.unicast.hosts: ["tribe1:9300"] - t2: - cluster.name: tribe2 - discovery.zen.ping.multicast.enabled: false - discovery.zen.ping.unicast.hosts: ["tribe2:9300"] ------------------------------------------------------------ - -The Tribe Node can then be started and once initialized it will be ready to accept requests like the following search, -which will return documents coming from the different connected clusters: - -[source,shell] ------------------------------------------------------------ -curl -XGET localhost:9200/_search -u tribe_user:tribe_user ------------------------------------------------------------ - -As for encrypted communication, the required settings are the same as described in <>, -but need to be specified per tribe as we did for discovery settings above. - -[float] -[[uninstalling-shield]] -=== Uninstalling Shield - -IMPORTANT: Uninstalling Shield requires a {ref}/restart-upgrade.html[full cluster restart]. - -To uninstall Shield: - -. Shut down Elasticsearch. - -. Remove the Shield plugin from Elasticsearch: -+ -[source,shell] ----------------------------------------------------------- -bin/plugin remove shield ----------------------------------------------------------- - -. Restart Elasticsearch. \ No newline at end of file diff --git a/shield/docs/public/introduction.asciidoc b/shield/docs/public/introduction.asciidoc deleted file mode 100644 index 8bfcfd74589..00000000000 --- a/shield/docs/public/introduction.asciidoc +++ /dev/null @@ -1,53 +0,0 @@ -[[introduction]] -== Introduction - -_Shield_ is a plugin for Elasticsearch that enables you to easily secure a cluster. With Shield, -you can password-protect your data as well as implement more advanced security measures such as encrypting communications, role-based access control, IP filtering, and auditing. This guide -describes how to install Shield, configure the security features you need, and interact with your secured cluster. - -[float] -=== Security for Elasticsearch - -Shield protects Elasticsearch clusters by: - -* <> with password protection, role-based access control, and IP filtering. -* <> with message authentication and SSL/TLS encryption. -* <> so you know who's doing what to your -data. - -[float] -[[preventing-unauthorized-access]] -==== Preventing Unauthorized Access - -To prevent unauthorized access to your Elasticsearch cluster, you must have a way to _authenticate_ users. This simply means that you need a way to validate that a user is who they claim to be. For example, you have to make sure only the person named _Kelsey Andorra_ can sign in as the user `kandorra`. Shield provides a standalone authentication mechanism that enables you to quickly <>. If you're already using <>, <>, or <>, Shield easily integrates with those systems to perform user authentication. - -In many cases, simply authenticating users isn't enough. You also need a way to control what data users have access to and what tasks they can perform. Shield enables you to _authorize_ users by -assigning access _privileges_ to a _role_, and assigning those roles to users. For example, this <> mechanism enables you to specify that the user `kandorra`can only perform read operations on the `events` index and can't do anything at all with other indices. - -Shield also supports <>. You can whitelist and blacklist specific IP addresses or subnets to control network-level access to a server. - -[float] -[[preserving-data-integrity]] -==== Preserving Data Integrity - -A critical part of security is keeping confidential data confidential. Elasticsearch has built-in protections against accidental data loss and corruption. However, there's nothing to stop deliberate tampering or data interception. Shield preserves the integrity of your data by <> to and from nodes and <> to verify that they have not been tampered with or corrupted in transit. For even greater protection, you can increase the <> and <>. - - -[float] -[[maintaining-audit-trail]] -==== Maintaining an Audit Trail - -Keeping a system secure takes vigilance. By using Shield to maintain an audit trail, you can easily see who is accessing your cluster and what they're doing. By analyzing access patterns and failed attempts to access your cluster, you can gain insights into attempted attacks and data breaches. Keeping an auditable log of the activity in your cluster can also help diagnose operational issues. - -[float] -=== Where to Go Next - -* <> steps through how to install and start using Shield for basic authentication. -* <> provides more information about how Shield supports user authentication, authorization, and encryption. -* <> shows you how to -interact with an Elasticsearch cluster protected by Shield. -* <> provides detailed information about the access privileges you can grant to users, the settings you can configure for Shield in `elasticsearch.yml`, and the files where Shield configuration information is stored. - -=== Have Comments, Questions, or Feedback? - -Head over to our https://discuss.elastic.co/c/shield[Shield Discussion Forum] to share your experience, questions, and suggestions. \ No newline at end of file diff --git a/shield/docs/public/limitations.asciidoc b/shield/docs/public/limitations.asciidoc deleted file mode 100644 index a07335bc21e..00000000000 --- a/shield/docs/public/limitations.asciidoc +++ /dev/null @@ -1,57 +0,0 @@ -[[limitations]] -== Limitations - -[float] -=== Plugins - -Elasticsearch's plugin infrastructure is extremely flexible in terms of what can be extended. While it opens up Elasticsearch -to a wide variety of (often custom) additional functionality, when it comes to security, this high extensibility level -comes at a cost. We have no control over the third-party plugins' code (open source or not) and therefore we cannot -guarantee their compliance with Shield. For this reason, third-party plugins are not officially supported on clusters -with the Shield security plugin installed. - -[float] -=== Changes in Index Wildcard Behavior - -Elasticsearch clusters with the Shield security plugin installed apply the `/_all` wildcard, and all other wildcards, -to the indices that the current user has privileges for, not the set of all indices on the cluster. There are two -notable results of this behavior: - -* Elasticsearch clusters with the Shield security plugin installed do not honor the `ignore_unavailable` option. - This behavior means that requests involving indices that the current user lacks authorization for throw an - `AuthorizationException` error, regardless of the option's setting. - -* The `allow_no_indices` option is ignored, resulting in the following behavior: when the final set of indices after - wildcard expansion and replacement is empty, the request throws a `IndexMissingException` error. - -As a general principle, core Elasticsearch will return empty results in scenarios where wildcard expansion returns no -indices, while Elasticsearch with Shield returns exceptions. Note that this behavior means that operations with -multiple items will fail the entire set of operations if any one operation throws an exception due to wildcard -expansion resulting in an empty set of authorized indices. - -[float] -=== Field and Document Level Security Limitations - -Bulk updates do not work when document and field level security is enabled. If you are not using document and field level -security, bulk updates can be enabled by setting `shield.dls_fls.enabled` to `false`. - -When document level security is enabled for an index: - -* The get, multi get, termsvector and multi termsvector APIs aren't executed in real time. The realtime option for these APIs is forcefully set to false. -* Document level security isn't applied for APIs that aren't document based. An example is the field stats API. -* Document level security doesn't affect global index statistics that relevancy scoring uses. So this means that scores are computed without taking the role query into account. - Note that documents not matching with the role query are never returned. -* The `has_child` and `has_parent` queries aren't supported as role query in the `roles.yml` file. - The `has_child` and `has_parent` queries can be used in the search API with document level security enabled. - -[float] -=== Document Expiration (_ttl) - -Document expiration handled using the built-in {ref}/mapping-ttl-field.html#mapping-ttl-field[`_ttl` (time to live) mechanism] -does not work with Shield. The document deletions will fail and the documents continue to live past their expiration. - -[float] -=== LDAP Realm - -The <> does not currently support the discovery of nested LDAP Groups. For example, if a user is a member -of GroupA and GroupA is a member of GroupB, only GroupA will be discovered. However, the <> _does_ support transitive group membership. diff --git a/shield/docs/public/managing-shield-licenses.asciidoc b/shield/docs/public/managing-shield-licenses.asciidoc deleted file mode 100644 index 052e03427ec..00000000000 --- a/shield/docs/public/managing-shield-licenses.asciidoc +++ /dev/null @@ -1,114 +0,0 @@ -[[license-management]] -== Managing Your License - -When you initially install Shield, a 30 day trial license is installed that allows access to all features. At the end of the trial period, you can https://www.elastic.co/subscriptions/[purchase a subscription] to keep using the full functionality of Shield along with Marvel and Watcher. - -IMPORTANT: When your license expires, Shield operates in a degraded mode where access to the Elasticsearch cluster health, cluster stats, and index stats APIs is blocked. Shield keeps on protecting your cluster, but you won't be able to monitor its operation until you update your license. For more information, see <>. - -[float] -[[installing-license]] -=== Updating Your License - -You can update your license at runtime without shutting down your nodes. License updates take -effect immediately. The license is provided as a _JSON_ file that you install with the `license` -API. You need cluster admin privileges to install the license. - -To update your license: - -. Send a request to the `license` API and specify the file that contains your new license: -+ -[source,shell] ------------------------------------------------------------------------ -curl -XPUT -u admin 'http://:/_license' -d @license.json ------------------------------------------------------------------------ -+ -Where: -+ -* `` is the hostname of the Elasticsearch node (`localhost` if executing locally) -* `` is the http port (defaults to `9200`) -* `license.json` is the license JSON file - -. If the license you are installing does not support all of the features available with your -previous license, you will be notified in the response. To complete the license installation, -you must resubmit the license update request and set the `acknowledge` parameter to `true` to -indicate that you are aware of the changes. -+ -[source,shell] ------------------------------------------------------------------------ -curl -XPUT -u admin 'http://:/_license?acknowledge=true' -d @license.json ------------------------------------------------------------------------ - -[float] -[[listing-licenses]] -=== Viewing the Installed License - -You can also use the `license` API to retrieve the currently installed license: - -[source,shell] ------------------------------------------------------ -curl -XGET -u admin:password 'http://:/_license' -{ - "license" : { - "status" : "active", - "uid" : "0a98411f-73f4-4c67-954c-724874ed5488", - "type" : "trial", - "issue_date" : "2015-10-13T18:18:20.709Z", - "issue_date_in_millis" : 1444760300709, - "expiry_date" : "2015-11-12T18:18:20.709Z", - "expiry_date_in_millis" : 1447352300709, - "max_nodes" : 1000, - "issued_to" : "elasticsearch", - "issuer" : "elasticsearch" - } -} --------------------------------------------- - -NOTE: You need cluster admin privileges to retrieve the license. - -[float] -[[license-expiration]] -=== License Expiration - -License expiration should never be a surprise. If you're using Marvel, a license expiration -warning is displayed prominently if your license expires within 30 days. Warnings are -also displayed on startup and written to the Elasticsearch log starting 30 days from the expiration date. These error messages tell you when the license expires and what features will be disabled if -you fail to update it: - -[source,sh] --------------------------------------------- -# License will expire on [Thursday, November 12, 2015]. If you have a new license, please update it. -# Otherwise, please reach out to your support contact. -# -# Commercial plugins operate with reduced functionality on license expiration: -# - marvel -# - The agent will stop collecting cluster and indices metrics -# - shield -# - Cluster health, cluster stats and indices stats operations are blocked -# - All data operations (read and write) continue to work --------------------------------------------- - -Once the license expires, calls to the cluster health, cluster stats, and index stats APIs -fail with a `ElasticsearchSecurityException` and return a 401 HTTP status code: - -[source,sh] --------------------------------------------- -{ - "error": { - "root_cause": [{ - "type": "security_exception", - "reason": "current license is non-compliant for [shield]", - "license.expired.feature": "shield" - }], - "type": "security_exception", - "reason": "current license is non-compliant for [shield]", - "license.expired.feature": "shield" - }, - "status": 401 -} --------------------------------------------- - -This enables automatic monitoring systems to easily detect the license failure without immediately impacting other users. - -IMPORTANT: You should update your license as soon as possible. You're essentially flying blind when running with an expired license. Access to the cluster health and stats APIs is critical -for monitoring and managing an Elasticsearch cluster. - diff --git a/shield/docs/public/managing-users.asciidoc b/shield/docs/public/managing-users.asciidoc deleted file mode 100644 index 335cdbb9462..00000000000 --- a/shield/docs/public/managing-users.asciidoc +++ /dev/null @@ -1,257 +0,0 @@ -[[managing-users]] -== Managing Users in an esusers Realm - -The `esusers` command line tool is located in `ES_HOME/bin/shield` and enables several -administrative tasks for managing users: - -* <> -* <> -* <> -* <> -* <> - -[float] -[[esusers-add]] -=== Adding Users - -The `esusers useradd` command adds a user to your cluster. - -NOTE: To ensure that Elasticsearch can read the user and role information at startup, run `esusers useradd` as the - same user you use to run Elasticsearch. Running the command as root or some other user will update the permissions - for the `users` and `users_roles` files and prevent Elasticsearch from accessing them. - -[source,shell] ----------------------------------------- -esusers useradd ----------------------------------------- - -A username must be at least 1 character and no longer than 30 characters. The first character must be a letter -(`a-z` or `A-Z`) or an underscore (`_`). Subsequent characters can be letters, underscores (`_`), digits (`0-9`), or any -of the following symbols `@`, `-`, `.` or `$` - -You can specify the user's password at the command line with the `-p` option. When this option is absent, the -`esusers` command prompts you for the password. Omit the `-p` option to keep plaintext passwords out of the terminal -session's command history. - -[source,shell] ----------------------------------------------------- -esusers useradd -p ----------------------------------------------------- - -Passwords must be at least 6 characters long. - -You can define a user's roles with the `-r` parameter. This parameter accepts a comma-separated list of role names to -associate with the user. - -[source,shell] -------------------------------------------------------------------- -esusers useradd -r -------------------------------------------------------------------- - -The following example adds a new user named `jacknich` to the _esusers_ realm. The password for this user is -`theshining`, and this user is associated with the `logstash` and `marvel` roles. - -[source,shell] ---------------------------------------------------------- -esusers useradd jacknich -p theshining -r logstash,marvel ---------------------------------------------------------- - -For valid role names please see <>. - -[float] -[[esusers-list]] -=== Listing Users -The `esusers list` command lists the users registered in the _esusers_ realm, as in the following example: - -[source, shell] ----------------------------------- -esusers list -rdeniro : admin -alpacino : power_user -jacknich : marvel,logstash ----------------------------------- - -Users are in the left-hand column and their corresponding roles are listed in the right-hand column. - -The `esusers list ` command lists a specific user. Use this command to verify that a user has been -successfully added to the cluster. - -[source,shell] ------------------------------------ -esusers list jacknich -jacknich : marvel,logstash ------------------------------------ - -[float] -[[esusers-pass]] -=== Managing User Passwords - -The `esusers passwd` command enables you to reset a user's password. You can specify the new password directly with the -`-p` option. When `-p` option is omitted, the tool will prompt you to enter and confirm a password in interactive mode. - -[source,shell] --------------------------------------------------- -esusers passwd --------------------------------------------------- - -[source,shell] --------------------------------------------------- -esusers passwd -p --------------------------------------------------- - -[float] -[[esusers-roles]] -=== Assigning Users to Roles - -The `esusers roles` command manages the roles associated to a particular user. The `-a` option adds a comma-separated -list of roles to a user. The `-r` option removes a comma-separated list of roles from a user. You can combine adding and -removing roles within the same command to change a user's roles. - -[source,shell] ------------------------------------------------------------------------------------------------------------- -esusers roles -a -r ------------------------------------------------------------------------------------------------------------- - -The following command removes the `logstash` and `marvel` roles from user `jacknich`, as well as adding the `user` role: - -[source,shell] ---------------------------------------------------------------- -esusers roles jacknich -r logstash,marvel -a user ---------------------------------------------------------------- - -Listing the user displays the new role assignment: - -[source,shell] ---------------------------------- -esusers list jacknich -jacknich : user ---------------------------------- - -[float] -[[esusers-del]] -=== Deleting Users - -The `esusers userdel` command deletes a user. - -[source,shell] --------------------------------------------------- -userdel --------------------------------------------------- - -[float] -=== About `esusers` - -The `esusers` tool manipulates two files, `users` and `users_roles`, in `CONFIG_DIR/shield/`. These two files store all user data for the _esusers_ realm and are read by Shield -on startup. - -By default, Shield checks these files for changes every 5 seconds. You can change this default behavior by changing the -value of the `watcher.interval.high` setting in the `elasticsearch.yml` file. - -[IMPORTANT] -============================== - -These files are managed locally by the node and are **not** managed -globally by the cluster. This means that with a typical multi-node cluster, -the exact same changes need to be applied on each and every node in the -cluster. - -A safer approach would be to apply the change on one of the nodes and have the -`users` and `users_roles` files distributed/copied to all other nodes in the -cluster (either manually or using a configuration management system such as -Puppet or Chef). - -============================== - -While it is possible to modify these files directly using any standard text -editor, we strongly recommend using the `esusers` command-line tool to apply -the required changes. - -[float] -[[users-file]] -==== The `users` File -The `users` file stores all the users and their passwords. Each line in the `users` file represents a single user entry -consisting of the username and **hashed** password. - -[source,bash] ----------------------------------------------------------------------- -rdeniro:$2a$10$BBJ/ILiyJ1eBTYoRKxkqbuDEdYECplvxnqQ47uiowE7yGqvCEgj9W -alpacino:$2a$10$cNwHnElYiMYZ/T3K4PvzGeJ1KbpXZp2PfoQD.gfaVdImnHOwIuBKS -jacknich:$2a$10$GYUNWyABV/Ols/.bcwxuBuuaQzV6WIauW6RdboojxcixBq3LtI3ni ----------------------------------------------------------------------- - -NOTE: The `esusers` command-line tool uses `bcrypt` to hash the password by default. - -[float] -[[users_defining-roles]] -==== The `users_roles` File - -The `users_roles` file stores the roles associated with the users, as in the following example: - -[source,shell] --------------------------------------------------- -admin:rdeniro -power_user:alpacino,jacknich -user:jacknich --------------------------------------------------- - -Each row maps a role to a comma-separated list of all the users that are associated with that role. - -[float] -[[user-cache]] -==== User Cache -The user credentials are not stored on disk in clear text. The esusers creates a `bcrypt` hashes of the passwords and -stores those. `bcrypt` is considered to be highly secured hash and by default it uses 10 rounds to generate the salts -it hashes with. While highly secured, it is also relatively slow. For this reason, Shield also introduce an in-memory -cache over the `esusers` store. This cache can use a different hashing algorithm for storing the passwords in memeory. -The default hashing algorithm that is used is `ssha256` - a salted SHA-256 algorithm. - -We've seen in the table <> that the cache characteristics can be configured. The following table -describes the different hash algorithm that can be set: - -[[esusers-cache-hash-algo]] - -.Cache hash algorithms -|======================= -| Algorithm | Description -| `ssha256` | Uses a salted `SHA-256` algorithm (default). -| `md5` | Uses `MD5` algorithm. -| `sha1` | Uses `SHA1` algorithm. -| `bcrypt` | Uses `bcrypt` algorithm with salt generated in 10 rounds. -| `bcrypt4` | Uses `bcrypt` algorithm with salt generated in 4 rounds. -| `bcrypt5` | Uses `bcrypt` algorithm with salt generated in 5 rounds. -| `bcrypt6` | Uses `bcrypt` algorithm with salt generated in 6 rounds. -| `bcrypt7` | Uses `bcrypt` algorithm with salt generated in 7 rounds. -| `bcrypt8` | Uses `bcrypt` algorithm with salt generated in 8 rounds. -| `bcrypt9` | Uses `bcrypt` algorithm with salt generated in 9 rounds. -| `noop`,`clear_text` | Doesn't hash the credentials and keeps it in clear text in memory. CAUTION: - keeping clear text is considered insecure and can be compromised at the OS - level (e.g. memory dumps and `ptrace`). -|======================= - -[float] -[[cache-eviction-api]] -==== Cache Eviction API -Shield exposes an API to force cached user eviction. The following example, evicts all users from the `esusers` -realm: - -[source, java] ------------------------------------------------------------- -$ curl -XPOST 'http://localhost:9200/_shield/realm/esusers/_cache/clear' ------------------------------------------------------------- - -NOTE: if no realm is defined, the default realm name, `default_esusers` can be used to clear the cache -of the default esusers realm. - -It is also possible to evict specific users: - -[source, java] ------------------------------------------------------------- -$ curl -XPOST 'http://localhost:9200/_shield/realm/esusers/_cache/clear?usernames=rdeniro,alpacino' ------------------------------------------------------------- - -Multiple realms can also be specified using comma-delimited list: - -[source, java] ------------------------------------------------------------- -$ curl -XPOST 'http://localhost:9200/_shield/realm/esusers,ldap1/_cache/clear' ------------------------------------------------------------- diff --git a/shield/docs/public/mapping-roles.asciidoc b/shield/docs/public/mapping-roles.asciidoc deleted file mode 100644 index c125a48d572..00000000000 --- a/shield/docs/public/mapping-roles.asciidoc +++ /dev/null @@ -1,61 +0,0 @@ -[[mapping-roles]] -=== Mapping Users and Groups to Roles - -If you authenticate users with an `esusers` realm, you can assign roles when you -<> and use the <> command to add or remove roles. - -For other types of realms, you configure role mappings for users and groups in a YAML file -and copy it to each node in the cluster. Tools like Puppet or Chef can help with this. - -By default, role mappings are stored in `CONF_DIR/shield/users/role_mapping.yml`, where `CONF_DIR` -is `ES_HOME/config` (zip/tar installations) or `/etc/elasticsearch` (package installations). -To specify a different location, you configure the `role_mapping` settings in `elasticsearch.yml`. -The `role_mapping` settings enable you to use a different set of mappings for each realm type: - - `shield.authc.ldap.files.role_mapping` :: The location of the role mappings for LDAP realms. - `shield.authc.active_directory.files.role_mapping` :: The location of the role mappings for Active Directory realms. - `shield.authc.pki.files.role_mapping` :: The location of the role mappings for PKI realms. - -IMPORTANT: For Shield to read the mapping file, it must be stored in the Elasticsearch `CONF_DIR`. - -Within the role mapping file, Shield roles are keys and groups and users are values. -The mappings can have a many-to-many relationship. When you map roles to groups, the roles of a -user in that group are the combination of the roles assigned to that group and the roles assigned -to that user. - -[[ad-role-mapping]] -The available roles are defined in the <>. To specify users and -groups in the role mappings, you use their _Distinguished Names_ (DNs). A DN -is a string that uniquely identifies the user or group, for example -`"cn=John Doe,cn=contractors,dc=example,dc=com"`. - -NOTE: Shield only supports Active Directory security groups. You cannot map distribution groups -to roles. - -[[ldap-role-mapping]] -For example, the following snippet maps the `admins` group to the `monitoring` role and maps -`John Doe`, the `users` group, and the `admins` group to the `user` role. - -[source, yaml] ------------------------------------------------------------- -monitoring: <1> - - "cn=admins,dc=example,dc=com" <2> -user: - - "cn=John Doe,cn=contractors,dc=example,dc=com" <3> - - "cn=users,dc=example,dc=com" - - "cn=admins,dc=example,dc=com" ------------------------------------------------------------- -<1> The name of a Shield role defined in the <>. -<2> The distinguished name of an LDAP group or an Active Directory security group. -<3> The distinguished name of an LDAP or Active Directory user. added[1.1.0] - -[[pki-role-mapping]] -PKI realms only support mapping users to roles, as there is no notion of a group in PKI. For example: - -[source, yaml] ------------------------------------------------------------- -monitoring: - - "cn=Admin,ou=example,o=com" -user: - - "cn=John Doe,ou=example,o=com" ------------------------------------------------------------- diff --git a/shield/docs/public/reference.asciidoc b/shield/docs/public/reference.asciidoc deleted file mode 100644 index 65fde4e4f52..00000000000 --- a/shield/docs/public/reference.asciidoc +++ /dev/null @@ -1,430 +0,0 @@ -[[reference]] -== Reference - -[[privileges-list]] - -[float] -=== Privileges - -[[privileges-list-cluster]] - -[float] -==== Cluster - -[horizontal] -`all`:: All cluster administration operations, like snapshotting, node shutdown/restart, settings update or rerouting -`monitor`:: All cluster read-ony operations, like cluster health & state, hot threads, node info, node & cluster -stats, snapshot/restore status, pending cluster tasks -`manage_shield`:: All Shield related operations (currently only exposing an API for clearing the realm caches) - -[[privileges-list-indices]] - -[float] -==== Indices - -[horizontal] -`all`:: Any action on an index -`manage`:: All `monitor` privileges plus index administration (aliases, analyze, cache clear, close, delete, exists, -flush, mapping, open, force merge, refresh, settings, search shards, templates, validate, warmers) -`monitor`:: All actions, that are required for monitoring and read-only (recovery, segments info, index stats & status) -`data_access`:: A shortcut of all of the below privileges -`crud`:: A shortcut of `read` and `write` privileges -`read`:: Read only access to actions (count, explain, get, mget, get indexed scripts, more like this, multi -percolate/search/termvector), percolate, scroll, clear_scroll, search, suggest, tv) -`search`:: All of `suggest` and executing an arbitrary search request (including multi-search API) -`get`:: Allow to execute a GET request for a single document or multiple documents via the multi-get API -`suggest`:: Allow to execute the `_suggest` API -`index`:: Privilege to index and update documents -`create_index`:: Privilege to create an index. A create index request may contain aliases to be added to the index once -created. In that case the request requires `manage_aliases` privilege as well, on both the index and the aliases names. -`manage_aliases`:: Privilege to add and remove aliases, as well as retrieve aliases information. Note that in order -to add an alias to an existing index, the `manage_aliases` privilege is required on the existing index as well as on the -alias name -`delete`:: Privilege to delete documents (includes delete by query) -`write`:: Privilege to index, update, delete, delete by query and bulk operations on documents, in addition to delete -and put indexed scripts - -[float] -==== Run As - -The `run_as` permission enables an authenticated user to submit requests on behalf of another -user. The value can be a user name or a comma-separated list of user names. (You can also specify -users as an array of strings or a YAML sequence.) For more information, -see <>. - -[horizontal] - -[[ref-actions-list]] - -[float] -==== Action level privileges - -Although rarely needed, it is also possible to define privileges on specific actions that are available in -Elasticsearch. This only applies to publicly available indices and cluster actions. - -[[ref-actions-list-cluster]] - -[float] -===== Cluster actions privileges - -* `cluster:admin/render/template/search` -* `cluster:admin/repository/delete` -* `cluster:admin/repository/get` -* `cluster:admin/repository/put` -* `cluster:admin/repository/verify` -* `cluster:admin/reroute` -* `cluster:admin/settings/update` -* `cluster:admin/snapshot/create` -* `cluster:admin/snapshot/delete` -* `cluster:admin/snapshot/get` -* `cluster:admin/snapshot/restore` -* `cluster:admin/snapshot/status` -* `cluster:admin/plugin/license/get` -* `cluster:admin/plugin/license/delete` -* `cluster:admin/plugin/license/put` -* `cluster:admin/shield/realm/cache/clear` -* `cluster:monitor/health` -* `cluster:monitor/nodes/hot_threads` -* `cluster:monitor/nodes/info` -* `cluster:monitor/nodes/stats` -* `cluster:monitor/state` -* `cluster:monitor/stats` -* `cluster:monitor/task` -* `indices:admin/template/delete` -* `indices:admin/template/get` -* `indices:admin/template/put` - -NOTE: While indices template actions typically relate to indices, they are categorized under cluster actions to avoid - potential security leaks. For example, having one user define a template that matches another user's index. - -[[ref-actions-list-indices]] - -[float] -===== Indices actions privileges - -* `indices:admin/aliases` -* `indices:admin/aliases/exists` -* `indices:admin/aliases/get` -* `indices:admin/analyze` -* `indices:admin/cache/clear` -* `indices:admin/close` -* `indices:admin/create` -* `indices:admin/delete` -* `indices:admin/get` -* `indices:admin/exists` -* `indices:admin/flush` -* `indices:admin/forcemerge` -* `indices:admin/mapping/put` -* `indices:admin/mappings/fields/get` -* `indices:admin/mappings/get` -* `indices:admin/open` -* `indices:admin/refresh` -* `indices:admin/settings/update` -* `indices:admin/shards/search_shards` -* `indices:admin/template/delete` -* `indices:admin/template/get` -* `indices:admin/template/put` -* `indices:admin/types/exists` -* `indices:admin/upgrade` -* `indices:admin/validate/query` -* `indices:admin/warmers/delete` -* `indices:admin/warmers/get` -* `indices:admin/warmers/put` -* `indices:monitor/recovery` -* `indices:monitor/segments` -* `indices:monitor/settings/get` -* `indices:monitor/shard_stores` -* `indices:monitor/stats` -* `indices:monitor/upgrade` -* `indices:data/read/explain` -* `indices:data/read/field_stats` -* `indices:data/read/get` -* `indices:data/read/mget` -* `indices:data/read/mpercolate` -* `indices:data/read/msearch` -* `indices:data/read/mtv` -* `indices:data/read/percolate` -* `indices:data/read/script/get` -* `indices:data/read/scroll` -* `indices:data/read/scroll/clear` -* `indices:data/read/search` -* `indices:data/read/suggest` -* `indices:data/read/tv` -* `indices:data/write/bulk` -* `indices:data/write/delete` -* `indices:data/write/index` -* `indices:data/write/script/delete` -* `indices:data/write/script/put` -* `indices:data/write/update` - -[[ref-shield-settings]] - -[float] -=== Shield Settings - -The parameters listed in this section are configured in the `config/elasticsearch.yml` configuration file. - -[[message-auth-settings]] - -.Shield Message Authentication Settings -[options="header"] -|====== -| Name | Default | Description -| `shield.system_key.file` |`CONFIG_DIR/shield/system_key` | Sets the <> of the `system_key` file. For more information, see <>. -|====== - -[[ref-anonymous-access]] - -.Shield Anonymous Access Settings added[1.1.0] -[options="header"] -|====== -| Name | Default | Description -| `shield.authc.anonymous.username` | `_es_anonymous_user` | The username/principal of the anonymous user (this setting is optional) -| `shield.authc.anonymous.roles` | - | The roles that will be associated with the anonymous user. This setting must be set to enable anonymous access. -| `shield.authc.anonymous.authz_exception` | `true` | When `true`, a HTTP 403 response will be returned when the anonymous user does not have the appropriate permissions for the requested action. The user will not be prompted to provide credentials to access the requested resource. When set to `false`, a HTTP 401 will be returned allowing for credentials to be provided for a user with the appropriate permissions. -|====== - -.Shield Document and Field Level Security Settings -[options="header"] -|====== -| Name | Default | Description -| `shield.dls_fls.enabled` | `false` | Set to `true` to enable document and field level security. -You cannot submit `_bulk` update requests when document and field level security is enabled. -|====== - - -[[ref-realm-settings]] - -[float] -==== Realm Settings - -All realms are configured under the `shield.authc.realms` settings, keyed by their names as follows: - -[source,yaml] ----------------------------------------- -shield.authc.realms: - - realm1: - type: esusers - order: 0 - ... - - realm2: - type: ldap - order: 1 - ... - - realm3: - type: active_directory - order: 2 - ... - ... ----------------------------------------- - -.Common Settings to All Realms -[options="header"] -|====== -| Name | Required | Default | Description -| `type` | yes | - | The type of the reamlm (currently `esusers`, `ldap` or `active_directory`) -| `order` | no | Integer.MAX_VALUE | The priority of the realm within the realm chain -| `enabled` | no | true | Enable/disable the realm -|====== - -[[ref-esusers-settings]] - -._esusers_ Realm -[options="header"] -|====== -| Name | Required | Default | Description -| `files.users` | no | `CONFIG_DIR/shield/users` | The <> of the <> file. -| `files.users_roles` | no | `CONFIG_DIR/shield/users_roles`| The <> of the <> file. -| `cache.ttl` | no | `20m` | The time-to-live for cached user entries--user credentials are cached for this configured period of time. Defaults to `20m`. Specify values using the standard Elasticsearch {ref}/common-options.html#time-units[time units]. -| `cache.max_users` | no | 100000 | The maximum number of user entries that can live in the cache at a given time. Defaults to 100,000. -| `cache.hash_algo` | no | `ssha256` | (Expert Setting) The hashing algorithm that is used for the in-memory cached user credentials. See the <> table for all possible values. -|====== - -[[ref-ldap-settings]] - -.Shield LDAP Settings -[options="header"] -|====== -| Name | Required | Default | Description -| `url` | yes | - | An LDAP URL in the format `ldap[s]://:`. -| `bind_dn` | no | Empty | The DN of the user that will be used to bind to the LDAP and perform searches. If this is not specified, an anonymous bind will be attempted. -| `bind_password` | no | Empty | The password for the user that will be used to bind to the LDAP. -| `user_dn_templates` | yes * | - | The DN template that replaces the user name with the string `{0}`. This element is multivalued, allowing for multiple user contexts. -| `user_group_attribute` | no | `memberOf` | Specifies the attribute to examine on the user for group membership. The default is `memberOf`. This setting will be ignored if any `group_search` settings are specified. -| `user_search.base_dn` | yes * | - | Specifies a container DN to search for users. -| `user_search.scope` | no | `sub_tree` | The scope of the user search. Valid values are `sub_tree`, `one_level` or `base`. `one_level` only searches objects directly contained within the `base_dn`. `sub_tree` searches all objects contained under `base_dn`. `base` specifies that the `base_dn` is the user object, and that it is the only user considered. -| `user_search.attribute` | no | `uid` | The attribute to match with the username presented to Shield. -| `user_search.pool.size` | no | `20` | The maximum number of connections to the LDAP server to allow in the connection pool. -| `user_search.pool.initial_size` | no | `5` | The initial number of connections to create to the LDAP server on startup. -| `user_search.pool.health_check.enabled` | no | `true` | Flag to enable or disable a health check on LDAP connections in the connection pool. Connections will be checked in the background at the specified interval. -| `user_search.pool.health_check.dn` | no | Value of `bind_dn` | The distinguished name to be retrieved as part of the health check. If `bind_dn` is not specified, a value must be specified. -| `user_search.pool.health_check.interval` | no | `60s` | The interval to perform background checks of connections in the pool. -| `group_search.base_dn` | no | - | The container DN to search for groups in which the user has membership. When this element is absent, Shield searches for the attribute specified by `user_group_attribute` set on the user in order to determine group membership. -| `group_search.scope` | no | `sub_tree` | Specifies whether the group search should be `sub_tree`, `one_level` or `base`. `one_level` only searches objects directly contained within the `base_dn`. `sub_tree` searches all objects contained under `base_dn`. `base` specifies that the `base_dn` is a group object, and that it is the only group considered. -| `group_search.filter` | no | See description | When not set, the realm will search for `group`, `groupOfNames`, or `groupOfUniqueNames`, with the attributes `member` or `memberOf`. Any instance of `{0}` in the filter will be replaced by the user attribute defined in `group_search.user_attribute` -| `group_search.user_attribute` | no | Empty | Specifies the user attribute that will be fetched and provided as a parameter to the filter. If not set, the user DN is passed into the filter. -| `unmapped_groups_as_roles` | no | false | Takes a boolean variable. When this element is set to `true`, the names of any unmapped LDAP groups are used as role names and assigned to the user. THe default value is `false`. -| `files.role_mapping` | no | `CONFIG_DIR/shield/users/role_mapping.yml` | The <> for the <>. -| `follow_referrals` | no | `true` | Boolean value that specifies whether Shield should follow referrals returned by the LDAP server. Referrals are URLs returned by the server that are to be used to continue the LDAP operation (e.g. search). -| `connect_timeout` | no | "5s" - for 5 seconds | The timeout period for establishing an LDAP connection. An `s` at the end indicates seconds, or `ms` indicates milliseconds. -| `read_timeout` | no | "5s" - for 5 seconds | The timeout period for an LDAP operation. An `s` at the end indicates seconds, or `ms` indicates milliseconds. -| `hostname_verification` | no | true | Performs hostname verification when using `ldaps` to protect against man in the middle attacks. -| `cache.ttl` | no | `20m` | Specified the time-to-live for cached user entries (a user and its credentials will be cached for this configured period of time). (use the standard Elasticsearch {ref}/common-options.html#time-units[time units]). -| `cache.max_users` | no | 100000 | Specified the maximum number of user entries that can live in the cache at a given time. -| `cache.hash_algo` | no | `ssha256` |(Expert Setting) Specifies the hashing algorithm that will be used for the in-memory cached user credentials (see <> table for all possible values). -|====== -NOTE: `user_dn_templates` is required to operate in user template mode and `user_search.base_dn` is required to operated in user search mode. Only one is required for a given realm configuration. For more information on the different modes, see <>. - -[[ref-ad-settings]] - -.Shield Active Directory Settings -[options="header"] -|====== -| Name | Required | Default | Description -| `url` | no | `ldap://:389` | A URL in the format `ldap[s]://:` If not specified the URL will be derived from the domain_name, assuming clear-text `ldap` and port `389` (e.g. `ldap://:389`). -| `domain_name` | yes | - | The domain name of Active Directory. The cluster can derive the URL and `user_search_dn` fields from values in this element if those fields are not otherwise specified. -| `unmapped_groups_as_roles` | no | false | Takes a boolean variable. When this element is set to `true`, the names of any unmapped groups and the user's relative distinguished name are used as role names and assigned to the user. THe default value is `false`. -| `files.role_mapping` | no | `CONFIG_DIR/shield/users/role_mapping.yml` | The <> for the <>. -| `user_search.base_dn` | no | Root of Active Directory | The context to search for a user. The default value for this element is the root of the Active Directory domain. -| `user_search.scope` | no | `sub_tree` | Specifies whether the user search should be `sub_tree`, `one_level` or `base`. `one_level` only searches users directly contained within the `base_dn`. `sub_tree` searches all objects contained under `base_dn`. `base` specifies that the `base_dn` is a user object, and that it is the only user considered. -| `user_search.filter` | no | See description | Specifies a filter to use to lookup a user given a username. The default filter looks up `user` objects with either `sAMAccountName` or `userPrincipalName` -| `group_search.base_dn` | no | Root of Active Directory | The context to search for groups in which the user has membership. The default value for this element is the root of the the Active Directory domain -| `group_search.scope` | no | `sub_tree` | Specifies whether the group search should be `sub_tree`, `one_level` or `base`. `one_level` searches for groups directly contained within the `base_dn`. `sub_tree` searches all objects contained under `base_dn`. `base` specifies that the `base_dn` is a group object, and that it is the only group considered. -| `timeout.tcp_connect` | no | `5s` - for 5 seconds | The TCP connect timeout period for establishing an LDAP connection. An `s` at the end indicates seconds, or `ms` indicates milliseconds. -| `timeout.tcp_read` | no | `5s` - for 5 seconds | The TCP read timeout period after establishing an LDAP connection. An `s` at the end indicates seconds, or `ms` indicates milliseconds. -| `timeout.ldap_search` | no | `5s` - for 5 seconds | The LDAP Server enforced timeout period for an LDAP search. An `s` at the end indicates seconds, or `ms` indicates milliseconds. -| `hostname_verification` | no | true | Performs hostname verification when using `ldaps` to protect against man in the middle attacks. -| `cache.ttl` | no | `20m` | Specified the time-to-live for cached user entries (a user and its credentials will be cached for this configured period of time). (use the standard Elasticsearch {ref}/common-options.html#time-units[time units]). -| `cache.max_users` | no | 100000 | Specified the maximum number of user entries that can live in the cache at a given time. -| `cache.hash_algo` | no | `ssha256` |(Expert Setting) Specifies the hashing algorithm that will be used for the in-memory cached user credentials (see <> table for all possible values). -|====== - -[[ref-pki-settings]] - -.Shield PKI Settings -[options="header"] -|====== -| Name | Required | Default | Description -| `username_pattern` | no | `CN=(.*?)(?:,\|$)` | The regular expression pattern used to extract the username from the certificate DN. The first match group is the used as the username. Default is `CN=(.*?)(?:,\|$)` -| `truststore.path` | no | `shield.ssl.keystore` | The path of a truststore to use. The default truststore is the one defined by <> -| `truststore.password` | no | - | The password to the truststore. Must be provided if `truststore.path` is set. -| `truststore.algorithm` | no | SunX509 | Algorithm for the trustsore. Default is `SunX509` -| `files.role_mapping` | no | `CONFIG_DIR/shield/users/role_mapping.yml` | Specifies the <> for the <>. -|====== - -[[ref-cache-hash-algo]] -.Cache hash algorithms -|======================= -| Algorithm | Description -| `ssha256` | Uses a salted `SHA-256` algorithm (default). -| `md5` | Uses `MD5` algorithm. -| `sha1` | Uses `SHA1` algorithm. -| `bcrypt` | Uses `bcrypt` algorithm with salt generated in 10 rounds. -| `bcrypt4` | Uses `bcrypt` algorithm with salt generated in 4 rounds. -| `bcrypt5` | Uses `bcrypt` algorithm with salt generated in 5 rounds. -| `bcrypt6` | Uses `bcrypt` algorithm with salt generated in 6 rounds. -| `bcrypt7` | Uses `bcrypt` algorithm with salt generated in 7 rounds. -| `bcrypt8` | Uses `bcrypt` algorithm with salt generated in 8 rounds. -| `bcrypt9` | Uses `bcrypt` algorithm with salt generated in 9 rounds. -| `noop`,`clear_text` | Doesn't hash the credentials and keeps it in clear text in memory. CAUTION: - keeping clear text is considered insecure and can be compromised at the OS - level (e.g. memory dumps and `ptrace`). -|======================= - -[[ref-roles-settings]] - -.Shield Roles Settings -[options="header"] -|====== -| Name | Default | Description -| `shield.authz.store.file.roles` | `CONFIG_DIR/shield/users/roles.yml` | The <> of the roles definition file. -|====== - -[[ref-ssl-tls-settings]] - -[float] -==== TLS/SSL Settings - -.Shield TLS/SSL Settings -[options="header"] -|====== -| Name | Default | Description -| `shield.ssl.keystore.path` | None | Absolute path to the keystore that holds the private keys -| `shield.ssl.keystore.password` | None | Password to the keystore -| `shield.ssl.keystore.key_password` | Same value as `shield.ssl.keystore.password` | Password for the private key in the keystore -| `shield.ssl.keystore.algorithm` | SunX509 | Format for the keystore -| `shield.ssl.truststore.path` | `shield.ssl.keystore.path` | If not set, this setting defaults to `shield.ssl.keystore` -| `shield.ssl.truststore.password` | `shield.ssl.keystore.password` | Password to the truststore -| `shield.ssl.truststore.algorithm` | SunX509 | Format for the truststore -| `shield.ssl.protocol` | `TLSv1.2` | Protocol for security: `SSL`, `SSLv2`, `SSLv3`, `TLS`, `TLSv1`, `TLSv1.1`, `TLSv1.2` -| `shield.ssl.supported_protocols` | `TLSv1`, `TLSv1.1`, `TLSv1.2` | Supported protocols with versions. Valid protocols: `SSLv2Hello`, `SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2` -| `shield.ssl.ciphers` | `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA`, `TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA` | Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/7/docs/technotes/guides/security/SunProviders.html[Java Cryptography Architecture documentation]. Cipher suites using key lengths greater than 128 bits require the <>. -| `shield.ssl.hostname_verification` | `true` | Performs hostname verification on transport connections. This is enabled by default to protect against man in the middle attacks. -| `shield.ssl.hostname_verification.resolve_name` | `true` | A reverse DNS lookup is necessary to find the hostname when connecting to a node via an IP Address. If this is disabled and IP addresses are used to connect to a node, the IP address must be specified as a `SubjectAlternativeName` when <> or hostname verification will fail. IP addresses will be used to connect to a node if they are used in following settings: `network.host`, `network.publish_host`, `transport.publish_host`, `transport.profiles.$PROFILE.publish_host`, `discovery.zen.ping.unicast.hosts` -| `shield.ssl.session.cache_size` | `1000` | Number of SSL Sessions to cache in order to support session resumption. Setting the value to `0` means there is no size limit. -| `shield.ssl.session.cache_timeout` | `24h` | The time after the creation of a SSL session before it times out. (uses the standard Elasticsearch {ref}/common-options.html#time-units[time units]). -| `shield.transport.ssl` | `false` | Set this parameter to `true` to enable SSL/TLS -| `shield.transport.ssl.client.auth` | `required` | Require client side certificates for transport protocol. Valid values are `required`, `optional`, and `no`. `required` forces a client to present a certificate, while `optional` requests a client certificate but the client is not required to present one. -| `shield.transport.filter.allow` | None | List of IP addresses to allow -| `shield.transport.filter.deny` | None | List of IP addresses to deny -| `shield.http.ssl` | `false` | Set this parameter to `true` to enable SSL/TLS -| `shield.http.ssl.client.auth` | `no` | Require client side certificates for HTTP. Valid values are `required`, `optional`, and `no`. `required` forces a client to present a certificate, while `optional` requests a client certificate but the client is not required to present one. -| `shield.http.filter.allow` | None | List of IP addresses to allow just for HTTP -| `shield.http.filter.deny` | None | List of IP addresses to deny just for HTTP -|====== - -[[ref-ssl-tls-profile-settings]] - -.Shield TLS/SSL settings per profile -[options="header"] -|====== -| Name | Default | Description -| `transport.profiles.$PROFILE.shield.ssl` | Same as `shield.transport.ssl`| Setting this parameter to true will enable SSL/TLS for this profile; false will disable SSL/TLS for this profile. -| `transport.profiles.$PROFILE.shield.truststore.path` | None | Absolute path to the truststore of this profile -| `transport.profiles.$PROFILE.shield.truststore.password` | None | Password to the truststore -| `transport.profiles.$PROFILE.shield.truststore.algorithm` | SunX509 | Format for the truststore -| `transport.profiles.$PROFILE.shield.keystore.path` | None | Absolute path to the keystore of this profile -| `transport.profiles.$PROFILE.shield.keystore.password` | None | Password to the keystore -| `transport.profiles.$PROFILE.shield.keystore.key_password` | Same value as `transport.profiles.$PROFILE.shield.keystore.password` | Password for the private key in the keystore -| `transport.profiles.$PROFILE.shield.keystore.algorithm` | SunX509 | Format for the keystore -| `transport.profiles.$PROFILE.shield.session.cache_size` | `1000` | Number of SSL Sessions to cache in order to support session resumption. Setting the value to `0` means there is no size limit. -| `transport.profiles.$PROFILE.shield.session.cache_timeout` | `24h` | The time after the creation of a SSL session before it times out. (uses the standard Elasticsearch {ref}/common-options.html#time-units[time units]). -| `transport.profiles.$PROFILE.shield.filter.allow` | None | List of IP addresses to allow for this profile -| `transport.profiles.$PROFILE.shield.filter.deny` | None | List of IP addresses to deny for this profile -| `transport.profiles.$PROFILE.shield.ssl.client.auth` | `required` | Require client side certificates. Valid values are `required`, `optional`, and `no`. `required` forces a client to present a certificate, while `optional` requests a client certificate but the client is not required to present one. -| `transport.profiles.$PROFILE.shield.type` | `node` | Defines allowed actions on this profile, allowed values: `node` and `client` -| `transport.profiles.$PROFILE.shield.ciphers` | `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA`, `TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA` | Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/7/docs/technotes/guides/security/SunProviders.html[Java Cryptography Architecture documentation]. Cipher suites using key lengths greater than 128 bits require the <>. -| `transport.profiles.$PROFILE.shield.protocol` | `TLSv1.2` | Protocol for security: `SSL`, `SSLv2`, `SSLv3`, `TLS`, `TLSv1`, `TLSv1.1`, `TLSv1.2` -| `transport.profiles.$PROFILE.shield.supported_protocols` | `TLSv1`, `TLSv1.1`, `TLSv1.2` | Supported protocols with versions. Valid protocols: `SSLv2Hello`, `SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2` -|====== - -[[ref-shield-files]] - -[float] -=== Files used by Shield - -The Shield security plugin uses the following files: - -* `CONFIG_DIR/shield/roles.yml` defines the roles in use on the cluster (read more <>). -* `CONFIG_DIR/shield/users` defines the hashed passwords for users on the cluster (read more <>). -* `CONFIG_DIR/shield/users_roles` defines the role assignments for users on the cluster (read more <>). -* `CONFIG_DIR/shield/role_mapping.yml` defines the role assignments for a Distinguished Name (DN) to a role. This allows for -LDAP and Active Directory groups and users and PKI users to be mapped to roles (read more <>). -* `CONFIG_DIR/shield/logging.yml` contains audit information (read more <>). -* `CONFIG_DIR/shield/system_key` holds a cluster secret key used for message authentication. For more information, see <>. - -[[ref-shield-files-location]] -IMPORTANT: Any files that Shield uses must be stored in the Elasticsearch {ref}/setup-dir-layout.html#setup-dir-layout[configuration directory]. -Elasticsearch runs with restricted permissions and is only permitted to read from the locations configured in the directory -layout for enhanced security. - -Several of these files are in the YAML format. When you edit these files, be aware that YAML is indentation-level -sensitive and indentation errors can lead to configuration errors. Avoid the tab character to set indentation levels, -or use an editor that automatically expands tabs to spaces. - -Be careful to properly escape YAML constructs such as `:` or leading exclamation points within quoted strings. Using -the `|` or `>` characters to define block literals instead of escaping the problematic characters can help avoid -problems. diff --git a/shield/docs/public/release-notes.asciidoc b/shield/docs/public/release-notes.asciidoc deleted file mode 100644 index 4512f716fde..00000000000 --- a/shield/docs/public/release-notes.asciidoc +++ /dev/null @@ -1,230 +0,0 @@ -[[release-notes]] -== Release Notes - -[float] -[[version-compatibility]] -=== Version Compatibility - -You must run the version of Shield that matches the version of Elasticsearch you are running. For -example, Shield {version} requires Elasticsearch {version}. - -[float] -[[upgrade-instructions]] -=== Upgrading Shield - -To upgrade Shield, just uninstall the current Shield plugin and install the new version of Shield. Your configuration -will be preserved and you do this with a rolling upgrade of Elasticsearch when upgrading to a new minor version; a full -cluster restart is required when upgrading from Elasticsearch 1.x. On each node, after you have stopped it run: - -[source,shell] ---------------------------------------------------- -bin/plugin remove shield -bin/plugin install shield ---------------------------------------------------- - -Then start the node. Larger sites should follow the steps in the {ref}/rolling-upgrades.html[rolling upgrade section] -to ensure recovery is as quick as possible. - -On upgrade, your current configuration files will remain untouched. The configuration files provided by the new version -of Shield will be added with a `.new` extension. - -[float] -==== Updated Role Definitions -The default role definitions in the `roles.yml` file may need to be changed to ensure proper functionality with other -applications such as Marvel and Kibana. Any role changes will be found in `roles.yml.new` after upgrading to the new -version of Shield. We recommend copying the changes listed below to your `roles.yml` file. - -* added[2.0.0] The permission on all the roles are updated to the verbose format to make it easier to enable field level and document level security. The `transport_client` role has been updated to work with Elasticsearch 2.0.0. - The `marvel_user` role has been updated to work with Marvel 2.0 and a `remote_marvel_agent` role has been added. The `kibana3` and `marvel_agent` roles have been removed. -* added[1.1.0] `kibana4_server` role added that defines the minimum set of permissions necessary for the Kibana 4 server. -* added[1.0.1] `kibana4` role updated to work with new features in Kibana 4 RC1 - -[float] -[[changelist]] -=== Change List - -[float] -==== 2.1.0 -November 24, 2015 - -.Breaking Changes -* Same as 2.0.1. <> is now disabled by default. Set `shield.dls_fls.enabled` to `true` in `elasticsearch.yml` to enable it. You cannot submit `_bulk` update requests when document and field level security is enabled. - -.Enhancements -* Adds support for Elasticsearch 2.1.0. - -[float] -==== 2.0.1 -November 24, 2015 - -.Breaking Changes -* <> is now disabled by default. Set `shield.dls_fls.enabled` to `true` in `elasticsearch.yml` to enable it. You cannot submit `_bulk` update requests when document and field level security is enabled. - -.Enhancement -* Adds support for Elasticsearch 2.0.1. - -[float] -==== 2.0.0 -October 28, 2015 - -.Breaking Changes -* All files that Shield uses must be kept in the <> due to the enhanced security of Elasticsearch 2.0. -* The network format has been changed from all previous versions of Shield and a full cluster restart is required to upgrade to Shield 2.0. - -.New Features -* <> support has been added and can be -configured per role. -* Support for <> has been added, allowing Shield to integrate with more authentication sources and methods. -* <> has also been added, which allows a user to send a request to elasticsearch that will be run -with the specified user's permissions. - -.Bug Fixes -* <> now captures requests from nodes using a different system key as tampered requests. -* The <> stores the type of request when available. -* `esusers` and `syskeygen` work when spaces are in the elasticsearch installation path. -* Fixed a rare issue where authentication fails even when the username and password are correct. - -[float] -==== 1.3.3 - -.Bug Fixes -* Fixed a rare issue where authentication fails even when the username and password are correct. -* The <> stores the type of request when available. - -.Enhancements -* Tampered requests with a bad header are now audited. - -[float] -==== 1.3.2 -August 10, 2015 - -.Bug Fixes -* When using the <> mechanism, connection errors during startup no longer cause the node to stop. -* The <> no longer generates invalid JSON. -* The <> starts properly when forwarding the audit events to a remote cluster and uses -the correct user to index the audit events. - -[float] -==== 1.3.1 -July 21, 2015 - -.Bug Fixes -* Fixes <> serialization to work with Shield 1.2.1 and earlier. -** NOTE: if you are upgrading from Shield 1.3.0 or Shield 1.2.2 a {ref-17}/setup-upgrade.html#restart-upgrade[cluster restart upgrade] -will be necessary. When upgrading from other versions of Shield, follow the normal <>. - -[float] -==== 1.3.0 -June 24, 2015 - -.Breaking Changes -* The `sha2` and `apr1` hashing algorithms have been removed as options for the <>. - If your existing Shield installation uses either of these options, remove the setting and use the default `ssha256` - algorithm. -* The `users` file now only supports `bcrypt` password hashing. All existing passwords stored using the `esusers` tool - have been hashed with `bcrypt` and are not affected. - -.New Features -* <>: Adds Public Key Infrastructure (PKI) authentication through the use of X.509 certificates in place of - username and password credentials. -* <>: An index based output has been added for storing audit events in an Elasticsearch index. - -.Enhancements -* TLS 1.2 is now the default protocol. -* Clients that do not support pre-emptive basic authentication can now support both anonymous and authenticated access - by specifying the `shield.authc.anonymous.authz_exception` <> with a value of `false`. -* Reduced logging for common SSL exceptions, such as a client closing the connection during a handshake. - -.Bug Fixes -* The `esusers` and `syskeygen` tools now work correctly with environment variables in the RPM and DEB installation - environment files `/etc/sysconfig/elasticsearch` and `/etc/default/elasticsearch`. -* Default ciphers no longer include `TLS_DHE_RSA_WITH_AES_128_CBC_SHA`. - -[float] -==== 1.2.3 -July 21, 2015 - -.Bug Fixes -* Fixes <> serialization to work with Shield 1.2.1 and earlier. -** NOTE: if you are upgrading from Shield 1.2.2 a {ref-17}/setup-upgrade.html#restart-upgrade[cluster restart upgrade] -will be necessary. When upgrading from other versions of Shield, follow the normal <>. - -[float] -==== 1.2.2 -June 24, 2015 - -.Bug Fixes -* The `esusers` tool no longer warns about missing roles that are properly defined in the `roles.yml` file. -* The period character, `.`, is now allowed in usernames and role names. -* The {ref-17}/query-dsl-terms-filter.html#_caching_19[terms filter lookup cache] has been disabled to ensure all requests - are properly authorized. This removes the need to manually disable the terms filter cache. -* For LDAP client connections, only the protocols and ciphers specified in the `shield.ssl.supported_protocols` and - `shield.ssl.ciphers` <> will be used. -* The auditing mechanism now logs authentication failed events when a request contains an invalid authentication token. - -[float] -==== 1.2.1 -April 29, 2015 - -.Bug Fixes -* Several bug fixes including a fix to ensure that {ref}/disk.html[Disk-based Shard Allocation] -works properly with Shield - -[float] -==== 1.2.0 -March 24, 2015 - -.Enhancements -* Adds support for Elasticsearch 1.5 - -[float] -==== 1.1.1 -April 29, 2015 - -.Bug Fixes -* Several bug fixes including a fix to ensure that {ref}/disk.html[Disk-based Shard Allocation] -works properly with Shield - -[float] -==== 1.1.0 -March 24, 2015 - -.New Features -* LDAP: -** Add the ability to bind as a specific user for LDAP searches, which removes the need to specify `user_dn_templates`. -This mode of operation also makes use of connection pooling for better performance. Please see <> -for more information. -** User distinguished names (DNs) can now be used for <>. -* Authentication: -** <> is now supported (disabled by default). -* IP Filtering: -** IP Filtering settings can now be <> using the {ref}/cluster-update-settings.html[Cluster Update Settings API]. - -.Enhancements -* Significant memory footprint reduction of internal data structures -* Test if SSL/TLS ciphers are supported and warn if any of the specified ciphers are not supported -* Reduce the amount of logging when a non-encrypted connection is opened and `https` is being used -* Added the <>, which is a role that contains the minimum set of permissions required for the Kibana 4 server. -* In-memory user credential caching hash algorithm defaults now to salted SHA-256 (see <> - -.Bug Fixes -* Filter out sensitive settings from the settings APIs - -[float] -==== 1.0.2 -March 24, 2015 - -.Bug Fixes -* Filter out sensitive settings from the settings APIs -* Significant memory footprint reduction of internal data structures - -[float] -==== 1.0.1 -February 13, 2015 - -.Bug Fixes -* Fixed dependency issues with Elasticsearch 1.4.3 and (Lucene 4.10.3 that comes with it) -* Fixed bug in how user roles were handled. When multiple roles were defined for a user, and one of the - roles only had cluster permissions, not all privileges were properly evaluated. -* Updated `kibana4` permissions to be compatible with Kibana 4 RC1 -* Ensure the mandatory `base_dn` settings is set in the `ldap` realm configuration diff --git a/shield/docs/public/securing-communications.asciidoc b/shield/docs/public/securing-communications.asciidoc deleted file mode 100644 index 4e11083d15d..00000000000 --- a/shield/docs/public/securing-communications.asciidoc +++ /dev/null @@ -1,19 +0,0 @@ -[[securing-communications]] -== Securing Communications with Encryption and IP Filtering - -Elasticsearch nodes store data that may be confidential. Attacks on the data may come from the network. These attacks -could include sniffing of the data, manipulation of the data, and attempts to gain access to the server and thus the -files storing the data. Securing your nodes with the procedures below helps to reduce risk from network-based attacks. - -This section shows how to: - -* Encrypt traffic to and from Elasticsearch nodes using SSL/TLS, -* Require that nodes authenticate new nodes that join the cluster using SSL certificates, and -* Make it more difficult for remote attackers to issue any commands to Elasticsearch. - -The authentication of new nodes helps prevent a rogue node from joining the cluster and receiving data through replication. - -include::securing-communications/setting-up-ssl.asciidoc[] -include::securing-communications/enabling-cipher-suites.asciidoc[] -include::securing-communications/separating-node-client-traffic.asciidoc[] -include::securing-communications/using-ip-filtering.asciidoc[] diff --git a/shield/docs/public/securing-communications/enabling-cipher-suites.asciidoc b/shield/docs/public/securing-communications/enabling-cipher-suites.asciidoc deleted file mode 100644 index acefe147696..00000000000 --- a/shield/docs/public/securing-communications/enabling-cipher-suites.asciidoc +++ /dev/null @@ -1,19 +0,0 @@ -[[ciphers]] -=== Enabling Cipher Suites for Stronger Encryption - -The SSL/TLS protocols use a cipher suite that determines the strength of encryption used to protect the data. You may -want to increase the strength of encryption used when using a Oracle JVM; the IcedTea OpenJDK ships without these -restrictions in place. This step is not required to successfully use Shield. - -The Java Cryptography Extension (JCE) Unlimited Strength Jurisdiction Policy Files enable additional cipher suites for -Java in a separate JAR file that you need to add to your Java installation. You can download this JAR file from -Oracle's http://www.oracle.com/technetwork/java/javase/downloads/index.html[download page]. The JCE Unlimited Strength -Jurisdiction Policy Files are required for encryption with key lengths greater than 128 bits, such as 256-bit AES -encryption. - -After installation, all cipher suites in the JCE are available for use. To enable the use of stronger cipher suites with -Shield, configure the `ciphers` parameter. See the <> section -of this document for specific parameter information. - -NOTE: The JCE Unlimited Strength Jurisdiction Policy Files must be installed on all nodes to establish an improved level -of encryption strength. diff --git a/shield/docs/public/securing-communications/separating-node-client-traffic.asciidoc b/shield/docs/public/securing-communications/separating-node-client-traffic.asciidoc deleted file mode 100644 index d1fcf77078f..00000000000 --- a/shield/docs/public/securing-communications/separating-node-client-traffic.asciidoc +++ /dev/null @@ -1,65 +0,0 @@ -[[separating-node-client-traffic]] -=== Separating node to node and client traffic - -Elasticsearch has the feature of so called {ref}/modules-transport.html#_tcp_transport_profiles[tcp transport profiles]. -This allows Elasticsearch to bind to several ports and addresses. Shield extends on this functionality to enhance the -security of the cluster by enabling the separation of node to node transport traffic from client transport traffic. This -is important if the client transport traffic is not trusted and could potentially be malicious. To separate the node to -node traffic from the client traffic, add the following to `elasticsearch.yml`: - -[source, yaml] --------------------------------------------------- -transport.profiles.client<1>: - port: 9500-9600 <2> - shield: - type: client <3> --------------------------------------------------- -<1> `client` is the name of this example profile -<2> The port range that will be used by transport clients to communicate with this cluster -<3> A type of `client` enables additional filters for added security by denying internal cluster operations (e.g shard -level actions and ping requests) - -If supported by your environment, an internal network can be used for node to node traffic and public network can be -used for client traffic by adding the following to `elasticsearch.yml`: -[source, yaml] --------------------------------------------------- -transport.profiles.default.bind_host: 10.0.0.1 <1> -transport.profiles.client.bind_host: 1.1.1.1 <2> --------------------------------------------------- -<1> The bind address for the network that will be used for node to node communication -<2> The bind address for the network used for client communication - -If separate networks are not available, then <> can be enabled to limit access to the profiles. - -The tcp transport profiles also allow for enabling SSL on a per profile basis. This is useful if you have a secured network -for the node to node communication, but the client is on an unsecured network. To enable SSL on a client profile when SSL is -disabled for node to node communication, add the following to `elasticsearch.yml`: -[source, yaml] --------------------------------------------------- -transport.profiles.client.ssl: true <1> --------------------------------------------------- -<1> This enables SSL on the client profile. The default value for this setting is the value of `shield.transport.ssl`. - -When using SSL for transport, a different set of certificates can also be used for the client traffic by adding the -following to `elasticsearch.yml`: -[source, yaml] --------------------------------------------------- -transport.profiles.client.shield.truststore: - path: /path/to/another/truststore - password: changeme - -transport.profiles.client.shield.keystore: - path: /path/to/another/keystore - password: changeme --------------------------------------------------- - -To change the default behavior that requires certificates for transport clients, set the following value in the `elasticsearch.yml` -file: - -[source, yaml] --------------------------------------------------- -transport.profiles.client.shield.ssl.client.auth: no --------------------------------------------------- - -This setting keeps certificate authentication active for node-to-node traffic, but removes the requirement to distribute -a signed certificate to transport clients. Please see the <> section. diff --git a/shield/docs/public/securing-communications/setting-up-ssl.asciidoc b/shield/docs/public/securing-communications/setting-up-ssl.asciidoc deleted file mode 100644 index fee8463e997..00000000000 --- a/shield/docs/public/securing-communications/setting-up-ssl.asciidoc +++ /dev/null @@ -1,267 +0,0 @@ -[[ssl-tls]] -=== Setting Up SSL/TLS on a Cluster - -Shield enables you to encrypt traffic to and from nodes in your Elasticsearch cluster. Connections -are secured using Transport Layer Security (TLS). - -WARNING: Nodes that do not have encryption enabled send passwords in plain text. - -To enable encryption, you need to perform the following steps on each node in the cluster: - -. <>. - -. <> to: -.. Identify itself using its signed certificate. -.. Enable SSL on the transport and HTTP layers. -.. Disable multicast. - -. Restart Elasticsearch. - -[[installing-node-certificates]] -==== Installing Node Certificates - -Node certificates should be signed by a certificate authority (CA) that is trusted by every node -in the cluster. You can use a third-party CA, your organization's existing CA, or -<> specifically for signing node certificates. - -When a client connects to a node using SSL/TLS, the node presents its certificate to the -client and proves that it owns the private key linked with the certificate. The client then -determines if the node's certificate is valid, trusted, and matches the hostname or IP address -it is trying to connect to. A node acts as a client when connecting to other nodes in the cluster, -so every node must trust all of the other nodes in the cluster. - -NOTE: While it is technically possible to use self-signed certificates, we strongly recommend using certificates signed by a CA to establish trust between nodes. Self-signed certificates must be trusted individually, which means that each node must have every other node's certificate installed. If you add a node to the cluster, you have to install the new node's self-signed certificate on all of the existing nodes and restart them. When you use CA-signed certificates, the existing nodes just need to trust the CA used to sign the new node's certificate. (You should use the same CA to sign all of your node certificates.) - -To install a signed certificate, you need to: - -. <>. -. <>. -. <>. -. <>. - -[[private-key]] -===== Creating a Keystore and Generating a Certificate -To create a keystore and generate a node certificate: - -. Create a node keystore and import your CA's certificate with https://docs.oracle.com/javase/8/docs/technotes/tools/unix/keytool.html[Java Keytool]. This configures the node to trust certificates signed by the CA. For Elasticsearch -to access the keystore, it must be located under the Elasticsearch configuration directory. For example, the following command creates a keystore for `node01` and and imports the CA certificate `cacert.pem`. -+ -[source,shell] --------------------------------------------------- -cd CONFIG_DIR/shield -keytool -importcert -keystore node01.jks -file cacert.pem -alias my_ca --------------------------------------------------- -+ -The Java keystore file (.jks) securely stores certificates for the node. The CA cert must be a -PEM encoded certificate. -+ -When you create a keystore, you are prompted to set a password. This password protects the -integrity of the keystore. You need to provide it whenever you interact with the keystore. -+ -IMPORTANT: When the CA certificate expires, you must update the node's keystore with the new CA -certificate. -+ -You can also store the CA certificate in a separate truststore. For more -information, see <>. - -. Generate a private key and certificate for the node with Java Keytool. For example, the following -command creates a key and certificate for `node01`: -+ -[source,shell] --------------------------------------------------- -keytool -genkey -alias node01 -keystore node01.jks -keyalg RSA -keysize 2048 -validity 712 -ext san=dns:node01.example.com,ip:192.168.1.1 --------------------------------------------------- -+ -This command creates an RSA private key with a key size of 2048 bits and a public certificate that -is valid for 712 days. The key and certificate are stored in the `node01.jks` keystore. -+ -The `san` value specifies all alternative names for the node. The generated certificate is valid for the DNS names and IP addresses specified as alternative names. You can specify multiple DNS or IP address entries as a comma-separated list. -+ -[IMPORTANT] -.Specifying the Node Identity -========================== -With SSL/TLS is enabled, when node A connects to node B, node A normally verifies the identity of -node B by checking the identity information specified in node B's certificate. This means that you -must include node identity information when you create a node's certificate. - -The recommended way to specify the node identity when creating a certificate is to specify the -`-ext` option and list all of the node's names and IP addresses in the `san` -(Subject Alternative Name) attribute. - -If you use a commercial CA, the DNS names and IP addresses used to identify a node must be publicly resolvable. Internal DNS names and private IP addresses are not accepted due to -https://cabforum.org/internal-names/[security concerns]. - -If you need to use private DNS names and IP addresses, using an internal CA is the most secure -option. It enables you to specify node identities and ensure node identities are verified when -nodes connect. If you must use a commercial CA and private DNS names or IP addresses, you cannot -include the node identity in the certificate, so the only option is to disable -<>. -========================== -+ -When you run `keytool -genkey`, Keytool prompts you for the information needed to populate the -node's distinguished name that's stored the certificate. For example: -+ -[source, shell] --------------------------------------------------- -What is your first and last name? - [Unknown]: Elasticsearch node01 <1> -What is the name of your organizational unit? - [Unknown]: test -What is the name of your organization? - [Unknown]: Elasticsearch -What is the name of your City or Locality? - [Unknown]: Amsterdam -What is the name of your State or Province? - [Unknown]: Amsterdam -What is the two-letter country code for this unit? - [Unknown]: NL -Is CN=Elasticsearch node01, OU=test, O=elasticsearch, L=Amsterdam, ST=Amsterdam, C=NL correct? - [no]: yes - -Enter key password for <2> - (RETURN if same as keystore password): --------------------------------------------------- -<1> Provides information about the node that this certificate is intended for. In the past, this field specified the node's identity using a DNS name, but that behavior has been deprecated. -<2> If you don't specify a password for the certificate, the keystore password is used. -+ -[IMPORTANT] -.Extended Key Usage -========================== -The Extended Key Usage attribute in a certificate is used to indicate the purpose of the key. By default `keytool` does not set this attribute in the certificate. If you are generating your certificates with another tool, please ensure the certificates support both `serverAuth` and `clientAuth` if the Extended Key Usage attribute is set. -========================== - -[float] -[[generate-csr]] -===== Creating a Certificate Signing Request - -A node's certificate needs to be signed by a trusted CA for the certificate to be trusted. To get a certificate signed, you need to create a certificate signing request (CSR) and send it to your CA. - -To create a CSR with Java Keytool, use the `keytool t-certreq` command. You specify the same alias, keystore, key algorithm, and DNS names and IP addresses that you used when you created the node certificate. Specify where you want to store the CSR with the `-file` option. - -[source, shell] --------------------------------------------------- -keytool -certreq -alias node01 -keystore node01.jks -file node01.csr -keyalg rsa -ext san=dns:node01.example.com,ip:192.168.1.1 --------------------------------------------------- - -[float] -[[send-csr]] -===== Send the Signing Request - -To get a signed certificate, send the generated CSR file to your CA. The CA will sign it and send -you the signed version of the certificate. - -NOTE: If you are running your own CA, see <> for signing instructions. - -[float] -[[install-signed-cert]] -===== Install the Signed Certificate - -To install the signed certificate, use `keytool -importcert` to add it to the node's keystore. You -specify the same alias and keystore that you used when you created the node certificate. - -[source, shell] --------------------------------------------------- -cd CONFIG_DIR/shield -keytool -importcert -keystore node01.jks -file node01-signed.crt -alias node01 --------------------------------------------------- - -[NOTE] -========================== -If you attempt to import a PEM-encoded certificate that contains extra text headers, you might get -the error: `java.security.cert.CertificateParsingException: invalid DER-encoded certificate data`. -Use the following `openssl` command to remove the extra headers and then use `keytool` to import -the certificate. - -[source, shell] --------------------------------------------------- -openssl x509 -in node01-signed.crt -out node01-signed-noheaders.crt --------------------------------------------------- -========================== - -[[enable-ssl]] -==== Enabling SSL in the Node Configuration - -Once you have added the signed certificate to the node's keystore, you need to modify the node -configuration to enable SSL. - -NOTE: All SSL/TLS related node settings that are considered to be highly sensitive and therefore -are not exposed via the {ref}/cluster-nodes-info.html#cluster-nodes-info[nodes info API]. - -[[configure-ssl]] -To enable SSL, make the following changes in `elasticsearch.yml`: - -. Specify the location of the node's keystore and the password(s) needed to access the node's -certificate. For example: -+ -[source, yaml] --------------------------------------------------- -shield.ssl.keystore.path: /home/es/config/shield/node01.jks <1> -shield.ssl.keystore.password: myPass <2> -shield.ssl.keystore.key_password: myKeyPass <3> --------------------------------------------------- -<1> The full path to the node keystore file. This must be a location within the Elasticsearch -configuration directory. -<2> The password used to access the keystore. -<3> The password used to access the certificate. This is only required if you specified a separate -certificate password when generating the certificate. - -. Enable SSL on the transport networking layer to ensure that communication between nodes is -encrypted: -+ -[source, yaml] --------------------------------------------------- -shield.transport.ssl: true --------------------------------------------------- -+ -NOTE: Transport clients can only connect to the cluster with a valid username and password even if -this setting is disabled. - -. Enable SSL on the HTTP layer to ensure that communication between HTTP clients and the cluster is encrypted: -+ -[source, yaml] --------------------------------------------------- -shield.http.ssl: true --------------------------------------------------- -+ -NOTE: HTTP clients can only connect to the cluster with a valid username and password even if this -setting is disabled. - -. Disable {ref}/modules-discovery.html[multicast discovery]: -+ -[source, yaml] --------------------------------------------------- -discovery.zen.ping.multicast.enabled: false -discovery.zen.ping.unicast.hosts: ["node01:9300", "node02:9301"] --------------------------------------------------- - -. Restart Elasticsearch so these configuration changes take effect. - -[[create-truststore]] -==== Configuring a Separate Truststore -You can store trusted CA certificates in a node's keystore, or create a separate truststore for CA -certificates. - -To use a separate truststore: - -. Create a node truststore and import the CA certificate(s) you want to trust with Java Keytool. For example, the following command imports the CA certificate `cacert.pem` into `truststore.jks`. If the specified truststore doesn't exist, it is created. -+ -[source,shell] --------------------------------------------------- -cd CONFIG_DIR/shield -keytool -importcert -keystore truststore.jks -file cacert.pem --------------------------------------------------- -+ -When you create a truststore, you are prompted to set a password. This password protects the -integrity of the truststore. You need to provide it whenever you interact with the truststore. - -. In `elasticsearch.yml`, specify the location of the node's truststore and the password needed to -access it. For example: -+ -[source, yaml] --------------------------------------------------- -shield.ssl.truststore.path: /home/es/config/shield/truststore.jks <1> -shield.ssl.truststore.password: myPass <2> --------------------------------------------------- -<1> The full path to the truststore file. This must be a location within the -Elasticsearch configuration directory. -<2> The password needed to access the truststore. diff --git a/shield/docs/public/securing-communications/using-ip-filtering.asciidoc b/shield/docs/public/securing-communications/using-ip-filtering.asciidoc deleted file mode 100644 index 701f87c412e..00000000000 --- a/shield/docs/public/securing-communications/using-ip-filtering.asciidoc +++ /dev/null @@ -1,131 +0,0 @@ -[[ip-filtering]] -=== Using IP Filtering - -You can apply IP filtering to application clients, node clients, or transport clients, in addition to other nodes that -are attempting to join the cluster. - -If a node's IP address is on the blacklist, Shield will still allow the connection to Elasticsearch. The connection will -be dropped immediately, and no requests will be processed. - -NOTE: Elasticsearch installations are not designed to be publicly accessible over the Internet. IP Filtering and the -other security capabilities of Shield do not change this condition. - -[float] -==== Enabling IP filtering - -Shield features an access control feature that allows or rejects hosts, domains, or subnets. - -You configure IP filtering by specifying the `shield.transport.filter.allow` and `shield.transport.filter.deny` settings in in `elasticsearch.yml`. Allow rules take prececence over the deny rules. - -.Example 1. Allow/Deny Statement Priority -[source,yaml] --------------------------------------------------- -shield.transport.filter.allow: "192.168.0.1" -shield.transport.filter.deny: "192.168.0.0/24" --------------------------------------------------- - -The `_all` keyword denies all connections that are not explicitly allowed earlier in the file. - -.Example 2. `_all` Keyword Usage -[source,yaml] --------------------------------------------------- -shield.transport.filter.allow: [ "192.168.0.1", "192.168.0.2", "192.168.0.3", "192.168.0.4" ] -shield.transport.filter.deny: _all --------------------------------------------------- - -IP Filtering configuration files support IPv6 addresses. - -.Example 3. IPv6 Filtering -[source,yaml] --------------------------------------------------- -shield.transport.filter.allow: "2001:0db8:1234::/48" -shield.transport.filter.deny: "1234:0db8:85a3:0000:0000:8a2e:0370:7334" --------------------------------------------------- - -Shield supports hostname filtering when DNS lookups are available. - -.Example 4. Hostname Filtering -[source,yaml] --------------------------------------------------- -shield.transport.filter.allow: localhost -shield.transport.filter.deny: '*.google.com' --------------------------------------------------- - -[float] -==== Disabling IP Filtering - -Disabling IP filtering can slightly improve performance under some conditions. To disable IP filtering entirely, set the -value of the `shield.transport.filter.enabled` attribute in the `elasticsearch.yml` configuration file to `false`. - -.Example 5. Disabled IP Filtering -[source,yaml] --------------------------------------------------- -shield.transport.filter.enabled: false --------------------------------------------------- - -You can also disable IP filtering for the transport protocol but enable it for HTTP only like this - -.Example 6. Enable HTTP based IP Filtering -[source,yaml] --------------------------------------------------- -shield.transport.filter.enabled: false -shield.http.filter.enabled: true --------------------------------------------------- - -[float] -==== Specifying TCP transport profiles - -In order to support bindings on multiple host, you can specify the profile name as a prefix in order to allow/deny based on profiles - -.Example 7. Profile based filtering -[source,yaml] --------------------------------------------------- -shield.transport.filter.allow: 172.16.0.0/24 -shield.transport.filter.deny: _all -transport.profiles.client.shield.filter.allow: 192.168.0.0/24 -transport.profiles.client.shield.filter.deny: _all --------------------------------------------------- - -Note: When you do not specify a profile, `default` is used automatically. - -[float] -==== HTTP Filtering - -You may want to have different filtering between the transport and HTTP protocol - -.Example 8. HTTP only filtering -[source,yaml] --------------------------------------------------- -shield.transport.filter.allow: localhost -shield.transport.filter.deny: '*.google.com' -shield.http.filter.allow: 172.16.0.0/16 -shield.http.filter.deny: _all --------------------------------------------------- - -[float] -[[dynamic-ip-filtering]] -==== Dynamically updating ip filter settings added[1.1.0] - -In case of running in an environment with highly dynamic IP addresses like cloud based hosting it is very hard to know the IP addresses upfront when provisioning a machine. Instead of changing the configuration file and restarting the node, you can use the Cluster Update Settings API like this - -[source,json] --------------------------------------------------- -curl -XPUT localhost:9200/_cluster/settings -d '{ - "persistent" : { - "shield.transport.filter.allow" : "172.16.0.0/24" - } -}' --------------------------------------------------- - -You can also disable filtering completely setting `shield.transport.filter.enabled` like this - -[source,json] --------------------------------------------------- -curl -XPUT localhost:9200/_cluster/settings -d '{ - "persistent" : { - "shield.transport.filter.enabled" : false - } -}' --------------------------------------------------- - -NOTE: In order to not lock yourself out, the default bound transport address will never be denied. This means you can always SSH into a system and use curl to apply changes. \ No newline at end of file diff --git a/shield/docs/public/setting-up-authentication.asciidoc b/shield/docs/public/setting-up-authentication.asciidoc deleted file mode 100644 index ac29733f2df..00000000000 --- a/shield/docs/public/setting-up-authentication.asciidoc +++ /dev/null @@ -1,117 +0,0 @@ -[[setting-up-authentication]] -== Setting Up User Authentication - -Authentication identifies an individual. To gain access to restricted resources, a user must prove their identity, via -passwords, credentials, or some other means (typically referred to as authentication tokens). - -NOTE: In Shield, an authenticated user can submit requests on behalf of other another user if they -have the appropriate `run_as` permission. For more information, see -<>. - -A _realm_ is an authentication mechanism, which Shield uses to resolve and authenticate users and their roles. Shield -currently provides four realm types: - -[horizontal] -_esusers_:: A native authentication system built into Shield and available by default. See <>. -_LDAP_:: Authentication via an external Lightweight Directory Protocol. See <>. -_Active Directory_:: Authentication via an external Active Directory service. See <>. -_PKI_:: Authentication through the use of trusted X.509 certificates. See <>. - -The _esusers_, _LDAP_, and _Active Directory_ realms authenticate using the username and password authentication tokens. - -Shield also supports custom realms. If you need to integrate with another authentication system, you -can build a custom realm plugin. For more information, see <>. - -Realms live within a _realm chain_. It is essentially a prioritized list of configured realms (typically of various types). -The order of the list determines the order in which the realms will be consulted. During the authentication process, -Shield will consult and try to authenticate the request one realm at a time. Once one of the realms successfully -authenticates the request, the authentication is considered to be successful and the authenticated user will be associated -with the request (which will then proceed to the authorization phase). If a realm cannot authenticate the request, the -next in line realm in the chain will be consulted. If all realms in the chain could not authenticate the request, the -authentication is then considered to be unsuccessful and an authentication error will be returned (as HTTP status code `401`). - -NOTE: Shield attempts to authenticate to each configured realm sequentially. Some systems (e.g. Active Directory) have a -temporary lock-out period after several successive failed login attempts. If the same username exists in multiple realms, -unintentional account lockouts are possible. For more information, please see <>. - -For example, if `UserA` exists in both Active Directory and esusers, and the Active Directory realm is checked first and -esusers is checked second, an attempt to authenticate as `UserA` in the esusers realm would first attempt to authenticate -against Active Directory and fail, before successfully authenticating against the esusers realm. Because authentication is -verified on each request, the Active Directory realm would be checked - and fail - on each request for `UserA` in the esusers -realm. In this case, while the Shield request completed successfully, the account on Active Directory would have received -several failed login attempts, and that account may become temporarily locked out. Plan the order of your realms accordingly. - -The realm chain can be configured in the `elasticsearch.yml` file. When not explicitly configured, a default chain will be -created that only holds the `esusers` realm in it. When explicitly configured, the created chain will be the exact reflection -of the configuration (e.g. the only realms in the chain will be those configured realms that are enabled) - -The following snippet shows an example of realms configuration: - - -[source,yaml] ----------------------------------------- -shield.authc: - - realms: - - esusers: - type: esusers - order: 0 - - ldap1: - type: ldap - order: 1 - enabled: false - url: 'url_to_ldap1' - ... - - ldap2: - type: ldap - order: 2 - url: 'url_to_ldap2' - ... - - ad1: - type: active_directory - order: 3 - url: 'url_to_ad' ----------------------------------------- - -As can be seen above, each realm has a unique name that identifies it. There are three settings that are common to all -realms: - -* `type` (required) - Identifies the type of the ream (currently can be `esusers`, `ldap` or `active_directory`). The realm - type determines what other settings the realms should be configured with. -* `order` (optional) - Defines the priority/index of the realm within the realm chain. This will determine when the realm - will be consulted during authentication. -* `enabled` (optional) - When set to `false` the realm will be disabled and will not be added to the realm chain. This is - useful for debugging purposes, where one can remove a realm from the chain without deleting and - losing its configuration. - -The realm types can roughly be categorized to two categories: - -* `internal` - Internal realm types are realms that are internal to Elasticsearch and don't require any communication with - external parties - they are fully managed by shield. There can only be a maximum of one configured realm - per internal realm type. (Currently, only one internal realm type exists - `esusers`). - -* `external` - External realm types are realms that require interaction with parties/components external to Elasticsearch, - typically, with enterprise level identity management systems. Unlike the `internal` realms, there can be - as many `external` realms as one would like - each with a unique name and different settings. (Currently - the only `external` realm types that exist are `ldap`, `active_directory`, and `pki`). - - -include::setting-up-authentication/enabling-anonymous-access.asciidoc[] - -include::setting-up-authentication/configuring-esusers-realm.asciidoc[] - -include::setting-up-authentication/configuring-ldap-realm.asciidoc[] - -include::setting-up-authentication/configuring-active-directory-realm.asciidoc[] - -include::setting-up-authentication/configuring-pki-realm.asciidoc[] - -include::setting-up-authentication/integrating-other-auth-systems.asciidoc[] - -include::setting-up-authentication/controlling-user-cache.asciidoc[] - diff --git a/shield/docs/public/setting-up-authentication/configuring-active-directory-realm.asciidoc b/shield/docs/public/setting-up-authentication/configuring-active-directory-realm.asciidoc deleted file mode 100644 index ee6d3db2348..00000000000 --- a/shield/docs/public/setting-up-authentication/configuring-active-directory-realm.asciidoc +++ /dev/null @@ -1,244 +0,0 @@ -[[active-directory]] -=== Using Active Directory to Authenticate Users - -You can configure Shield to communicate with Active Directory to authenticate users. To integrate -with Active Directory, you configure an Active Directory realm and assign Active Directory users -and groups to Shield roles in the <>. - -To protect passwords, communications between Shield and the LDAP server should be encrypted -using SSL/TLS. Clients and nodes that connect via SSL/TLS to the LDAP server need to have the -LDAP server's certificate or the server's root CA certificate installed in their keystore or -truststore. For more information about installing certificates, see <>. - -==== Configuring an Active Directory Realm - -Shield uses LDAP to communicate with Active Directory, so Active Directory realms are similar -to <>. Like LDAP directories, Active Directory stores users and groups -hierarchically. The directory's hierarchy is built from containers such as the _organizational -unit_ (`ou`), _organization_ (`o`), and _domain controller_ (`dc`). - -The path to a entry is a _Distinguished Name_ (DN) that uniquely identifies a user or group. User -and group names typically have attributes such as a _common name_ (`cn`) or _unique ID_ (`uid`). -A DN is specified as a string, for example `"cn=admin,dc=example,dc=com"`. White space is ignored. -Shield only supports Active Directory security groups. You cannot map distribution groups -to roles. - -NOTE: When you use Active Directory for authentication, the username entered by the user is expected - to match the `sAMAccountName` or `userPrincipalName`, not the common name. - -To configure an Active Directory realm: - -. Add a realm configuration of type `active_directory` to `elasticsearch.yml` in the -`shield.authc.realms` namespace. At a minimum, you must set the realm -`type` to `active_directory` and specify the Active Directory `domain_name`. To use SSL/TLS to -encrypt communications with the Active Directory server, you must also set the `url` attribute and -specify the LDAPS protocol and secure port number. If you are configuring multiple realms, you -should also explicitly set the `order` attribute to control the order in which the realms are -consulted during authentication. See <> -for all of the options you can set for an Active Directory realm. -+ -NOTE: Binding to Active Directory fails if the domain name is not mapped in DNS. If DNS is not - being provided by a Windows DNS server, add a mapping for the domain in the local - `/etc/hosts` file. -+ -For example, the following realm configuration configures Shield to connect to -`ldaps://example.com:636` to authenticate users through Active Directory. -+ -[source, yaml] ------------------------------------------------------------- -shield: - authc: - realms: - active_directory: - type: active_directory - order: 0 <1> - domain_name: ad.example.com - url: ldaps://ad.example.com:636 <2> - unmapped_groups_as_roles: true <3> - ... ------------------------------------------------------------- -<1> The realm order controls the order in which the configured realms are checked when -authenticating a user. -<2> If you don't specify the URL, it defaults to `ldap::389`. -<3> When this option is enabled, Shield automatically maps Active Directory groups to -roles of the same name defined in `roles.yml`. - -. Restart Elasticsearch. - -[[ad-settings]] -===== Active Directory Realm Settings - -|======================= -| Setting | Required | Description -| `type` | yes | Indicates the realm type. Must be set to - `active_directory`. -| `order` | no | Indicates the priority of this realm within the realm - chain. Realms with a lower order are consulted first. - Although not required, we recommend explicitly - setting this value when you configure multiple realms. - Defaults to `Integer.MAX_VALUE`. -| `enabled` | no | Indicates whether this realm is enabled or disabled. - Enables you to disable a realm without removing its - configuration. Defaults to `true`. -| `domain_name` | yes | Specifies the domain name of the Active Directory. Shield - uses the domain name to derive the LDAP URL and - `user_search_dn` if those fields are not specified. -| `url` | no | Specifies an LDAP URL of the form - `ldap[s]://:`. Shield attempts to - authenticate against this URL. If the URL is not - specified, Shield derives it from the `domain_name`, - assuming an unencrypted connection to port 389. For - example, `ldap://:389`. You must specify the - URL to connect using SSL/TLS or connect to a custom port. -| `user_search.base_dn` | no | Specifies the context to search for the user. Defaults to - the root of the Active Directory domain. -| `user_search.scope` | no | Specifies whether the user search should be `sub_tree` - (default), `one_level`, or `base`. `sub_tree` searches - all objects contained under `base_dn`. `one_level` only - searches users directly contained within the `base_dn`. - `base` specifies that the `base_dn` is a user object and - that it is the only user considered. -| `user_search.filter` | no | Specifies a filter to use to lookup a user given a - username. The default filter looks up `user` objects - with either `sAMAccountName` or `userPrincipalName`. If - specified, this must be a proper LDAP user search filter, - for example `(&(objectClass=user)(sAMAccountName={0}))`. - For more information, see https://msdn.microsoft.com/en-us/library/aa746475(v=vs.85).aspx[Search Filter Syntax]. -| `group_search.base_dn` | no | Specifies the context to search for groups in which the - user has membership. Defaults to the root of the Active - Directory domain. -| `group_search.scope` | no | Specifies whether the group search should be `sub_tree` - (default), `one_level` or `base`. `sub_tree` searches - all objects contained under `base_dn`. `one_level` - searches for groups directly contained within the - `base_dn`. `base` specifies that the `base_dn` is a - group object and that it is the only group considered. -| `unmapped_groups_as_roles` | no | Specifies whether the names of any unmapped LDAP groups - should be used as role names and assigned to the user. - Defaults to `false`. -| `files.role_mapping` | no | Specifies the path and file name of the - <>. - Defaults to `CONF_DIR/shield/users/role_mapping.yml`, - where `CONF_DIR` is `ES_HOME/config` (zip/tar installations) - or `/etc/elasticsearch` (package installations). -| `follow_referrals` | no | Specifies whether Shield should follow referrals returned - by the LDAP server. Referrals are URLs returned by the - server that are to be used to continue the LDAP operation - (such as `search`). Defaults to `true`. -| `hostname_verification` | no | Specifies whether hostname verification is performed when - connecting to an LDAP server. When `true`, the hostname - or IP address used in the `url` must match one of the - names in the certificate or the connection will not be - allowed. Due to its potential security impact, - `hostname_verification` is not exposed via the - {ref}/cluster-nodes-info.html#cluster-nodes-info[nodes info API]. - Defaults to `true`. -| `cache.ttl` | no | Specifies the time-to-live for cached user entries. A - user's credentials are cached for this period of time. - Specify the time period using the standard Elasticsearch - {ref}/common-options.html#time-units[time units]. - Defaults to `20m`. -| `cache.max_users` | no | Specifies the maximum number of user entries that can be - stored in the cache at one time. Defaults to 100,000. -| `cache.hash_algo` | no | Specifies the hashing algorithm that is used for the - cached user credentials. See <> for the possible values. - (Expert Setting) -|======================= - -[[assigning-roles-ad]] -==== Assigning Active Directory Users and Groups to Roles - -To configure privileges for Active Directory users and groups, you assign them to roles in -the role mapping file stored on each node. You specify users and groups using their distinguished -names. For example, the following mapping configuration assigns the Active Directory `admins` -group both the `monitoring` and `user` roles, and assigns the `user` role to the `users` -group and `John Doe`. - -[source, yaml] ------------------------------------------------------------- -monitoring: <1> - - "cn=admins,dc=example,dc=com" <2> -user: - - "cn=users,dc=example,dc=com" <3> - - "cn=admins,dc=example,dc=com" - - "cn=John Doe,cn=contractors,dc=example,dc=com" <4> ------------------------------------------------------------- -<1> The name of a role defined in <>. -<2> The distinguished name of the `admins` group. -<3> The distinguished name of the `users` group. -<4> The distinguished name of the user `John Doe`. - -For more information, see <>. - -[[active-directory-ssl]] -==== Encrypting Communications Between Shield and Active Directory with SSL/TLS - -You should encrypt communications between Shield and Active Directory to protect the user -credentials that are sent to Active Directory for authentication. Connecting via SSL/TLS -ensures that the identity of the Active Directory server is authenticated before Shield -transmits the user credentials, and the user names and passwords are encrypted in transit. - -To encrypt communications between Shield and Active Directory: - -. Configure each node to trust certificates signed by the CA that signed your Active Directory server -certificates. For example, the following command imports `cacert.pem` -into node01's keystore. (For information about using truststores, see <>.) -+ -[source,shell] --------------------------------------------------- -cd CONFIG_DIR/shield -keytool -importcert -keystore node01.jks -file cacert.pem -alias ad_ca --------------------------------------------------- -+ -The CA cert must be a PEM encoded certificate. -+ -[NOTE] -=============================== -You can also import the individual server certificates rather than the CA certificate, but -this is only recommended if you have a single Active Directory server. -You can fetch the Active Directory server certificate with `openssl`. -For example, the following command gets the certificate for `ad.example.com` and stores it locally -in `ldap.crt`. - -[source, shell] ----------------------------------------------------------------------------------------------- -echo | openssl s_client -connect ad.example.com:636 2>/dev/null | openssl x509 > ldap.crt ----------------------------------------------------------------------------------------------- - -If you are using an older version of openssl you might need to use use the `-host` and -`-port` options rather than the `-connect` option. -=============================== - -. If you haven't already configured the path to the node's keystore or truststore in -`elasticsearch.yml`, set the `shield.ssl.keystore.path` or `shield.ssl.truststore.path` -attributes. For example: -+ -[source, yaml] --------------------------------------------------- -shield.ssl.keystore.path: /home/es/config/shield/node01.jks <1> -shield.ssl.keystore.password: myPass <2> -shield.ssl.keystore.key_password: myKeyPass <3> --------------------------------------------------- -<1> The full path to the node keystore file. This must be a location within the Elasticsearch -configuration directory. -<2> The password used to access the keystore. -<3> The password used to access the certificate. This is only required if you specified a separate -certificate password when generating the certificate. -+ -For more information, see <>. - -. Set the `url` attribute in the realm configuration to specify the LDAPS protocol and -the secure port number. For example, `url: ldaps://ad.example.com:636`. - -. Restart Elasticsearch to pick up the changes to `elasticsearch.yml`. - -NOTE: By default, when you configure Shield to connect to Active Directory using SSL/TLS, - Shield attempts to verify the hostname or IP address specified with the `url` attribute in - the realm configuration with the values in the certificate. If the values in the certificate - and realm configuration do not match, Shield does not allow a connection to the Active - Directory server. This is done to protect against man in the middle attacks. If necessary, - you can disable this behavior by setting the <> - property to `false`. \ No newline at end of file diff --git a/shield/docs/public/setting-up-authentication/configuring-esusers-realm.asciidoc b/shield/docs/public/setting-up-authentication/configuring-esusers-realm.asciidoc deleted file mode 100644 index b83ce7ceb45..00000000000 --- a/shield/docs/public/setting-up-authentication/configuring-esusers-realm.asciidoc +++ /dev/null @@ -1,74 +0,0 @@ -[[esusers]] -=== Using esusers to Authenticate Users - -You can manage and authenticate users with Shield's built-in system, `esusers`. -An _esusers_ realm is created by default when you install Shield. You use the -<> to add and remove users, assign user roles, -and manage user passwords. - -==== Configuring an esusers Realm - -Like other realms, you can configure options for an `esusers` realm in the -`shield.authc.realms` namespace in `elasticsearch.yml`. - -To configure an esusers realm: - -. Add a realm configuration of type `esusers` to `elasticsearch.yml` in the -`shield.authc.realms` namespace. At a minimum, you must set the realm -`type` to `esusers`. If you are configuring multiple realms, you -should also explicitly set the `order` attribute. See <> -for all of the options you can set for an `esusers` realm. -+ -For example, the following snippet shows an `esusers` realm configuration that sets the `order` to -zero so the realm is checked first: -+ -[source, yaml] ------------------------------------------------------------- -shield: - authc: - realms: - esusers1: - type: esusers - order: 0 ------------------------------------------------------------- - -. Restart Elasticsearch. - -[[esusers-settings]] -===== esusers Realm Settings - -|======================= -| Setting | Required | Description -| `type` | yes | Indicates the realm type. Must be set to `esusers`. -| `order` | no | Indicates the priority of this realm within the realm - chain. Realms with a lower order are consulted first. - Although not required, we recommend explicitly - setting this value when you configure multiple realms. - Defaults to `Integer.MAX_VALUE`. -| `enabled` | no | Indicates whether this realm is enabled or disabled. - Enables you to disable a realm without removing its - configuration. Defaults to `true`. -| `files.users` | no | Points to the <> - of the `users` file where the users and their passwords - are stored. By default, it is `CONFIG_DIR/shield/users`. -| `files.users_roles` | no | Points to the <> - of the `users_roles` file where the users and their - roles are stored. Defaults to - `CONFIG_DIR/shield/users_roles`. -| `cache.ttl` | no | Specifies the time-to-live for cached user entries. A - user's credentials are cached for this period of time. - Specify the time period using the standard Elasticsearch - {ref}/common-options.html#time-units[time units]. - Defaults to `20m`. -| `cache.max_users` | no | Specifies the maximum number of user entries that can be - stored in the cache at one time. Defaults to 100,000. -| `cache.hash_algo` | no | Specifies the hashing algorithm that is used for the - cached user credentials. See <> for the possible values. - (Expert Setting) -|======================= - -NOTE: When no realms are explicitly configured in `elasticsearch.yml`, a default realm chain is - created that holds a single `esusers` realm. If you wish to only work with `esusers` realm - and you're satisfied with the default files paths, there is no real need to add the above - configuration. diff --git a/shield/docs/public/setting-up-authentication/configuring-ldap-realm.asciidoc b/shield/docs/public/setting-up-authentication/configuring-ldap-realm.asciidoc deleted file mode 100644 index 4e813bd4deb..00000000000 --- a/shield/docs/public/setting-up-authentication/configuring-ldap-realm.asciidoc +++ /dev/null @@ -1,341 +0,0 @@ -[[ldap]] -=== Using LDAP to Authenticate Users - -You can configure Shield to communicate with a Lightweight Directory Access Protocol -(LDAP) directory to authenticate users. To integrate with LDAP, you configure an LDAP realm and -assign LDAP groups to Shield roles in the <>. - -To protect passwords, communications between Shield and the LDAP server should be encrypted -using SSL/TLS. Clients and nodes that connect via SSL/TLS to the LDAP server need to have the -LDAP server's certificate or the server's root CA certificate installed in their keystore or -truststore. For more information about installing certificates, see <>. - -==== Configuring an LDAP Realm - -LDAP stores users and groups hierarchically, similar to the way folders are grouped in a file -system. An LDAP directory's hierarchy is built from containers such as the - _organizational unit_ (`ou`), _organization_ (`o`), and _domain controller_ (`dc`). - -The path to a entry is a _Distinguished Name_ (DN) that uniquely identifies a user or group. -User and group names typically have attributes such as a _common name_ (`cn`) or _unique ID_ (`uid`). -A DN is specified as a string, for example `"cn=admin,dc=example,dc=com"`. White space is ignored. - -The LDAP realm supports two modes of operation, a user search mode -and a mode with specific templates for user DNs. See <> -for all of the options you can set for an LDAP realm. - -[[ldap-user-search]] -===== Configuring an LDAP Realm with User Search added[1.1.0] -LDAP user search is the most common mode of operation. In this mode, a specific user with -permission to search the LDAP directory is used to search for the user DN based on the username -and an LDAP attribute. - -To configure an LDAP Realm with User Search: - -. Add a realm configuration of type `ldap` to `elasticsearch.yml` in the -`shield.authc.realms` namespace. At a minimum, you must set the realm -`type` to `ldap`, specify the `url` of the LDAP server, and specify the container DN to search for -users with the `user_search.base_dn` option. If you are configuring multiple realms, you -should also explicitly set the `order` attribute to control the order in which the realms are -consulted during authentication. See <> -for all of the options you can set for an LDAP realm. -+ -For example, the following snippet shows an LDAP realm configured with a user search: -+ -[source, yaml] ------------------------------------------------------------- -shield: - authc: - realms: - ldap1: - type: ldap - order: 0 - url: "ldaps://ldap.example.com:636" - bind_dn: "cn=ldapuser, ou=users, o=services, dc=example, dc=com" - bind_password: changeme - user_search: - base_dn: "dc=example,dc=com" - attribute: cn - group_search: - base_dn: "dc=example,dc=com" - files: - role_mapping: "/mnt/elasticsearch/group_to_role_mapping.yml" - unmapped_groups_as_roles: false ------------------------------------------------------------- - -. Restart Elasticsearch - -===== Configuring an LDAP Realm with User DN Templates -If your LDAP environment uses a few specific standard naming conditions for users, you can use -User DN templates to configure the realm. The advantage of this method is that a search does not -have to be performed to find the user DN. However, multiple bind operations might be needed to find -the correct user DN. - -To configure an LDAP Realm with User Search: - -. Add a realm configuration of type `ldap` to `elasticsearch.yml` in the -`shield.authc.realms` namespace. At a minimum, you must set the realm -`type` to `ldap`, specify the `url` of the LDAP server, and specify at least one template with -the `user_dn_templates` option. If you are configuring multiple realms, you -should also explicitly set the `order` attribute to control the order in which the realms are -consulted during authentication. See <> -for all of the options you can set for an LDAP realm. -+ -For example, the following snippet shows an LDAP realm configured with User DN templates: -+ -[source, yaml] ------------------------------------------------------------- -shield: - authc: - realms: - ldap1: - type: ldap - order: 0 - url: "ldaps://ldap.example.com:636" - user_dn_templates: - - "cn={0}, ou=users, o=marketing, dc=example, dc=com" - - "cn={0}, ou=users, o=engineering, dc=example, dc=com" - group_search: - base_dn: "dc=example,dc=com" - files: - role_mapping: "/mnt/elasticsearch/group_to_role_mapping.yml" - unmapped_groups_as_roles: false ------------------------------------------------------------- - -[[ldap-settings]] -===== LDAP Realm Settings - -.Common LDAP Realm Settings -|======================= -| Setting | Required | Description -| `type` | yes | Indicates the realm type. Must be set to `ldap`. -| `order` | no | Indicates the priority of this realm within the realm - chain. Realms with a lower order are consulted first. - Although not required, we recommend explicitly - setting this value when you configure multiple realms. - Defaults to `Integer.MAX_VALUE`. -| `enabled` | no | Indicates whether this realm is enabled or disabled. - Enables you to disable a realm without removing its - configuration. Defaults to `true`. -| `url` | yes | Specifies an LDAP URL of the form of - `ldap[s]://:`. Shield attempts to - authenticate against this URL. -| `user_group_attribute` | no | Specifies the attribute to examine on the user for group - membership. The default is `memberOf`. This setting will - be ignored if any `group_search` settings are specified. -| `group_search.base_dn` | no | Specifies a container DN to search for groups in which - the user has membership. When this element is absent, - Shield searches for the attribute specified by - `user_group_attribute` set on the user to determine - group membership. -| `group_search.scope` | no | Specifies whether the group search should be - `sub_tree`, `one_level` or `base`. `one_level` only - searches objects directly contained within the - `base_dn`. The default `sub_tree` searches all objects - contained under `base_dn`. `base` specifies that the - `base_dn` is a group object, and that it is the only - group considered. -| `group_search.filter` | no | Specifies a filter to use to lookup a group. If not - set, the realm searches for `group`, - `groupOfNames`, or `groupOfUniqueNames`, with the - attributes `member` or `memberOf`. Any instance of - `{0}` in the filter is replaced by the user - attribute defined in `group_search.user_attribute` -| `group_search.user_attribute` | no | Specifies the user attribute that is fetched and - provided as a parameter to the filter. If not set, - the user DN is passed to the filter. -| `unmapped_groups_as_roles` | no | Specifies whether the names of any unmapped LDAP groups - should be used as role names and assigned to the user. - Defaults to `false`. -| `connect_timeout` | no | Specifies the timeout period for establishing an - LDAP connection. An `s` at the end indicates seconds, - `ms` indicates milliseconds. Defaults to `5s` (5 seconds). -| `read_timeout` | no | The timeout period for an LDAP operation. An `s` at - the end indicates seconds, `ms` indicates - milliseconds. Defaults to `5s` (5 seconds). -| `files.role_mapping` | no | Specifies the path and file name for the - <>. Defaults to - `ES_HOME/config/shield/role_mapping.yml`. -| `follow_referrals` | no | Specifies whether Shield should - follow referrals returned by the LDAP server. Referrals - are URLs returned by the server that are to be used to - continue the LDAP operation (e.g. search). Defaults to - `true`. -| `hostname_verification` | no | Specifies whether hostname verification is performed when - connecting to an LDAP server. When `true`, the hostname - or IP address used in the `url` must match one of the - names in the certificate or the connection will not be - allowed. Due to its potential security impact, - `hostname_verification` is not exposed via the - {ref}/cluster-nodes-info.html#cluster-nodes-info[nodes - info API]. Defaults to `true`. -| `cache.ttl` | no | Specifies the time-to-live for cached user entries. A - user's credentials are cached for this period of time. - Specify the time period using the standard Elasticsearch - {ref}/common-options.html#time-units[time units]. - Defaults to `20m`. -| `cache.max_users` | no | Specifies the maximum number of user entries that can be - stored in the cache at one time. Defaults to 100,000. -| `cache.hash_algo` | no | Specifies the hashing algorithm that is used for the - cached user credentials. See <> for the possible values. - (Expert Setting) -|======================= - -.User Template LDAP Realm Settings -|======================= -| Setting | Required | Description -| `user_dn_templates` | yes | Specifies the DN template that replaces the user name - with the string `{0}`. This element is multivalued, - allowing for multiple user contexts. -|======================= - -.User Search LDAP Realm Settings added[1.1.0] -|======================= -| Setting | Required | Description -| `bind_dn` | no | The DN of the user that is used to - bind to the LDAP and perform searches. If - not specified, an anonymous bind - is attempted. Due to its potential security - impact, `hostname_verification` is not - exposed via the - {ref}/cluster-nodes-info.html#cluster-nodes-info[ - nodes info API]. -| `bind_password` | no | The password for the user that is used - to bind to the LDAP. Due to its potential - security impact, `hostname_verification` is - not exposed via the - {ref}/cluster-nodes-info.html#cluster-nodes-info[ - nodes info API]. -| `user_search.base_dn` | yes | Specifies a container DN to search for users. -| `user_search.scope` | no | The scope of the user search. Valid values - are `sub_tree`, `one_level` or `base`. - `one_level` only searches objects directly - contained within the `base_dn`. - `sub_tree` searches all objects contained - under `base_dn`. `base` specifies that the - `base_dn` is the user object, and that it is - the only user considered. Defaults to - `sub_tree`. -| `user_search.attribute` | no | Specifies the attribute to match with the - username presented to Shield. Defaults to - `uid`. -| `user_search.pool.size` | no | Specifies the maximum number of connections - to the LDAP server to allow in the connection - pool. Defaults to `20`. -| `user_search.pool.initial_size` | no | The initial number of connections to create - to the LDAP server on startup. Defaults to `5`. -| `user_search.pool.health_check.enabled` | no | Enables or disables a health check on - LDAP connections in the connection pool. - Connections are checked in the background at - the specified interval. Defaults to `true`. -| `user_search.pool.health_check.dn` | no | Specifies the distinguished name to retrieve - as part of the health check. Defaults to the - value of `bind_dn`. If `bind_dn` is not - configured, you must specify a value. -| `user_search.pool.health_check.interval` | no | How often to perform background checks - of connections in the pool. Defaults to - `60s`. -|======================= - -NOTE: If any settings starting with `user_search` are specified, the `user_dn_templates` - the settings are ignored. - -[[assigning-roles-ldap]] -==== Assigning LDAP Groups to Roles - -To configure privileges for LDAP users, you assign LDAP groups to roles in -the role mapping file stored on each node. When a user authenticates with LDAP, -the privileges for that user are the union of all privileges defined by -the roles assigned to the set of groups that the user belongs to. - -You specify groups using their distinguished names. For example, the following mapping -configuration assigns the LDAP `admins` group both the `monitoring` and `user` roles, and -assigns the `user` role to the `users` group. - -[source, yaml] ------------------------------------------------------------- -monitoring: <1> - - "cn=admins,dc=example,dc=com" <2> -user: - - "cn=users,dc=example,dc=com" <3> - - "cn=admins,dc=example,dc=com" ------------------------------------------------------------- -<1> The name of a role defined in <>. -<2> The distinguished name of the `admins` group. -<3> The distinguished name of the `users` group. - -For more information, see <>. - -==== Encrypting Communications Between Shield and LDAP with SSL/TLS - -You should encrypt communications between Shield and your LDAP server to protect the user -credentials that are sent to for authentication. Connecting via SSL/TLS -ensures that the identity of the LDAP server is authenticated before Shield -transmits the user credentials, and the user names and passwords are encrypted in transit. - -To encrypt communications between Shield and your LDAP server: - -. Configure each node to trust certificates signed by the CA that signed your LDAP server -certificates. For example, the following command imports `cacert.pem` -into node01's keystore. (For information about using truststores, see <>.) -+ -[source,shell] --------------------------------------------------- -cd CONFIG_DIR/shield -keytool -importcert -keystore node01.jks -file cacert.pem -alias ldap_ca --------------------------------------------------- -+ -The CA cert must be a PEM encoded certificate. -+ -[NOTE] -=============================== -You can also import the individual server certificates rather than the CA certificate, but -this is only recommended if you have a single LDAP server. -You can fetch the LDAP server certificate with `openssl`. -For example, the following command gets the certificate for `ldap.example.com` and stores it locally -in `ldap.crt`. - -[source, shell] ----------------------------------------------------------------------------------------------- -echo | openssl s_client -connect ldap.example.com:636 2>/dev/null | openssl x509 > ldap.crt ----------------------------------------------------------------------------------------------- - -If you are using an older version of openssl you might need to use use the `-host` and -`-port` options rather than the `-connect` option. -=============================== - -. If you haven't already configured the path to the node's keystore or truststore in -`elasticsearch.yml`, set the `shield.ssl.keystore.path` or `shield.ssl.truststore.path` -attributes. For example: -+ -[source, yaml] --------------------------------------------------- -shield.ssl.keystore.path: /home/es/config/shield/node01.jks <1> -shield.ssl.keystore.password: myPass <2> -shield.ssl.keystore.key_password: myKeyPass <3> --------------------------------------------------- -<1> The full path to the node keystore file. This must be a location within the Elasticsearch -configuration directory. -<2> The password used to access the keystore. -<3> The password used to access the certificate. This is only required if you specified a separate -certificate password when generating the certificate. -+ -For more information, see <>. - -. Set the `url` attribute in the realm configuration to specify the LDAPS protocol and -the secure port number. For example, `url: ldaps://ldap.example.com:636`. - -. Restart Elasticsearch to pick up the changes to `elasticsearch.yml`. - - -NOTE: By default, when you configure Shield to connect to an LDAP server using SSL/TLS, - Shield attempts to verify the hostname or IP address specified with the `url` attribute in - the realm configuration with the values in the certificate. If the values in the certificate - and realm configuration do not match, Shield does not allow a connection to the LDAP server. - This is done to protect against man in the middle attacks. If necessary, - you can disable this behavior by setting the <> - property to `false`. `hostname_verification` is considered to be a sensitive setting and - is not exposed via {ref}/cluster-nodes-info.html#cluster-nodes-info[nodes info API]. \ No newline at end of file diff --git a/shield/docs/public/setting-up-authentication/configuring-pki-realm.asciidoc b/shield/docs/public/setting-up-authentication/configuring-pki-realm.asciidoc deleted file mode 100644 index 16574d85241..00000000000 --- a/shield/docs/public/setting-up-authentication/configuring-pki-realm.asciidoc +++ /dev/null @@ -1,123 +0,0 @@ -[[pki]] -=== Using PKI to Authenticate Users added[1.3.0] - -You can configure Shield to use Public Key Infrastructure (PKI) certificates to authenticate users. -This requires clients to present X.509 certificates. To use PKI, you configure a PKI realm, -enable client authentication on the desired network layers (transport or http), -and map the DNs from the user certificates to Shield roles in the <>. - -You can use a combination of PKI encryption and username and password authentication. For example, -you can enable SSL/TLS on the transport layer and define a PKI realm to require transport clients to -authenticate with X.509 certificates, while still authenticating HTTP traffic using usernames and -passwords. You can also set `shield.transport.ssl.client.auth` to `optional` -to allow clients without certificates to authenticate with other credentials. - -IMPORTANT: You must enable SSL/TLS to use PKI. For more information, see -<>. - -==== PKI Realm Configuration - -Like realms, you configure options for a `pki` realm in the `shield.authc.realms` namespace in -`elasticsearch.yml`. - -To configure PKI realm: - -. Add a realm configuration of type `pki` to `elasticsearch.yml` in the -`shield.authc.realms` namespace. At a minimum, you must set the realm -`type` to `pki`. If you are configuring multiple realms, you -should also explicitly set the `order` attribute. See <> -for all of the options you can set for an `pki` realm. -+ -For example, the following snippet shows the most basic PKI realm configuration: -+ -[source, yaml] ------------------------------------------------------------- -shield: - authc: - realms: - pki1: - type: pki ------------------------------------------------------------- -+ -With this configuration, any certificate trusted by the SSL/TLS layer is accepted for -authentication. The username is the common name (CN) extracted from the DN of the certificate. -+ -If you want to use something other than the CN of the DN as the username, you can specify -a regex to extract the desired username. For example, the regex in the following configuration -extracts the email address from the DN: -+ -[source, yaml] ------------------------------------------------------------- -shield: - authc: - realms: - pki1: - type: pki - username_pattern: "EMAILADDRESS=(.*?)(?:,|$)" ------------------------------------------------------------- -+ -You can also specify which truststore to use for authentication. This is useful -when the SSL/TLS layer trusts clients with certificates that are signed by a different -CA than the one that signs your users' certificates. To specify the location of the truststore, -specify the `truststore.path` option: -+ -[source, yaml] ------------------------------------------------------------- -shield: - authc: - realms: - pki1: - type: pki - truststore: - path: "/path/to/pki_truststore.jks" - password: "changeme" ------------------------------------------------------------- - -. Restart Elasticsearch. - -[[pki-settings]] -===== PKI Realm Settings - -|======================= -| Setting | Required | Description -| `type` | yes | Indicates the realm type. Must be set to `pki` -| `order` | no | Indicates the priority of this realm within the realm - chain. Realms with a lower order are consulted first. - Although not required, we recommend explicitly - setting this value when you configure multiple realms. - Defaults to `Integer.MAX_VALUE`. -| `enabled` | no | Indicates whether this realm is enabled or disabled. - Enables you to disable a realm without removing its - configuration. Defaults to `true`. -| `username_pattern` | no | Specifies the regular expression pattern used to extract - the username from the certificate DN. The first match - group is used as the username. Defaults to - `CN=(.*?)(?:,\|$)`. -| `truststore.path` | no | The path to the truststore. Defaults to the path - defined by <>. -| `truststore.password` | no | Specifies the password for the truststore. Must be - provided if `truststore.path` is set. -| `truststore.algorithm` | no | Specifies the algorithm used for the trustsore. Defaults to - `SunX509`. -| `files.role_mapping` | no | Specifies the <> - for the <>. Defaults to - `CONFIG_DIR/shield/role_mapping.yml`. -|======================= - -[[assigning-roles-pki]] -==== Assigning Roles for PKI Users - -You assign roles for PKI users in the role mapping file stored on each node. You identify a user -by the distinguished name in their certificate. For example, the following mapping -configuration assigns `John Doe` the `user` role: - -[source, yaml] ------------------------------------------------------------- -user: <1> - - "cn=John Doe,ou=example,o=com" <2> ------------------------------------------------------------- -<1> The name of a Shield role defined in the <> -<2> The distinguished name of a PKI user. - -For more information, see <>. \ No newline at end of file diff --git a/shield/docs/public/setting-up-authentication/controlling-user-cache.asciidoc b/shield/docs/public/setting-up-authentication/controlling-user-cache.asciidoc deleted file mode 100644 index b1808076d47..00000000000 --- a/shield/docs/public/setting-up-authentication/controlling-user-cache.asciidoc +++ /dev/null @@ -1,61 +0,0 @@ -[[controlling-user-cache]] -=== Controlling the User Cache - -User credentials are cached in memory on each node to avoid connecting to a remote authentication -server or hitting the disk for every incoming request. You can configure characteristics of the -user cache with the `cache.ttl`, `cache.max_users`, and ``cache.hash_algo` realm settings. - -NOTE: PKI realms do not use the user cache. - -The cached user credentials are hashed in memory. By default, Shield uses a salted `sha-256` -hash algorigthm. You can use a different algorithm by setting the `cache-hash_algo` setting -to any of the supported <>. - -[[cache-hash-algo]] -.Cache hash algorithms -|======================= -| Algorithm | Description -| `ssha256` | Uses a salted `sha-256` algorithm (default). -| `md5` | Uses `MD5` algorithm. -| `sha1` | Uses `SHA1` algorithm. -| `bcrypt` | Uses `bcrypt` algorithm with salt generated in 10 rounds. -| `bcrypt4` | Uses `bcrypt` algorithm with salt generated in 4 rounds. -| `bcrypt5` | Uses `bcrypt` algorithm with salt generated in 5 rounds. -| `bcrypt6` | Uses `bcrypt` algorithm with salt generated in 6 rounds. -| `bcrypt7` | Uses `bcrypt` algorithm with salt generated in 7 rounds. -| `bcrypt8` | Uses `bcrypt` algorithm with salt generated in 8 rounds. -| `bcrypt9` | Uses `bcrypt` algorithm with salt generated in 9 rounds. -| `sha2` | Uses `SHA2` algorithm. -| `apr1` | Uses `apr1` algorithm (md5 crypt). -| `noop`,`clear_text` | Doesn't hash the credentials and keeps it in clear text in - memory. CAUTION: keeping clear text is considered insecure - and can be compromised at the OS level (for example through - memory dumps and using `ptrace`). -|======================= - -[float] -==== Evicting Users from the Cache - -Shield exposes an API to force the eviction of cached users. For example, the following request -evicts all users from the `ad1` realm: - -[source, java] ------------------------------------------------------------- -$ curl -XPOST 'http://localhost:9200/_shield/realm/ad1/_cache/clear' ------------------------------------------------------------- - -To clear the cache for multiple realms, specify the realms as a comma-separated list: - -[source, java] ------------------------------------------------------------- -$ curl -XPOST 'http://localhost:9200/_shield/realm/ad1,ad2/_cache/clear' ------------------------------------------------------------- - -You can also evict specific users: - -[source, java] ------------------------------------------------------------- -$ curl -XPOST 'http://localhost:9200/_shield/realm/ad1/_cache/clear?usernames=rdeniro,alpacino' ------------------------------------------------------------- - - diff --git a/shield/docs/public/setting-up-authentication/enabling-anonymous-access.asciidoc b/shield/docs/public/setting-up-authentication/enabling-anonymous-access.asciidoc deleted file mode 100644 index da1f6675f36..00000000000 --- a/shield/docs/public/setting-up-authentication/enabling-anonymous-access.asciidoc +++ /dev/null @@ -1,34 +0,0 @@ -[[anonymous-access]] -=== Enabling Anonymous Access added[1.1.0] - -The authentication process can be split into two phases - token extraction and user authentication. During the first -phase (token extraction phase), the configured realms are requested to try and extract/resolve an authentication token -from the incoming request. The first realm that finds an authentication token in the request "wins", meaning, the found -authentication token will be used for authentication (moving to the second phase - user authentication - where each realm -that support this authentication token type will try to authenticate the user). - -In the event where no authentication token was resolved by any of the active realms, the incoming request is considered -to be anonymous. - -By default, anonymous requests are rejected and an authentication error is returned (status code `401`). It is possible -to change this behaviour and instruct Shield to associate an default/anonymous user with the anonymous request. This can -be done by configuring the following settings in the `elasticsearch.yml` file: - -[source,yaml] ----------------------------------------- -shield.authc: - anonymous: - username: anonymous_user <1> - roles: role1, role2 <2> - authz_exception: true <3> ----------------------------------------- -<1> The username/principal of the anonymous user. This setting is optional and will be set to `_es_anonymous_user` by default - when not configured. -<2> The roles that will be associated with the anonymous user. This setting is mandatory - without it, anonymous access - will be disabled (i.e. anonymous requests will be rejected and return an authentication error) -<3> When `true`, a HTTP 403 response will be returned when the anonymous user does not have the appropriate permissions - for the requested action. The web browser will not be prompt the user to provide credentials to access the requested - resource. When set to `false`, a HTTP 401 will be returned allowing for credentials to be provided for a user with - the appropriate permissions. If you are using anonymous access in combination with HTTP, setting this to `false` may - be necessary if your client does not support preemptive basic authentication. This setting is optional and will be - set to `true` by default. \ No newline at end of file diff --git a/shield/docs/public/setting-up-authentication/integrating-other-auth-systems.asciidoc b/shield/docs/public/setting-up-authentication/integrating-other-auth-systems.asciidoc deleted file mode 100644 index 62f305ca949..00000000000 --- a/shield/docs/public/setting-up-authentication/integrating-other-auth-systems.asciidoc +++ /dev/null @@ -1,56 +0,0 @@ -[[custom-realms]] -=== Integrating with Other Authentication Systems - -If you are using an authentication system other than LDAP, Active Directory, or PKI, you can -create a custom realm to interact with the system to authenticate users. You implement a custom -realm as an Elasticsearch plugin. - -[[implementing-custom-realm]] -==== Implementing a Custom Realm - -Sample code that illustrates the structure and implementation of a custom realm is provided in the -https://github.com/elastic/shield-custom-realm-example[shield-custom-realm-example] repository on -GitHub. You can use this code as a starting point for creating your own realm. - -To create a custom realm, you need to: - -. Extend `org.elasticsearch.shield.authc.Realm` to communicate with your authentication system -to authenticate users. -. Extend `org.elasticsearch.shield.authc.Realm.Factory` to construct your new realm type. -. Extend `org.elasticsearch.shield.authc.DefaultAuthenticationFailureHandler` to handle authentication -failures when using your custom realm. - -To package your custom realm as an Elasticsearch plugin: - -. Implement a plugin class for your realm that extends `org.elasticsearch.plugins.Plugin`. -You need to: -.. Import your realm implementation files, `org.elasticsearch.plugins.Plugin`, and -`org.elasticsearch.shield.authc.AuthenticationModule`. -.. Implement the `name` and `description` methods. -.. Implement the `onModule` method to register the custom realm with the Shield `AuthenticationModule` -and specify your authentication failure handler. -. Create a Maven configuration file (`pom.xml`) for the plugin. -. Create a https://github.com/elastic/elasticsearch/blob/master/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties[plugin-descriptor.properties] file for the plugin. - -For more information about Elasticsearch plugins, see https://www.elastic.co/guide/en/elasticsearch/plugins/2.0/index.html[Elasticsearch Plugins and Integrations]. - -[[using-custom-realm]] -==== Using a Custom Realm to Authenticate Users - -To use a custom realm: - -. Install the realm plugin on each node in the cluster. You run `bin/plugin` with the `install` -option and specify the location of the zip file that contains the plugin. For example: -+ -[source,shell] ----------------------------------------- -bin/plugin install file:////example-realm-plugin-1.0.zip ----------------------------------------- - -. Add a realm configuration of the appropriate realm type to `elasticsearch.yml` in the -`shield.authc.realms` namespace. The options you can set depend on the settings exposed by your -custom realm. At a minimum, you must set the realm `type` to the type defined in the plugin -implementation. If you are configuring multiple realms, you should also explicitly set the -`order` attribute to control the order in which the realms are consulted during authentication. - -. Restart Elasticsearch. \ No newline at end of file diff --git a/shield/docs/public/setting-up-certificate-authority.asciidoc b/shield/docs/public/setting-up-certificate-authority.asciidoc deleted file mode 100644 index 6e2bc782a3c..00000000000 --- a/shield/docs/public/setting-up-certificate-authority.asciidoc +++ /dev/null @@ -1,190 +0,0 @@ -[[certificate-authority]] -== Setting Up a Certificate Authority - -You can set up your own Certificate Authority (CA) to sign node certificates. Using a CA -makes it easier to manage trust within your cluster than using self-signed certificates. Each node -only needs to trust the CA, rather that trusting all of the other nodes' certificates individually. When nodes are added to the cluster, as long as their certificates are signed by a trusted CA, the new nodes are trusted automatically. In contrast, if you use self-signed certificates, you would have to manually configure each node to trust the new node and restart Elasticsearch. - -This topic shows how you can use https://www.openssl.org/[OpenSSL] to create a self-signed CA -certificate and sign CSRs. While it demonstrates how you can set up a CA, it does not necessarily -address your organization's particular security requirements. Before moving to production, you -should consult your organization's security experts to discuss the security requirements for your -use case. - -IMPORTANT: Because a Certificate Authority is a central point of trust, the private keys to the - CA must be protected from compromise. - -To set up a CA: - -. Create the directory structure where the CA configuration and certificates will be stored. You -need to create a `ca` directory and three subdirectories: `private`, `certs`, and `conf`. -+ -[source,shell] --------------------------------------------------- -mkdir -p ca/private ca/certs ca/conf --------------------------------------------------- - -. Populate two required files, `serial` and `index.txt`. -+ -[source,shell] --------------------------------------------------- -cd ca -echo '01' > serial -touch index.txt --------------------------------------------------- - -. Create a CA configuration template and store it in `conf/caconfig.cnf`. You use the -configuration template to set options for the CA that cannot be passed in from the -command line. The following template defines a basic CA configuration you -can use as a starting point. -+ -[source,shell] -------------------------------------------------------------------------------------- -#.................................. -[ ca ] -default_ca = CA_default -[ CA_default ] -copy_extensions = copy <1> -dir = /PATH/TO/YOUR/DIR/ca <2> -serial = $dir/serial -database = $dir/index.txt -new_certs_dir = $dir/certs -certificate = $dir/certs/cacert.pem -private_key = $dir/private/cakey.pem -default_days = 712 <3> -default_md = sha256 -preserve = no -email_in_dn = no -x509_extensions = v3_ca -name_opt = ca_default -cert_opt = ca_default -policy = policy_anything -[ policy_anything ] -countryName = optional -stateOrProvinceName = optional -localityName = optional -organizationName = optional -organizationalUnitName = optional -commonName = supplied -emailAddress = optional -[ req ] -default_bits = 2048 # Size of keys -default_keyfile = key.pem # name of generated keys -default_md = sha256 # message digest algorithm -string_mask = nombstr # permitted characters -distinguished_name = req_distinguished_name -req_extensions = v3_req -[ req_distinguished_name ] -# Variable name Prompt string -#------------------------- ---------------------------------- -0.organizationName = Organization Name (company) -organizationalUnitName = Organizational Unit Name (department, division) -emailAddress = Email Address -emailAddress_max = 40 -localityName = Locality Name (city, district) -stateOrProvinceName = State or Province Name (full name) -countryName = Country Name (2 letter code) -countryName_min = 2 -countryName_max = 2 -commonName = Common Name (hostname, IP, or your name) -commonName_max = 64 -# Default values for the above, for consistency and less typing. -# Variable name Value -#------------------------ ------------------------------ -0.organizationName_default = Elasticsearch Test Org <4> -localityName_default = Amsterdam -stateOrProvinceName_default = Amsterdam -countryName_default = NL -emailAddress_default = cacerttest@YOUR.COMPANY.TLD -[ v3_ca ] -basicConstraints = CA:TRUE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always,issuer:always -[ v3_req ] -basicConstraints = CA:FALSE -subjectKeyIdentifier = hash ---------------------------------------------------------------------------------------- -<1> Copy extensions: Copies all X509 V3 extensions from a Certificate Signing Request into the -signed certificate. With the value set to `copy`, you need to ensure the extensions and their -values are valid for the certificate being requested prior to signing the certificate. -<2> CA directory: Add the full path to this newly created CA. -<3> Certificate validity period: The default number of days that a certificate signed by this -CA is valid for. Note that certificates signed by a CA must expire before the CA certificate -expires. -<4> Certificate defaults: The `OrganizationName`, `localityName`, `stateOrProvinceName`, -`countryName`, and `emailAddress` fields specify the default Distinguished Name information. - -. Generate a self-signed CA certificate to establish your CA as an authority. For example, the following command generates a key and certificate using the `caconfig.cnf` template. You specify -where you want to store the CA's private key and certificate with the `-keyout` and `-out` options, and specify how long the certificate is valid with the `-days` option. -+ -[source,shell] ------------------------------------------------------------------------------- -openssl req -new -x509 -extensions v3_ca -keyout private/cakey.pem -out certs/cacert.pem -days 1460 -config conf/caconfig.cnf ------------------------------------------------------------------------------- -+ -NOTE: When the CA certificate expires, trust in the CA is revoked and you need to generate a new CA certificate and re-sign your node certificates. -+ -When you run `openssl` to generate a CA certificate, you are prompted to enter a PEM passphrase to encrypt the CA's private key and can override the default Distinguished Name information. -+ -WARNING: You cannot recover the CA without the PEM passphrase. -+ -The following example shows what the interaction looks like: -+ -[source,shell] ---------------------------------------------------------------------------------------------------- -openssl req -new -x509 -extensions v3_ca -keyout private/cakey.pem -out certs/cacert.pem -days 1460 -config conf/caconfig.cnf -Generating a 2048 bit RSA private key -.....................++++++ -.......++++++ -writing new private key to 'private/cakey.pem' -Enter PEM pass phrase: -Verifying - Enter PEM pass phrase: -#----- -You are about to be asked to enter information that will be incorporated -into your certificate request. -What you are about to enter is what is called a Distinguished Name or a DN. -There are quite a few fields but you can leave some blank -For some fields there will be a default value, -If you enter '.', the field will be left blank. -#----- -Organization Name (company) [Elasticsearch Test Org]: -Organizational Unit Name (department, division) []:. -Email Address [cacerttest@YOUR.COMPANY.TLD]:. -Locality Name (city, district) [Amsterdam]:. -State or Province Name (full name) [Amsterdam]:. -Country Name (2 letter code) [NL]:. -Common Name (hostname, IP, or your name) []:Elasticsearch Test CA ---------------------------------------------------------------------------------------------------- - -Once you've generated a certificate for your CA, you can use it to enable trust between your nodes. -You sign each node's certificate using your CA, and install the CA certificate and signed node -certificate in each node's keystore or truststore. For more information, see -<> and <>. - -[float] -[[sign-csr]] -=== Signing CSRs - -You sign a node's certificate to vouch for that node's identity. If a node trusts your CA -certificate, it automatically trusts any certificates you sign. - -To sign a node's certificate: - -. <> from the node. - -. Use your CA to sign the CSR using `openssl`. For example, the following command signs -the `node01.csr` and saves the signed certificate in `node01-signed.crt`. You must specify -your CA's configuration file with the `-config` option. -+ -[source,shell] ------------------------------------------------------------------------------ -openssl ca -in node01.csr -notext -out node01-signed.crt -config conf/caconfig.cnf -extensions v3_req ------------------------------------------------------------------------------ -+ -The signed certificate contains the node's original unsigned certificate, your CA certificate, and -a signature. - -Once you've signed the CSR, you need to <> in the node's keystore. - - diff --git a/shield/docs/public/setting-up-field-and-document-level-security.asciidoc b/shield/docs/public/setting-up-field-and-document-level-security.asciidoc deleted file mode 100644 index 873693efb1c..00000000000 --- a/shield/docs/public/setting-up-field-and-document-level-security.asciidoc +++ /dev/null @@ -1,162 +0,0 @@ -[[setting-up-field-and-document-level-security]] -=== Setting Up Field and Document Level Security - -You can control access to data within an index by adding field and document level security permissions to a role. -Field level security permissions restrict access to particular fields within a document. -Document level security permissions restrict access to particular documents within an index. - -Field and document level permissions are specified separately, but a role can define both field and document level permissions. -Field and document level security permissions can be configured on a per-index basis. - -IMPORTANT: Document and Field Level Security is disabled by default. Set `shield.dls_fls.enabled` to `true` in `elasticsearch.yml` to enable it. You cannot submit `_bulk` update requests when document and field level security is enabled. - -==== Field Level Security - -To enable field level security, you specify the fields that each role can access in the `roles.yml` file. -You list the allowed fields with the `fields` option. Fields are associated with a particular index or index pattern and -operate in conjunction with the privileges specified for the indices. - -[source,yaml] --------------------------------------------------- -: - indices: - : - privileges: - fields: - - - - - - --------------------------------------------------- - -To allow access to the `_all` meta field, you must explicitly list it as an allowed field. Access to the following meta fields -is always allowed: _id, _type, _parent, _routing, _timestamp, _ttl, _size and _index. If you specify an empty list of fields, -only these meta fields are accessible. - -NOTE: Omitting the fields entry entirely disables field-level security. - -For example, the following `customer_care` role grants read access to six fields in any index: - -[source,yaml] --------------------------------------------------- -customer_care: - indices: - '*': - privileges: read - fields: - - issue_id - - description - - customer_handle - - customer_email - - customer_address - - customer_phone --------------------------------------------------- - -Also wildcard field expressions can be added to the `fields` options in the `roles.yml` file. For example the following -example has the same effect as the previous example: - -[source,yaml] --------------------------------------------------- -customer_care: - indices: - '*': - privileges: read - fields: - - issue_id - - description - - 'customer_*' --------------------------------------------------- - -If documents are more complex and contain json objects then the fields with dot notion should be used. - -Assume the following document: - -[source,json] --------------------------------------------------- -{ - "customer": { - "handle": "Jim", - "email": "jim@mycompany.com", - "phone": "555-555-5555" - } -} --------------------------------------------------- - -If only access to the `handle` field is allowed then the following role should be setup: - -[source,yaml] --------------------------------------------------- -my_role: - indices: - '*': - privileges: read - fields: - - customer.handle --------------------------------------------------- - -If access to the entire `customer` object is allowed then the wildcard dot notation can be used to make this easier: - -[source,yaml] --------------------------------------------------- -my_role: - indices: - '*': - privileges: read - fields: - - customer.* --------------------------------------------------- - -===== Limitations - -When field level security is enabled for an index: - -* The get, multi get, termsvector and multi termsvector APIs aren't executed in real time. The realtime option for these APIs is forcefully set to false. -* The query cache and the request cache are disabled for search requests. -* The update API is blocked. An update request needs to be executed via a role that doesn't have field level security enabled. - -[[document-level-security]] -==== Document Level Security - -Enabling document level security restricts which documents can be accessed from any document based API. -To enable document level security, you use a query to specify the documents that each role can access in the `roles.yml` file. -You specify the document query with the `query` option. The document query is associated with a particular index or index pattern and -operates in conjunction with the privileges specified for the indices. - -[source,yaml] --------------------------------------------------- -: - indices: - : - privileges: - query: - --------------------------------------------------- - -NOTE: Omitting the `query` entry entirely disables document-level security. - -The `query` should follow the same format as if a query was defined in the request body of a search request, -but here the format is YAML. Any query from the query-dsl can be defined in the `query` entry. - -For example, the following `customer_care` role grants read access to all indices, but restricts access to documents whose `department_id` equals `12`. - -[source,yaml] --------------------------------------------------- -customer_care: - indices: - '*': - privileges: read - query: - term: - department_id: 12 --------------------------------------------------- - -Alternatively the query can also be defined in JSON as a string. This makes it easier to define queries that already have -been defined in the JSON body of search request body elsewhere. - -[source,yaml] --------------------------------------------------- -customer_care: - indices: - '*': - privileges: read - query: '{"term" : {"department_id" : "12"}}'' --------------------------------------------------- diff --git a/shield/docs/public/submitting-requests-for-other-users.asciidoc b/shield/docs/public/submitting-requests-for-other-users.asciidoc deleted file mode 100644 index 1b3eda8922e..00000000000 --- a/shield/docs/public/submitting-requests-for-other-users.asciidoc +++ /dev/null @@ -1,31 +0,0 @@ -[[submitting-requests-for-other-users]] -=== Submitting Requests on Behalf of Other Users - -Shield supports a permission that enables an authenticated user to submit requests on behalf -of other users. If your application already authenticates users, you can use this -_run as_ mechanism to restrict data access according to Shield permissions without having -to re-authenticate each user through Shield. - -To run as another user, you must be able to retrieve the user from the realm you use -to authenticate. The `esusers` realm supports this out of the box. To use `run_as` with -an LDAP realm, it must be configured to enable user search. For more information, -see <>. - -To submit requests on behalf of other users, you need to have the `run_as` -permission. For example, the following `run_as_role` grants permision to submit request on -behalf of `jacknich` or `redeniro`: - -[source,yaml] ---------------------------------------------------- -run_as_role: - run_as: jacknich, rdeniro ---------------------------------------------------- - -For information about assigning roles, see <>. - -To submit a request as another user, you specify the user in the request header. For example: - -[source,console] ---------------------------------------------------- -curl -H "es-shield-runas-user: jacknich" -u es_admin -XGET 'http://localhost:9200/' ---------------------------------------------------- \ No newline at end of file diff --git a/shield/docs/public/troubleshooting.asciidoc b/shield/docs/public/troubleshooting.asciidoc deleted file mode 100644 index 2019f74c9d2..00000000000 --- a/shield/docs/public/troubleshooting.asciidoc +++ /dev/null @@ -1,218 +0,0 @@ -[[troubleshooting]] -== Troubleshooting - -[float] -=== `settings` - -Some settings are not returned via the nodes settings API:: -+ --- -This is intentional. Some of the settings are considered to be highly sensitive (e.g. all `ssl` settings, ldap `bind_dn`, -`bind_password` and `hostname_verification`). For this reason, we filter these settings and not exposing them via the -nodes info API rest endpoint. It is also possible to define additional sensitive settings that should be hidden using -the `shield.hide_settings` setting: - -[source, yaml] ------------------------------------------- -shield.hide_settings: shield.authc.realms.ldap1.url, shield.authc.realms.ad1.* ------------------------------------------- - -The snippet above will also hide the `url` settings of the `ldap1` realm and all settings of the `ad1` realm. --- - -[float] -=== `esusers` - -I configured the appropriate roles and the users, but I still get an authorization exception:: -+ --- -Verify that the role names associated with the users match the roles defined in the `roles.yml` file. You -can use the `esusers` tool to list all the users. Any unknown roles are marked with `*`. - -[source, shell] ------------------------------------------- -esusers list -rdeniro : admin -alpacino : power_user -jacknich : marvel,unknown_role* <1> ------------------------------------------- - -<1> `unknown_role` was not found in `roles.yml` --- - -ERROR: extra arguments [...] were provided:: -+ --- -This error occurs when the esusers tool is parsing the input and finds unexepected arguments. This can happen when there -are special characters used in some of the arguments. For example, on Windows systems the `,` character is considered -a parameter separator; in other words `-r role1,role2` is translated to `-r role1 role2` and the `esusers` tool only recognizes -`role1` as an expected parameter. The solution here is to quote the parameter: `-r "role1,role2"`. --- - -[[trouble-shoot-active-directory]] -[float] -=== Active Directory - -Certain users are being frequently locked out of Active Directory:: -+ --- -Check your realm configuration; realms are checked serially, one after another. If your Active Directory realm is being checked before other realms and there are usernames -that appear in both Active Directory and another realm, a valid login for one realm may be causing failed login attempts in another realm. - -For example, if `UserA` exists in both Active Directory and esusers, and the Active Directory realm is checked first and -esusers is checked second, an attempt to authenticate as `UserA` in the esusers realm would first attempt to authenticate -against Active Directory and fail, before successfully authenticating against the esusers realm. Because authentication is -verified on each request, the Active Directory realm would be checked - and fail - on each request for `UserA` in the esusers -realm. In this case, while the Shield request completed successfully, the account on Active Directory would have received -several failed login attempts, and that account may become temporarily locked out. Plan the order of your realms accordingly. - -Also note that it is not typically necessary to define multiple Active Directory realms to handle domain controller failures. When using Microsoft DNS, the DNS entry for -the domain should always point to an available domain controller. --- - -[float] -=== LDAP - -I can authenticate to LDAP, but I still get an authorization exception:: -+ --- -A number of configuration options can cause this error. - -|====================== -|_group identification_ | - -Groups are located by either an LDAP search or by the "memberOf" attribute on -the user. Also, If subtree search is turned off, it will search only one -level deep. See the <> for all the options. -There are many options here and sticking to the defaults will not work for all -scenarios. - -| _group to role mapping_| - -Either the `role_mapping.yml` file or the location for this file could be -misconfigured. See <> for more. - -|_role definition_| - -Either the `roles.yml` file or the location for this file could be -misconfigured. See <> for more. - -|====================== - -To help track down these possibilities, add `shield.authc: DEBUG` to the `logging.yml` configuration file in `CONFIG_DIR`. -A successful authentication should produce debug statements that list groups and role mappings. --- - - -[float] -=== Encryption & Certificates - -`curl` on the Mac returns a certificate verification error even when the `--cacert` option is used:: -+ --- -Apple's integration of `curl` with their keychain technology disables the `--cacert` option. -See http://curl.haxx.se/mail/archive-2013-10/0036.html for more information. - -You can use another tool, such as `wget`, to test certificates. Alternately, you can add the certificate for the -signing certificate authority MacOS system keychain, using a procedure similar to the one detailed at the -http://support.apple.com/kb/PH14003[Apple knowledge base]. Be sure to add the signing CA's certificate and not the server's certificate. --- - -[float] -==== SSLHandshakeException causing connections to fail - -A `SSLHandshakeException` will cause a connection to a node to fail and indicates that there is a configuration issue. Some of the -common exceptions are shown below with tips on how to resolve these issues. - -`java.security.cert.CertificateException: No name matching node01.example.com found`:: -+ --- -Indicates that a client connection was made to `node01.example.com` but the certificate returned did not contain the name `node01.example.com`. -In most cases, the issue can be resolved by ensuring the name is specified as a `SubjectAlternativeName` during <>. -Another scenario is when the environment does not wish to use DNS names in certificates at all. In this scenario, all settings -in `elasticsearch.yml` should only use IP addresses and the following setting needs to be set in `elasticsearch.yml`: -[source, yaml] --------------------------------------------------- -shield.ssl.hostname_verification.resolve_name: false --------------------------------------------------- --- - -`java.security.cert.CertificateException: No subject alternative names present`:: -+ --- -Indicates that a client connection was made to an IP address but the returned certificate did not contain any `SubjectAlternativeName` entries. -IP addresses are only used for hostname verification if they are specified as a `SubjectAlternativeName` during -<>. If the intent was to use IP addresses for hostname verification, then the certificate -will need to be regenerated. Also verify that `shield.ssl.hostname_verification.resolve_name: false` is *not* set in -`elasticsearch.yml`. --- - -`javax.net.ssl.SSLHandshakeException: null cert chain` and `javax.net.ssl.SSLException: Received fatal alert: bad_certificate`:: -+ --- -The `SSLHandshakeException` above indicates that a self-signed certificate was returned by the client that is not trusted -as it cannot be found in the `truststore` or `keystore`. The `SSLException` above is seen on the client side of the connection. --- - -`sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target` and `javax.net.ssl.SSLException: Received fatal alert: certificate_unknown`:: -+ --- -The `SunCertPathBuilderException` above indicates that a certificate was returned during the handshake that is not trusted. -This message is seen on the client side of the connection. The `SSLException` above is seen on the server side of the -connection. The CA certificate that signed the returned certificate was not found in the `keystore` or `truststore` and -needs to be added to trust this certificate. --- - -[float] -==== Other SSL/TLS related exceptions - -The are other exceptions related to SSL that may be seen in the logs. Below you will find some common exceptions and their -meaning. - -WARN: received plaintext http traffic on a https channel, closing connection:: -+ --- -Indicates that there was an incoming plaintext http request. This typically occurs when an external applications attempts -to make an unencrypted call to the REST interface. Please ensure that all applications are using `https` when calling the -REST interface with SSL enabled. --- - -`org.elasticsearch.common.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record:`:: -+ --- -Indicates that there was incoming plaintext traffic on an SSL connection. This typically occurs when a node is not -configured to use encrypted communication and tries to connect to nodes that are using encrypted communication. Please -verify that all nodes are using the same setting for `shield.transport.ssl`. --- - -`java.io.StreamCorruptedException: invalid internal transport message format, got`:: -+ --- -Indicates an issue with data received on the transport interface in an unknown format. This can happen when a node with -encrypted communication enabled connects to a node that has encrypted communication disabled. Please verify that all -nodes are using the same setting for `shield.transport.ssl`. --- - -`java.lang.IllegalArgumentException: empty text`:: -+ --- -The exception is typically seen when a `https` request is made to a node that is not using `https`. If `https` is desired, -please ensure the following setting is in `elasticsearch.yml`: - -[source,yaml] ----------------- -shield.http.ssl: true ----------------- --- - -ERROR: unsupported ciphers [...] were requested but cannot be used in this JVM:: -+ --- -This error occurs when a SSL/TLS cipher suite is specified that cannot supported by the JVM that Elasticsearch is running -in. Shield will try to use the specified cipher suites that are supported by this JVM. This error can occur when using -the Shield defaults as some distributions of OpenJDK do not enable the PKCS11 provider by default. In this case, we -recommend consulting your JVM documentation for details on how to enable the PKCS11 provider. - -Another common source of this error is requesting cipher suites that use encrypting with a key length greater than 128 bits -when running on an Oracle JDK. In this case, you will need to install the <>. --- \ No newline at end of file diff --git a/watcher/docs/administering-watcher.asciidoc b/watcher/docs/administering-watcher.asciidoc deleted file mode 100644 index a16a195c390..00000000000 --- a/watcher/docs/administering-watcher.asciidoc +++ /dev/null @@ -1,25 +0,0 @@ -[[administering-watcher]] -== Administering Watcher - -This section describes how to configure options for watcher, use Shield to secure access to the -Watcher APIs, get information about Watcher, and monitor watch execution. - -include::administering-watcher/configuring-email.asciidoc[] - -include::administering-watcher/configuring-hipchat.asciidoc[] - -include::administering-watcher/configuring-slack.asciidoc[] - -include::administering-watcher/integrating-with-shield.asciidoc[] - -include::administering-watcher/integrating-with-logstash.asciidoc[] - -include::administering-watcher/configuring-default-throttle-period.asciidoc[] - -include::administering-watcher/configuring-default-http-timeouts.asciidoc[] - -include::administering-watcher/configuring-default-internal-ops-timeouts.asciidoc[] - -include::administering-watcher/getting-watcher-statistics.asciidoc[] - -include::administering-watcher/monitoring-watch-execution.asciidoc[] diff --git a/watcher/docs/administering-watcher/configuring-default-http-timeouts.asciidoc b/watcher/docs/administering-watcher/configuring-default-http-timeouts.asciidoc deleted file mode 100644 index 4afbf7d8e30..00000000000 --- a/watcher/docs/administering-watcher/configuring-default-http-timeouts.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -[[configuring-default-http-timeouts]] -=== Configuring the Default HTTP Timeouts - -All HTTP requests in watcher (e.g. used by the <> and <>) -are associated with two timeouts: - -Connection Timeout :: Determines how long should the request wait for the HTTP - connection to be established before failing the request. - -Read Timeout :: Assuming the connenction was established, this timeout - determines how long should the request wait for a - response before failing the request. - -By default, both timeouts are set to 10 seconds. It is possible to change this -default using the following settings in `elasticsearch.yml`: - -[source,yaml] --------------------------------------------------- -watcher.http.default_connection_timeout: 5s -watcher.http.default_read_timeout: 20s --------------------------------------------------- \ No newline at end of file diff --git a/watcher/docs/administering-watcher/configuring-default-internal-ops-timeouts.asciidoc b/watcher/docs/administering-watcher/configuring-default-internal-ops-timeouts.asciidoc deleted file mode 100644 index 0e806378b0a..00000000000 --- a/watcher/docs/administering-watcher/configuring-default-internal-ops-timeouts.asciidoc +++ /dev/null @@ -1,22 +0,0 @@ -[[configuring-default-internal-ops-timeouts]] -=== Configuring the Default Internal Operations Timeouts - -While Watcher is active, it often accesses different indices in Elasticsearch. -These can be internal indices used for its ongoing operation (such as the `.watches` -index where all the watches are stored) or as part of a watch execution via the -<>, <> or the -<>. - -To to ensure that Watcher's workflow doesn't hang on long running search or -indexing operations, these operations time out after a set period of time. You can -change the default timeouts in `elasticsearch.yml`. The timeouts you can configure -are shown in the following table. - -[[default-internal-ops-timeouts]] -[options="header"] -|====== -| Name | Default | Description -| `watcher.internal.ops.search.default_timeout` | 30s | The default timeout for all internal search operations. -| `watcher.internal.ops.index.default_timeout` | 60s | The default timeout for all internal index operations. -| `watcher.internal.ops.bulk.default_timeout` | 120s | The default timeout for all internal bulk operations. -|====== \ No newline at end of file diff --git a/watcher/docs/administering-watcher/configuring-default-throttle-period.asciidoc b/watcher/docs/administering-watcher/configuring-default-throttle-period.asciidoc deleted file mode 100644 index 1b12012152e..00000000000 --- a/watcher/docs/administering-watcher/configuring-default-throttle-period.asciidoc +++ /dev/null @@ -1,19 +0,0 @@ -[[configuring-default-throttle-period]] -=== Configuring the Default Throttle Period - -By default, Watcher uses a default throttle period of 5 seconds. You can override this -for particular actions by setting the throttle period in the action. You can also -define a throttle period on the watch level that will serve as a default period for -all those actions that don't specify a throttle period themselves. - -To change the default throttle period for all actions that are not configured with a -throttle period neither on the action level nor the watch level, you configure the -`watcher.execution.default_throttle_period` setting in `elasticsearch.yml`. - -For example, to set the default throttle period to 15 minutes, add the following entry -to your `elasticsearch.yml` file and restart Elasticsearch: - -[source,yaml] --------------------------------------------------- -watcher.execution.default_throttle_period: 15m --------------------------------------------------- \ No newline at end of file diff --git a/watcher/docs/administering-watcher/configuring-email.asciidoc b/watcher/docs/administering-watcher/configuring-email.asciidoc deleted file mode 100644 index 50b5a9a8254..00000000000 --- a/watcher/docs/administering-watcher/configuring-email.asciidoc +++ /dev/null @@ -1,270 +0,0 @@ -[[email-services]] -=== Configuring Watcher to Send Email -You can configure Watcher to send email from any SMTP email service. Email messages can contain -basic HTML tags. You can control which tags are allowed by -<>. - -[[email-account]] -==== Configuring Email Accounts -You configure the accounts Watcher can use to send email in your `elasticsearch.yml` configuration file. -Each account configuration has a unique name and specifies all of the SMTP information needed -to send email from that account. You can also specify defaults for all emails that are sent through -the account. For example, you can set defaults for the `from` and `bcc` fields to ensure that all -emails are sent from the same address and always blind copied to the same address. - -IMPORTANT: If your email account is configured to require two step verification, - you need to generate and use a unique App Password to send email from - Watcher. Authentication will fail if you use your primary password. - -If you configure multiple email accounts, you specify which account the email should be sent -with in the <> action. If there is only one account configured, you -do not have to specify the `account` attribute in the action definition. However, if you configure -multiple accounts and omit the `account` attribute, there is no guarantee which account will be -used to send the email. - -To add an email account, set the `watcher.actions.email.service.account` property in -`elasticsearch.yml`. See <> for the -supported attributes. - -For example, the following snippet configures a single Gmail account named `work`. - -[source,yaml] --------------------------------------------------- -watcher.actions.email.service.account: - work: - profile: gmail - email_defaults: - from: 'John Doe ' - bcc: archive@host.domain - smtp: - auth: true - starttls.enable: true - host: smtp.gmail.com - port: 587 - user: - password: --------------------------------------------------- - -[[email-profile]] -The _email profile_ defines a strategy for building a MIME message. As with almost every standard -out there, different email systems interpret the MIME standard differently and have slightly -different ways of structuring MIME messages. Watcher provides three email profiles: `standard` -(default), `gmail`, and `outlook`. - -If you are using Gmail or Outlook, we recommend using the corresponding profile. Use the `standard` -profile if you are using some other email system. For more information about configuring Watcher -to work with different email systems, see: - -* <> -* <> -* <> -* <> - -[[email-account-attributes]] -.Email Account Attributes -[options="header"] -|====== -| Name | Required | Default | Description -| `profile` | no | standard | The <> to use to - build the MIME messages that are sent from - the account. Valid values: `standard` - (default), `gmail` and `outlook`. -| `email_defaults.*` | no | - | An optional set of email attributes to use - as defaults for the emails sent from the - account. See <> for the supported - attributes. for the possible email - attributes) -| `smtp.auth` | no | false | When `true`, attempt to authenticate the - user using the AUTH command. -| `smtp.host` | yes | - | The SMTP server to connect to. -| `smtp.port` | no | 25 | The SMTP server port to connect to. -| `smtp.user` | yes | - | The user name for SMTP. -| `smtp.password` | no | - | The password for the specified SMTP user. -| `smtp.starttls.enable` | no | false | When `true`, enables the use of the - `STARTTLS` command (if supported by - the server) to switch the connection to a - TLS-protected connection before issuing any - login commands. Note that an appropriate - trust store must configured so that the - client will trust the server's certificate. - Defaults to `false`. -| `smtp.*` | no | - | SMTP attributes that enable fine control - over the SMTP protocol when sending messages. - See https://javamail.java.net/nonav/docs/api/com/sun/mail/smtp/package-summary.html[com.sun.mail.smtp] - for the full list of SMTP properties you can - set. -|====== - -[[gmail]] -===== Sending Email From Gmail - -Use the following email account settings to send email from the https://mail.google.com[Gmail] -SMTP service: - -[source,yaml] --------------------------------------------------- -watcher.actions.email.service.account: - gmail_account: - profile: gmail - smtp: - auth: true - starttls.enable: true - host: smtp.gmail.com - port: 587 - user: - password: --------------------------------------------------- - -If you get an authentication error that indicates that you need to continue the -sign-in process from a web browser when Watcher attempts to send email, you need -to configure Gmail to https://support.google.com/accounts/answer/6010255?hl=en[Allow Less -Secure Apps to access your account]. - -If two-step verification is enabled for your account, you must generate and use -a unique App Password to send email from Watcher.See -https://support.google.com/accounts/answer/185833?hl=en[Sign in using App Passwords] -for more information. - -[[outlook]] -===== Sending Email from Outlook.com - -Use the following email account settings to send email action from the -https://www.outlook.com/[Outlook.com] SMTP service: - -[source,yaml] --------------------------------------------------- -watcher.actions.email.service.account: - outlook_account: - profile: outlook - smtp: - auth: true - starttls.enable: true - host: smtp-mail.outlook.com - port: 587 - user: - password: --------------------------------------------------- - -NOTE: You need to use a unique App Password if two-step verification is enabled. - See http://windows.microsoft.com/en-us/windows/app-passwords-two-step-verification[App - passwords and two-step verification] for more information. - -[[amazon-ses]] -===== Sending Email from Amazon SES (Simple Email Service) - -Use the following email account settings to send email from the -http://aws.amazon.com/ses[Amazon Simple Email Service] (SES) SMTP service: - -[source,yaml] --------------------------------------------------- -watcher.actions.email.service.account: - ses_account: - smtp: - auth: true - starttls.enable: true - starttls.required: true - host: email-smtp.us-east-1.amazonaws.com <1> - port: 587 - user: - password: --------------------------------------------------- - -<1> `smtp.host` varies depending on the region - -NOTE: You need to use your Amazon SES SMTP credentials to send email through - Amazon SES. For more information, see http://docs.aws.amazon.com/ses/latest/DeveloperGuide/smtp-credentials.html[Obtaining Your Amazon SES SMTP Credentials]. - -[[exchange]] -===== Sending Email from Microsoft Exchange - -Use the following email account settings to send email action from Microsoft Exchange: - -[source,yaml] --------------------------------------------------- -watcher.actions.email.service.account: - exchange_account: - profile: outlook - email_defaults: - from: <1> - smtp: - auth: true - starttls.enable: true - host: - port: 587 - user: <2> - password: --------------------------------------------------- - -<1> Some organizations configure Exchange to validate that the `from` field is a - valid local email account. -<2> Many organizations support use of your email address as your username, though - it is a good idea to check with your system administrator if you receive - authentication-related failures. - - -// [[postfix]] -// ===== Sending Email from Postfix - -// Use the following email account settings to send email from the http://www.postfix.org[Postfix] SMTP service: - -// [source,yaml] -// -------------------------------------------------- -// TODO -// -------------------------------------------------- - -[[email-html-sanitization]] -==== Configuring HTML Sanitization Options - -The `email` action supports sending messages with an HTML body. However, for security reasons, -Watcher https://en.wikipedia.org/wiki/HTML_sanitization[sanitizes] the HTML. - -You can control which HTML features are allowed or disallowed by configuring the -`watcher.actions.email.html.sanitization.allow` and -`watcher.actions.email.html.sanitization.disallow` settings in `elasticsearch.yml`. You can specify -individual HTML elements and the feature groups described in the following table. By default, -Watcher allows the following features: `body`, `head`, `_tables`, `_links`, `_blocks`, `_formatting` -and `img:embedded`. - - -[options="header"] -|====== -| Name | Description - -| `_tables` | All table related elements: ``, `` and `
`, `
`. - -| `_blocks` | The following block elements: `

`, `

`, `

`, `

`, `

`, - `

`, `

`, `
`, `