From 556e8ca401d5dd8e541a722935613687838eb2eb Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 2 Oct 2015 15:28:19 +0200 Subject: [PATCH 01/12] Remove use of com.google.common.net.InetAddresses This commit removes all uses of com.google.common.net.InetAddresses across the codebase. This is one of the few remaining steps in the eventual removal of Guava as a dependency. Relates elastic/elasticsearchelastic/elasticsearch#13224 Original commit: elastic/x-pack-elasticsearch@f6a0ed139570d340359b55eeae14cef9c838e5e7 --- .../shield/transport/filter/ShieldIpFilterRule.java | 2 +- .../elasticsearch/shield/transport/filter/IPFilterTests.java | 2 +- .../transport/netty/IPFilterNettyUpstreamHandlerTests.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shield/src/main/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRule.java b/shield/src/main/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRule.java index 652d6bff274..522708c55bf 100644 --- a/shield/src/main/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRule.java +++ b/shield/src/main/java/org/elasticsearch/shield/transport/filter/ShieldIpFilterRule.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.shield.transport.filter; -import com.google.common.net.InetAddresses; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.transport.TransportAddress; diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/filter/IPFilterTests.java b/shield/src/test/java/org/elasticsearch/shield/transport/filter/IPFilterTests.java index 41c88b13114..dfed968f21f 100644 --- a/shield/src/test/java/org/elasticsearch/shield/transport/filter/IPFilterTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/transport/filter/IPFilterTests.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.shield.transport.filter; -import com.google.common.net.InetAddresses; import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; diff --git a/shield/src/test/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandlerTests.java b/shield/src/test/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandlerTests.java index bc4711d45aa..f0f35554ab6 100644 --- a/shield/src/test/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandlerTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/transport/netty/IPFilterNettyUpstreamHandlerTests.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.shield.transport.netty; -import com.google.common.net.InetAddresses; import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.InetSocketTransportAddress; From 98095a5ca8f4e20356ab2b245712ee75225b87b1 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 7 Oct 2015 07:18:02 -0400 Subject: [PATCH 02/12] add shield 2.0.0-rc1 release notes Original commit: elastic/x-pack-elasticsearch@22e6a1499f3c0548afd754bc35c23d6df26b7d46 --- shield/docs/public/release-notes.asciidoc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/shield/docs/public/release-notes.asciidoc b/shield/docs/public/release-notes.asciidoc index b336b513a32..c3eba5ea7b2 100644 --- a/shield/docs/public/release-notes.asciidoc +++ b/shield/docs/public/release-notes.asciidoc @@ -42,6 +42,18 @@ version of Shield. We recommend copying the changes listed below to your `roles. [[changelist]] === Change List +[float] +==== 2.0.0-rc1 + +.enhancements +* Added a caching interface that can be used by <> to integrate with the <>. + +.bug fixes +* <> now captures requests from nodes using a different system key as tampered requests. +* The <> stores the type of request when available. +* <> could have allowed a user to block all access to their node if the system was incorrectly configured, but now explicitly +allows connections from all addresses that the node is bound to so that connections coming from the node's host will not be blocked. + [float] ==== 2.0.0-beta2 From 56981b5fff25f38e9225eef84a2f8e0e3829b88f Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 7 Oct 2015 14:53:57 +0200 Subject: [PATCH 03/12] watcher: removed calibrating with round clock in ticker trigger engine Closes elastic/elasticsearch#749 Original commit: elastic/x-pack-elasticsearch@f9f2db50d50dfd2d0b6deb8c7304b3bc4f6b657c --- watcher/docs/release-notes.asciidoc | 6 ++++++ .../schedule/engine/TickerScheduleTriggerEngine.java | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/watcher/docs/release-notes.asciidoc b/watcher/docs/release-notes.asciidoc index 5cd770f172d..621c4d7514b 100644 --- a/watcher/docs/release-notes.asciidoc +++ b/watcher/docs/release-notes.asciidoc @@ -36,6 +36,12 @@ bin/plugin remove watcher [[change-list]] === Change List +[float] +==== 2.0.0 + +.Bug fixes +* Fixed an issue where the scheduler may get stuck during Watcher startup. This caused no watches to ever fire. + [float] ==== 2.0.0-rc1 diff --git a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java b/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java index 2167bfb219e..d3271e9c2fa 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java @@ -136,10 +136,6 @@ public class TickerScheduleTriggerEngine extends ScheduleTriggerEngine { @Override public void run() { - - // calibrate with round clock - while (clock.millis() % 1000 > 15) { - } while (active) { logger.trace("checking jobs [{}]", clock.nowUTC()); checkJobs(); From a876755fd5c483e5e46da26938ee5fde6baae9fa Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 7 Oct 2015 16:27:47 -0400 Subject: [PATCH 04/12] fix compile error. SimilarityLookupService -> SimilarityService Original commit: elastic/x-pack-elasticsearch@d2f7e6dcf49e97f50cee82fd678295f37138cca2 --- .../accesscontrol/ShieldIndexSearcherWrapperUnitTests.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java index f627bb0122b..61b40f06022 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java @@ -45,8 +45,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.similarity.SimilarityLookupService; -import org.elasticsearch.indices.InternalIndicesLifecycle; +import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.shield.authz.InternalAuthorizationService; @@ -81,9 +80,9 @@ public class ShieldIndexSearcherWrapperUnitTests extends ESTestCase { Index index = new Index("_index"); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); AnalysisService analysisService = new AnalysisService(index, settings); - SimilarityLookupService similarityLookupService = new SimilarityLookupService(index, settings); + SimilarityService similarityService = new SimilarityService(index, settings); ScriptService scriptService = mock(ScriptService.class); - mapperService = new MapperService(index, settings, analysisService, similarityLookupService, scriptService); + mapperService = new MapperService(index, settings, analysisService, similarityService, scriptService); shardId = new ShardId(index, 0); shieldIndexSearcherWrapper = new ShieldIndexSearcherWrapper(settings, null, mapperService, null); From 144d9e85df790086ba4808fd321967830b4406de Mon Sep 17 00:00:00 2001 From: debadair Date: Wed, 7 Oct 2015 13:34:57 -0700 Subject: [PATCH 05/12] Shield Docs: Fixed GS verification step. Closes elastic/elasticsearch#760. Original commit: elastic/x-pack-elasticsearch@9a2f810131fb181d425d3ea9832585df62825081 --- shield/docs/public/getting-started.asciidoc | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/shield/docs/public/getting-started.asciidoc b/shield/docs/public/getting-started.asciidoc index a2fc253c2dd..1130d539454 100644 --- a/shield/docs/public/getting-started.asciidoc +++ b/shield/docs/public/getting-started.asciidoc @@ -33,14 +33,8 @@ NOTE: If you are using a <> of Elasticsea bin/elasticsearch ---------------------------------------------------------- -. To verify that Shield is up and running, use the `_shield` API to get the Shield version: -+ -[source,shell] ----------------------------------------------------------- -curl -u es_admin -XGET 'http://localhost:9200/_shield' ----------------------------------------------------------- -+ -You can also check the startup log entries. When Shield is operating normally, the log indicates that the network transports are using Shield: +. To verify that Shield is up and running, check the startup log entries. When Shield is operating +normally, the log indicates that the network transports are using Shield: + [source,shell] ---------------- @@ -49,7 +43,6 @@ You can also check the startup log entries. When Shield is operating normally, t [2014-10-09 13:47:38,842][INFO ][http ] [Ezekiel Stane] Using [org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport] as http transport, overridden by [shield] ---------------- - Now you're ready to secure your cluster! Here are a few things you might want to do to start with: From 192526ff00a8e8f7911b676c2b3a251b469e7490 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 7 Oct 2015 11:40:47 +0200 Subject: [PATCH 06/12] Marvel: Add state_uuid to cluster_state documents Closes elastic/elasticsearch#750 Original commit: elastic/x-pack-elasticsearch@4005fe0090deb1a0d6cf0ba35a05b335cf2f63f9 --- .../marvel/agent/renderer/cluster/ClusterStateRenderer.java | 1 + marvel/src/main/resources/marvel_index_template.json | 4 ++++ marvel/src/test/resources/samples/cluster_state.json | 1 + 3 files changed, 6 insertions(+) diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRenderer.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRenderer.java index 51966bcbbc4..78f473edbcc 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRenderer.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRenderer.java @@ -20,6 +20,7 @@ public class ClusterStateRenderer extends AbstractRenderer Date: Thu, 8 Oct 2015 09:09:00 -0400 Subject: [PATCH 07/12] updates to handle renamed RenderSearchTemplateAction Original commit: elastic/x-pack-elasticsearch@03cb49ce5269006307dd13a40b68b8d070327517 --- qa/shield-core-rest-tests/pom.xml | 2 +- shield/docs/public/reference.asciidoc | 2 +- shield/src/test/resources/org/elasticsearch/transport/actions | 2 +- shield/src/test/resources/org/elasticsearch/transport/handlers | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/qa/shield-core-rest-tests/pom.xml b/qa/shield-core-rest-tests/pom.xml index 0889f3e929c..62f358d7944 100644 --- a/qa/shield-core-rest-tests/pom.xml +++ b/qa/shield-core-rest-tests/pom.xml @@ -23,7 +23,7 @@ true ${project.basedir}/integration-tests.xml true - indices.get/10_basic/*allow_no_indices*,cat.count/10_basic/Test cat count output,cat.aliases/10_basic/Empty cluster,indices.segments/10_basic/no segments test,indices.clear_cache/10_basic/clear_cache test,indices.status/10_basic/Indices status test,cat.indices/10_basic/Test cat indices output,cat.recovery/10_basic/Test cat recovery output,cat.shards/10_basic/Test cat shards output,termvector/20_issue7121/*,index/10_with_id/Index with ID,indices.get_alias/20_emtpy/*,cat.segments/10_basic/Test cat segments output,indices.put_settings/10_basic/Test indices settings allow_no_indices,indices.put_settings/10_basic/Test indices settings ignore_unavailable,indices.refresh/10_basic/Indices refresh test no-match wildcard,indices.stats/10_index/Index - star*,indices.recovery/10_basic/Indices recovery test*,template/30_render_search_template/*,indices.shard_stores/10_basic/no indices test,cat.nodeattrs/10_basic/Test cat nodes attrs output + indices.get/10_basic/*allow_no_indices*,cat.count/10_basic/Test cat count output,cat.aliases/10_basic/Empty cluster,indices.segments/10_basic/no segments test,indices.clear_cache/10_basic/clear_cache test,indices.status/10_basic/Indices status test,cat.indices/10_basic/Test cat indices output,cat.recovery/10_basic/Test cat recovery output,cat.shards/10_basic/Test cat shards output,termvector/20_issue7121/*,index/10_with_id/Index with ID,indices.get_alias/20_emtpy/*,cat.segments/10_basic/Test cat segments output,indices.put_settings/10_basic/Test indices settings allow_no_indices,indices.put_settings/10_basic/Test indices settings ignore_unavailable,indices.refresh/10_basic/Indices refresh test no-match wildcard,indices.stats/10_index/Index - star*,indices.recovery/10_basic/Indices recovery test*,indices.shard_stores/10_basic/no indices test,cat.nodeattrs/10_basic/Test cat nodes attrs output license,shield diff --git a/shield/docs/public/reference.asciidoc b/shield/docs/public/reference.asciidoc index 22e3c849ec6..fda22e60356 100644 --- a/shield/docs/public/reference.asciidoc +++ b/shield/docs/public/reference.asciidoc @@ -67,6 +67,7 @@ Elasticsearch. This only applies to publicly available indices and cluster actio [float] ===== Cluster actions privileges +* `cluster:admin/render/template/search` * `cluster:admin/repository/delete` * `cluster:admin/repository/get` * `cluster:admin/repository/put` @@ -118,7 +119,6 @@ NOTE: While indices template actions typically relate to indices, they are categ * `indices:admin/open` * `indices:admin/optimize` * `indices:admin/refresh` -* `indices:admin/render/template/search` * `indices:admin/settings/update` * `indices:admin/shards/search_shards` * `indices:admin/template/delete` diff --git a/shield/src/test/resources/org/elasticsearch/transport/actions b/shield/src/test/resources/org/elasticsearch/transport/actions index 70f8218bdab..36e7cff6027 100644 --- a/shield/src/test/resources/org/elasticsearch/transport/actions +++ b/shield/src/test/resources/org/elasticsearch/transport/actions @@ -1,3 +1,4 @@ +cluster:admin/render/template/search cluster:admin/repository/delete cluster:admin/repository/get cluster:admin/repository/put @@ -33,7 +34,6 @@ indices:admin/mappings/get indices:admin/open indices:admin/optimize indices:admin/refresh -indices:admin/render/template/search indices:admin/settings/update indices:admin/shards/search_shards indices:admin/template/delete diff --git a/shield/src/test/resources/org/elasticsearch/transport/handlers b/shield/src/test/resources/org/elasticsearch/transport/handlers index 26eac0ea059..bf30466c5a4 100644 --- a/shield/src/test/resources/org/elasticsearch/transport/handlers +++ b/shield/src/test/resources/org/elasticsearch/transport/handlers @@ -1,3 +1,4 @@ +cluster:admin/render/template/search cluster:admin/snapshot/status[nodes] cluster:admin/snapshot/status[nodes][n] cluster:monitor/nodes/hot_threads[n] @@ -16,7 +17,6 @@ indices:admin/mappings/fields/get[index][s] indices:admin/optimize[n] indices:admin/refresh[s] indices:admin/refresh[s][r] -indices:admin/render/template/search indices:admin/upgrade indices:admin/upgrade[n] indices:admin/validate/query[s] From e589d2e46e61d15059a8465d7cad9ce6849c174e Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Thu, 8 Oct 2015 15:52:44 +0200 Subject: [PATCH 08/12] Marvel: Fixing compilation errors for changed render search template Original commit: elastic/x-pack-elasticsearch@d04257df85c4dc771ba1fe6ed1f6490920da5269 --- .../org/elasticsearch/marvel/shield/SecuredClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/marvel/src/main/java/org/elasticsearch/marvel/shield/SecuredClient.java b/marvel/src/main/java/org/elasticsearch/marvel/shield/SecuredClient.java index d8ee86d08b6..9a0b13e3ae8 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/shield/SecuredClient.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/shield/SecuredClient.java @@ -82,6 +82,10 @@ import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksAction; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; +import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateAction; +import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequest; +import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequestBuilder; +import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; @@ -194,10 +198,6 @@ import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; -import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateAction; -import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateRequest; -import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateRequestBuilder; -import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateResponse; import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerAction; import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequest; import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequestBuilder; From fa85d045238916c506193423a60187fa8e2082dc Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 8 Oct 2015 11:11:09 -0400 Subject: [PATCH 09/12] Marvel: update client after move of renderSearchTemplate methods Original commit: elastic/x-pack-elasticsearch@7713f2fc721a50294abaed24ebd861fea2fc9e3c --- .../marvel/shield/SecuredClient.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/marvel/src/main/java/org/elasticsearch/marvel/shield/SecuredClient.java b/marvel/src/main/java/org/elasticsearch/marvel/shield/SecuredClient.java index 9a0b13e3ae8..3e7904fc7ed 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/shield/SecuredClient.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/shield/SecuredClient.java @@ -1047,18 +1047,6 @@ public class SecuredClient implements Client { return new ValidateQueryRequestBuilder(this, ValidateQueryAction.INSTANCE).setIndices(indices); } - public ActionFuture renderSearchTemplate(RenderSearchTemplateRequest request) { - return this.execute(RenderSearchTemplateAction.INSTANCE, request); - } - - public void renderSearchTemplate(RenderSearchTemplateRequest request, ActionListener listener) { - this.execute(RenderSearchTemplateAction.INSTANCE, request, listener); - } - - public RenderSearchTemplateRequestBuilder prepareRenderSearchTemplate() { - return new RenderSearchTemplateRequestBuilder(this, RenderSearchTemplateAction.INSTANCE); - } - public ActionFuture putWarmer(PutWarmerRequest request) { return this.execute(PutWarmerAction.INSTANCE, request); } @@ -1370,6 +1358,18 @@ public class SecuredClient implements Client { public SnapshotsStatusRequestBuilder prepareSnapshotStatus() { return new SnapshotsStatusRequestBuilder(this, SnapshotsStatusAction.INSTANCE); } + + public ActionFuture renderSearchTemplate(RenderSearchTemplateRequest request) { + return this.execute(RenderSearchTemplateAction.INSTANCE, request); + } + + public void renderSearchTemplate(RenderSearchTemplateRequest request, ActionListener listener) { + this.execute(RenderSearchTemplateAction.INSTANCE, request, listener); + } + + public RenderSearchTemplateRequestBuilder prepareRenderSearchTemplate() { + return new RenderSearchTemplateRequestBuilder(this, RenderSearchTemplateAction.INSTANCE); + } } static class Admin implements AdminClient { From daf4a9765c5dbb515391300c7c037b01f0a1aa34 Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Fri, 9 Oct 2015 00:32:15 -0400 Subject: [PATCH 10/12] [License] Feature agnostic licensing model This commit changes the license plugin to work with license that are not tied to any specific feature in a bwc way. It refactors the license plugin api into a lighter weight API, enabling the license plugin to manage license expiration and acknowledgment triggers. closes elastic/elasticsearch#683, elastic/elasticsearch#686, elastic/elasticsearch#687, elastic/elasticsearch#691 Original commit: elastic/x-pack-elasticsearch@537cd3933a6a293bba3ce389d5ba7916c7a50549 --- .../cluster/ClusterInfoCollector.java | 3 +- .../renderer/cluster/ClusterInfoRenderer.java | 2 - .../marvel/license/LicenseService.java | 164 +++++++----------- .../org/elasticsearch/marvel/mode/Mode.java | 12 +- .../shield/MarvelInternalUserHolder.java | 2 +- .../collector/AbstractCollectorTestCase.java | 54 +++--- .../agent/renderer/cluster/ClusterInfoIT.java | 1 - .../license/LicenseIntegrationTests.java | 29 ++-- .../public/managing-shield-licenses.asciidoc | 4 +- .../shield/action/ShieldActionFilter.java | 10 +- .../shield/license/LicenseEventsNotifier.java | 16 +- .../shield/license/LicenseService.java | 156 +++++++---------- .../rest/action/RestShieldInfoAction.java | 3 +- .../integration/LicensingTests.java | 34 ++-- .../action/ShieldActionFilterTests.java | 3 +- .../watcher/license/LicenseService.java | 136 +++++---------- .../license/LicenseIntegrationTests.java | 29 ++-- 17 files changed, 258 insertions(+), 400 deletions(-) diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java index e6f202371f5..d1fb7b61e78 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java @@ -60,7 +60,8 @@ public class ClusterInfoCollector extends AbstractCollector results = new ArrayList<>(1); // Retrieves all licenses - List licenses = licenseService.licenses(); + // TODO: we should only work with one license + List licenses = Collections.singletonList(licenseService.license()); // Retrieves additional cluster stats ClusterStatsResponse clusterStats = client.admin().cluster().prepareClusterStats().get(marvelSettings.clusterStatsTimeout()); diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoRenderer.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoRenderer.java index 4a7337ab3cd..0985a898716 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoRenderer.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoRenderer.java @@ -37,7 +37,6 @@ public class ClusterInfoRenderer extends AbstractRenderer builder.field(Fields.UID, license.uid()); builder.field(Fields.TYPE, license.type()); builder.dateValueField(Fields.ISSUE_DATE_IN_MILLIS, Fields.ISSUE_DATE, license.issueDate()); - builder.field(Fields.FEATURE, license.feature()); builder.dateValueField(Fields.EXPIRY_DATE_IN_MILLIS, Fields.EXPIRY_DATE, license.expiryDate()); builder.field(Fields.MAX_NODES, license.maxNodes()); builder.field(Fields.ISSUED_TO, license.issuedTo()); @@ -77,7 +76,6 @@ public class ClusterInfoRenderer extends AbstractRenderer static final XContentBuilderString STATUS = new XContentBuilderString("status"); static final XContentBuilderString UID = new XContentBuilderString("uid"); static final XContentBuilderString TYPE = new XContentBuilderString("type"); - static final XContentBuilderString FEATURE = new XContentBuilderString("feature"); static final XContentBuilderString ISSUE_DATE_IN_MILLIS = new XContentBuilderString("issue_date_in_millis"); static final XContentBuilderString ISSUE_DATE = new XContentBuilderString("issue_date"); static final XContentBuilderString EXPIRY_DATE_IN_MILLIS = new XContentBuilderString("expiry_date_in_millis"); diff --git a/marvel/src/main/java/org/elasticsearch/marvel/license/LicenseService.java b/marvel/src/main/java/org/elasticsearch/marvel/license/LicenseService.java index 638fd923110..2d8036bd4e6 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/license/LicenseService.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/license/LicenseService.java @@ -6,110 +6,45 @@ package org.elasticsearch.marvel.license; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.joda.FormatDateTimeFormatter; -import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.logging.support.LoggerMessageFormat; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.license.core.License; -import org.elasticsearch.license.plugin.core.LicensesClientService; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.license.plugin.core.Licensee; +import org.elasticsearch.license.plugin.core.LicenseeRegistry; import org.elasticsearch.license.plugin.core.LicensesManagerService; -import org.elasticsearch.license.plugin.core.LicensesService; import org.elasticsearch.marvel.MarvelPlugin; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.mode.Mode; -import java.util.*; -public class LicenseService extends AbstractLifecycleComponent { +public class LicenseService extends AbstractLifecycleComponent implements Licensee { public static final String FEATURE_NAME = MarvelPlugin.NAME; - private static final LicensesService.TrialLicenseOptions TRIAL_LICENSE_OPTIONS = - new LicensesService.TrialLicenseOptions(TimeValue.timeValueHours(30 * 24), 1000); - - private static final FormatDateTimeFormatter DATE_FORMATTER = Joda.forPattern("EEEE, MMMMM dd, yyyy", Locale.ROOT); - - static final TimeValue GRACE_PERIOD = days(7); - private final LicensesManagerService managerService; - private final LicensesClientService clientService; + private final LicenseeRegistry clientService; private final MarvelSettings marvelSettings; - private final Collection expirationLoggers; - private final LicensesClientService.AcknowledgementCallback acknowledgementCallback; private volatile Mode mode; - private volatile boolean enabled; + private volatile LicenseState state; private volatile long expiryDate; @Inject - public LicenseService(Settings settings, LicensesClientService clientService, LicensesManagerService managerService, MarvelSettings marvelSettings) { + public LicenseService(Settings settings, LicenseeRegistry clientService, LicensesManagerService managerService, MarvelSettings marvelSettings) { super(settings); this.managerService = managerService; this.clientService = clientService; this.marvelSettings = marvelSettings; this.mode = Mode.LITE; - this.expirationLoggers = Arrays.asList( - new LicensesService.ExpirationCallback.Pre(days(7), days(30), days(1)) { - @Override - public void on(License license, LicensesService.ExpirationStatus status) { - logger.error("\n" + - "#\n" + - "# Marvel license will expire on [{}].\n" + - "# Have a new license? please update it. Otherwise, please reach out to your support contact.\n" + - "#", DATE_FORMATTER.printer().print(license.expiryDate())); - } - }, - new LicensesService.ExpirationCallback.Pre(days(0), days(7), minutes(10)) { - @Override - public void on(License license, LicensesService.ExpirationStatus status) { - logger.error("\n" + - "#\n" + - "# Marvel license will expire on [{}].\n" + - "# Have a new license? please update it. Otherwise, please reach out to your support contact.\n" + - "#", DATE_FORMATTER.printer().print(license.expiryDate())); - } - }, - new LicensesService.ExpirationCallback.Post(days(0), GRACE_PERIOD, minutes(10)) { - @Override - public void on(License license, LicensesService.ExpirationStatus status) { - long endOfGracePeriod = license.expiryDate() + GRACE_PERIOD.getMillis(); - logger.error("\n" + - "#\n" + - "# MARVEL LICENSE HAS EXPIRED ON [{}].\n" + - "# MARVEL WILL STOP COLLECTING DATA ON [{}].\n" + - "# HAVE A NEW LICENSE? PLEASE UPDATE IT. OTHERWISE, PLEASE REACH OUT TO YOUR SUPPORT CONTACT.\n" + - "#", DATE_FORMATTER.printer().print(endOfGracePeriod), DATE_FORMATTER.printer().print(license.expiryDate())); - } - } - ); - this.acknowledgementCallback = new LicensesClientService.AcknowledgementCallback() { - @Override - public List acknowledge(License currentLicense, License newLicense) { - switch (newLicense.type()) { - - case "trial": - case "gold": - case "platinum": - return Collections.emptyList(); - - default: // "basic" - we also fall back to basic for an unknown type - return Collections.singletonList(LoggerMessageFormat.format( - "Marvel: Multi-cluster support is disabled for clusters with [{}] licenses.\n" + - "If you are running multiple customers, users won't be able to access this\n" + - "all the clusters with [{}] licenses from a single Marvel instance. To access them\n" + - "a dedicated and separated marvel instance will be required for each cluster", - newLicense.type(), newLicense.type())); - } - } - }; } @Override protected void doStart() throws ElasticsearchException { - clientService.register(FEATURE_NAME, TRIAL_LICENSE_OPTIONS, expirationLoggers, acknowledgementCallback, new InternalListener(this)); + clientService.register(this); } @Override @@ -120,14 +55,6 @@ public class LicenseService extends AbstractLifecycleComponent { protected void doClose() throws ElasticsearchException { } - static TimeValue days(int days) { - return TimeValue.timeValueHours(days * 24); - } - - static TimeValue minutes(int minutes) { - return TimeValue.timeValueMinutes(minutes); - } - /** * @return the current marvel's operating mode */ @@ -138,18 +65,20 @@ public class LicenseService extends AbstractLifecycleComponent { /** * @return all registered licenses */ - public List licenses() { - return managerService.getLicenses(); + public License license() { + return managerService.getLicense(); } /** * @return true if the marvel license is enabled */ public boolean enabled() { - return enabled; + return state == LicenseState.ENABLED || state == LicenseState.GRACE_PERIOD; } /** + * TODO: remove licensing grace period, just check for state == LicensesClientService.LicenseState.GRACE_PERIOD instead + * * @return true if marvel is running within the "grace period", ie when the license * is expired but a given extra delay is not yet elapsed */ @@ -164,30 +93,57 @@ public class LicenseService extends AbstractLifecycleComponent { return expiryDate; } - class InternalListener implements LicensesClientService.Listener { + @Override + public String id() { + return FEATURE_NAME; + } - private final LicenseService service; + @Override + public String[] expirationMessages() { + // TODO add messages to be logged around license expiry + return Strings.EMPTY_ARRAY; + } - public InternalListener(LicenseService service) { - this.service = service; + @Override + public String[] acknowledgmentMessages(License currentLicense, License newLicense) { + switch (newLicense.operationMode()) { + case BASIC: + if (currentLicense != null) { + switch (currentLicense.operationMode()) { + case TRIAL: + case GOLD: + case PLATINUM: + return new String[] { + LoggerMessageFormat.format( + "Multi-cluster support is disabled for clusters with [{}] licenses.\n" + + "If you are running multiple customers, users won't be able to access this\n" + + "all the clusters with [{}] licenses from a single Marvel instance. To access them\n" + + "a dedicated and separated marvel instance will be required for each cluster", + newLicense.type(), newLicense.type()) + }; + } + } } + return Strings.EMPTY_ARRAY; + } - @Override - public void onEnabled(License license) { - try { - service.enabled = true; - service.expiryDate = license.expiryDate(); - service.mode = Mode.fromName(license.type()); - } catch (IllegalArgumentException e) { - service.mode = Mode.LITE; + @Override + public void onChange(License license, LicenseState state) { + synchronized (this) { + this.state = state; + if (license != null) { + try { + mode = Mode.fromName(license.type()); + } catch (IllegalArgumentException e) { + mode = Mode.LITE; + } + expiryDate = license.expiryDate(); + } else { + mode = Mode.LITE; + } + if (state == LicenseState.DISABLED) { + mode = Mode.LITE; } - } - - @Override - public void onDisabled(License license) { - service.enabled = false; - service.expiryDate = license.expiryDate(); - service.mode = Mode.LITE; } } } diff --git a/marvel/src/main/java/org/elasticsearch/marvel/mode/Mode.java b/marvel/src/main/java/org/elasticsearch/marvel/mode/Mode.java index 9947aa0008a..4b553363567 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/mode/Mode.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/mode/Mode.java @@ -16,6 +16,8 @@ public enum Mode { /** * Marvel runs in downgraded mode + * + * TODO: do we really need mode? */ TRIAL(0), @@ -55,9 +57,13 @@ public enum Mode { public static Mode fromName(String name) { switch (name.toLowerCase(Locale.ROOT)) { - case "trial": return TRIAL; - case "lite": return LITE; - case "standard" : return STANDARD; + case "trial": + return LITE; + case "basic": + case "gold" : + case "silver": + case "platinum": + return STANDARD; default: throw new ElasticsearchException("unknown marvel mode name [" + name + "]"); } diff --git a/marvel/src/main/java/org/elasticsearch/marvel/shield/MarvelInternalUserHolder.java b/marvel/src/main/java/org/elasticsearch/marvel/shield/MarvelInternalUserHolder.java index 9a804f6a540..687e9d8475e 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/shield/MarvelInternalUserHolder.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/shield/MarvelInternalUserHolder.java @@ -33,7 +33,7 @@ public class MarvelInternalUserHolder { // and full access to .marvel-* and .marvel-data indices .add(Privilege.Index.ALL, MarvelSettings.MARVEL_INDICES_PREFIX + "*") - // note, we don't need _licenses permission as we're taking the licenses + // note, we don't need _license permission as we're taking the licenses // directly form the license service. .build(); diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java index dc021054911..ba1bd9e864d 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java @@ -17,12 +17,11 @@ import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.license.core.License; -import org.elasticsearch.license.plugin.core.LicensesClientService; -import org.elasticsearch.license.plugin.core.LicensesManagerService; -import org.elasticsearch.license.plugin.core.LicensesService; +import org.elasticsearch.license.plugin.action.delete.DeleteLicenseRequest; +import org.elasticsearch.license.plugin.action.put.PutLicenseRequest; +import org.elasticsearch.license.plugin.core.*; import org.elasticsearch.marvel.MarvelPlugin; import org.elasticsearch.marvel.agent.settings.MarvelSettings; -import org.elasticsearch.marvel.license.LicenseService; import org.elasticsearch.marvel.shield.MarvelShieldIntegration; import org.elasticsearch.marvel.shield.SecuredClient; import org.elasticsearch.marvel.test.MarvelIntegTestCase; @@ -88,15 +87,13 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase { private static License createTestingLicense(long issueDate, long expiryDate) { return License.builder() - .feature(LicenseService.FEATURE_NAME) .expiryDate(expiryDate) .issueDate(issueDate) .issuedTo("AbstractCollectorTestCase") .issuer("test") .maxNodes(Integer.MAX_VALUE) .signature("_signature") - .type("standard") - .subscriptionType("all_is_good") + .type("basic") .uid(String.valueOf(RandomizedTest.systemPropertyAsInt(SysGlobals.CHILDVM_SYSPROP_JVM_ID, 0)) + System.identityHashCode(AbstractCollectorTestCase.class)) .build(); } @@ -107,7 +104,7 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase { final License license = createTestingLicense(issueDate, expiryDate); for (LicenseServiceForCollectors service : internalCluster().getInstances(LicenseServiceForCollectors.class)) { - service.enable(license); + service.onChange(license, LicenseState.ENABLED); } for (LicensesManagerServiceForCollectors service : internalCluster().getInstances(LicensesManagerServiceForCollectors.class)) { service.update(license); @@ -120,7 +117,7 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase { final License license = createTestingLicense(issueDate, expiryDate); for (LicenseServiceForCollectors service : internalCluster().getInstances(LicenseServiceForCollectors.class)) { - service.disable(license); + service.onChange(license, LicenseState.GRACE_PERIOD); } for (LicensesManagerServiceForCollectors service : internalCluster().getInstances(LicensesManagerServiceForCollectors.class)) { service.update(license); @@ -133,7 +130,7 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase { final License license = createTestingLicense(issueDate, expiryDate); for (LicenseServiceForCollectors service : internalCluster().getInstances(LicenseServiceForCollectors.class)) { - service.disable(license); + service.onChange(license, LicenseState.DISABLED); } for (LicensesManagerServiceForCollectors service : internalCluster().getInstances(LicensesManagerServiceForCollectors.class)) { service.update(license); @@ -146,7 +143,7 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase { final License license = createTestingLicense(issueDate, expiryDate); for (LicenseServiceForCollectors service : internalCluster().getInstances(LicenseServiceForCollectors.class)) { - service.disable(license); + service.onChange(license, LicenseState.DISABLED); } for (LicensesManagerServiceForCollectors service : internalCluster().getInstances(LicensesManagerServiceForCollectors.class)) { service.update(license); @@ -205,7 +202,7 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase { @Override protected void configure() { bind(LicenseServiceForCollectors.class).asEagerSingleton(); - bind(LicensesClientService.class).to(LicenseServiceForCollectors.class); + bind(LicenseeRegistry.class).to(LicenseServiceForCollectors.class); bind(LicensesManagerServiceForCollectors.class).asEagerSingleton(); bind(LicensesManagerService.class).to(LicensesManagerServiceForCollectors.class); } @@ -213,9 +210,9 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase { } } - public static class LicenseServiceForCollectors extends AbstractComponent implements LicensesClientService { + public static class LicenseServiceForCollectors extends AbstractComponent implements LicenseeRegistry { - private final List listeners = new ArrayList<>(); + private final List licensees = new ArrayList<>(); @Inject public LicenseServiceForCollectors(Settings settings) { @@ -223,19 +220,13 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase { } @Override - public void register(String feature, TrialLicenseOptions trialLicenseOptions, Collection expirationCallbacks, AcknowledgementCallback acknowledgementCallback, Listener listener) { - listeners.add(listener); + public void register(Licensee licensee) { + licensees.add(licensee); } - public void enable(License license) { - for (Listener listener : listeners) { - listener.onEnabled(license); - } - } - - public void disable(License license) { - for (Listener listener : listeners) { - listener.onDisabled(license); + public void onChange(License license, LicenseState state) { + for (Licensee licensee : licensees) { + licensee.onChange(license, state); } } } @@ -245,21 +236,24 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase { private final Map licenses = Collections.synchronizedMap(new HashMap()); @Override - public void registerLicenses(LicensesService.PutLicenseRequestHolder requestHolder, ActionListener listener) { + public void registerLicense(PutLicenseRequest request, ActionListener listener) { } @Override - public void removeLicenses(LicensesService.DeleteLicenseRequestHolder requestHolder, ActionListener listener) { + public void removeLicense(DeleteLicenseRequest request, ActionListener listener) { } @Override - public Set enabledFeatures() { + public List licenseesWithState(LicenseState state) { return null; } @Override - public List getLicenses() { - return new ArrayList<>(licenses.values()); + public License getLicense() { + // TODO: we only take the first of the licenses that are updated + // FIXME + Iterator iterator = licenses.values().iterator(); + return iterator.hasNext() ? iterator.next() : null; } public void update(License license) { diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoIT.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoIT.java index fbb44ec612c..8b36b4692b5 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoIT.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoIT.java @@ -93,7 +93,6 @@ public class ClusterInfoIT extends MarvelIntegTestCase { String recalculated = ClusterInfoRenderer.hash(status, uid, type, String.valueOf(expiryDate), clusterUUID); assertThat(hkey, equalTo(recalculated)); - assertThat((String) license.get(ClusterInfoRenderer.Fields.FEATURE.underscore().toString()), not(isEmptyOrNullString())); assertThat((String) license.get(ClusterInfoRenderer.Fields.ISSUER.underscore().toString()), not(isEmptyOrNullString())); assertThat((String) license.get(ClusterInfoRenderer.Fields.ISSUED_TO.underscore().toString()), not(isEmptyOrNullString())); assertThat((Long) license.get(ClusterInfoRenderer.Fields.ISSUE_DATE_IN_MILLIS.underscore().toString()), greaterThan(0L)); diff --git a/marvel/src/test/java/org/elasticsearch/marvel/license/LicenseIntegrationTests.java b/marvel/src/test/java/org/elasticsearch/marvel/license/LicenseIntegrationTests.java index 98f7a5d82a3..7b1d6836e98 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/license/LicenseIntegrationTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/license/LicenseIntegrationTests.java @@ -13,8 +13,9 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.core.License; -import org.elasticsearch.license.plugin.core.LicensesClientService; -import org.elasticsearch.license.plugin.core.LicensesService; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.license.plugin.core.Licensee; +import org.elasticsearch.license.plugin.core.LicenseeRegistry; import org.elasticsearch.marvel.MarvelPlugin; import org.elasticsearch.marvel.mode.Mode; import org.elasticsearch.marvel.test.MarvelIntegTestCase; @@ -106,26 +107,24 @@ public class LicenseIntegrationTests extends MarvelIntegTestCase { @Override protected void configure() { bind(MockLicenseService.class).asEagerSingleton(); - bind(LicensesClientService.class).to(MockLicenseService.class); + bind(LicenseeRegistry.class).to(MockLicenseService.class); } } - public static class MockLicenseService extends AbstractComponent implements LicensesClientService { + public static class MockLicenseService extends AbstractComponent implements LicenseeRegistry { static final License DUMMY_LICENSE = License.builder() - .feature(LicenseService.FEATURE_NAME) .expiryDate(System.currentTimeMillis()) .issueDate(System.currentTimeMillis()) .issuedTo("LicensingTests") .issuer("test") .maxNodes(Integer.MAX_VALUE) .signature("_signature") - .type("standard") - .subscriptionType("all_is_good") + .type("basic") .uid(String.valueOf(RandomizedTest.systemPropertyAsInt(SysGlobals.CHILDVM_SYSPROP_JVM_ID, 0)) + System.identityHashCode(LicenseIntegrationTests.class)) .build(); - private final List listeners = new ArrayList<>(); + private final List licensees = new ArrayList<>(); @Inject public MockLicenseService(Settings settings) { @@ -134,22 +133,20 @@ public class LicenseIntegrationTests extends MarvelIntegTestCase { } @Override - public void register(String s, LicensesService.TrialLicenseOptions trialLicenseOptions, Collection collection, AcknowledgementCallback acknowledgementCallback, Listener listener) { - listeners.add(listener); + public void register(Licensee licensee) { + licensees.add(licensee); enable(); } public void enable() { - // enabled all listeners (incl. shield) - for (Listener listener : listeners) { - listener.onEnabled(DUMMY_LICENSE); + for (Licensee licensee : licensees) { + licensee.onChange(DUMMY_LICENSE, randomBoolean() ? LicenseState.GRACE_PERIOD : LicenseState.ENABLED); } } public void disable() { - // only disable watcher listener (we need shield to work) - for (Listener listener : listeners) { - listener.onDisabled(DUMMY_LICENSE); + for (Licensee licensee : licensees) { + licensee.onChange(DUMMY_LICENSE, LicenseState.DISABLED); } } } diff --git a/shield/docs/public/managing-shield-licenses.asciidoc b/shield/docs/public/managing-shield-licenses.asciidoc index 8dc8ac8e574..6c4d5d51f9d 100644 --- a/shield/docs/public/managing-shield-licenses.asciidoc +++ b/shield/docs/public/managing-shield-licenses.asciidoc @@ -22,7 +22,7 @@ To install or update the license use the following REST API: [source,shell] ----------------------------------------------------------------------- -curl -XPUT -u admin 'http://:/_licenses' -d @license.json +curl -XPUT -u admin 'http://:/_license' -d @license.json ----------------------------------------------------------------------- Where: @@ -42,7 +42,7 @@ You can list all currently installed licenses by executing the following REST AP [source,shell] ----------------------------------------------------- -curl -XGET -u admin:password 'http://:/_licenses' +curl -XGET -u admin:password 'http://:/_license' ----------------------------------------------------- The response of this command will be a JSON listing all available licenses. In the case of Shield, the following diff --git a/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionFilter.java b/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionFilter.java index c62e9c85552..69aff42244f 100644 --- a/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionFilter.java +++ b/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionFilter.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.support.ActionFilterChain; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.core.LicenseState; import org.elasticsearch.license.plugin.core.LicenseUtils; import org.elasticsearch.shield.User; import org.elasticsearch.shield.action.interceptor.RequestInterceptor; @@ -62,13 +63,8 @@ public class ShieldActionFilter extends AbstractComponent implements ActionFilte this.actionMapper = actionMapper; licenseEventsNotifier.register(new LicenseEventsNotifier.Listener() { @Override - public void enabled() { - licenseEnabled = true; - } - - @Override - public void disabled() { - licenseEnabled = false; + public void notify(LicenseState state) { + licenseEnabled = state != LicenseState.DISABLED; } }); this.requestInterceptors = requestInterceptors; diff --git a/shield/src/main/java/org/elasticsearch/shield/license/LicenseEventsNotifier.java b/shield/src/main/java/org/elasticsearch/shield/license/LicenseEventsNotifier.java index 5de13445386..1ea9f42b10a 100644 --- a/shield/src/main/java/org/elasticsearch/shield/license/LicenseEventsNotifier.java +++ b/shield/src/main/java/org/elasticsearch/shield/license/LicenseEventsNotifier.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.shield.license; +import org.elasticsearch.license.plugin.core.LicenseState; + import java.util.HashSet; import java.util.Set; @@ -26,22 +28,14 @@ public class LicenseEventsNotifier { listeners.add(listener); } - protected void notifyEnabled() { + protected void notify(LicenseState state) { for (Listener listener : listeners) { - listener.enabled(); - } - } - - protected void notifyDisabled() { - for (Listener listener : listeners) { - listener.disabled(); + listener.notify(state); } } public static interface Listener { - void enabled(); - - void disabled(); + void notify(LicenseState state); } } diff --git a/shield/src/main/java/org/elasticsearch/shield/license/LicenseService.java b/shield/src/main/java/org/elasticsearch/shield/license/LicenseService.java index b7506278440..95ae692bfda 100644 --- a/shield/src/main/java/org/elasticsearch/shield/license/LicenseService.java +++ b/shield/src/main/java/org/elasticsearch/shield/license/LicenseService.java @@ -6,99 +6,95 @@ package org.elasticsearch.shield.license; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.joda.FormatDateTimeFormatter; -import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.license.core.License; -import org.elasticsearch.license.plugin.core.LicensesClientService; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.license.plugin.core.Licensee; +import org.elasticsearch.license.plugin.core.LicenseeRegistry; import org.elasticsearch.shield.ShieldPlugin; -import java.util.*; - /** * */ -public class LicenseService extends AbstractLifecycleComponent { +public class LicenseService extends AbstractLifecycleComponent implements Licensee { public static final String FEATURE_NAME = ShieldPlugin.NAME; - private static final LicensesClientService.TrialLicenseOptions TRIAL_LICENSE_OPTIONS = - new LicensesClientService.TrialLicenseOptions(TimeValue.timeValueHours(30 * 24), 1000); - - private static final FormatDateTimeFormatter DATE_FORMATTER = Joda.forPattern("EEEE, MMMMM dd, yyyy", Locale.ROOT); - - private final LicensesClientService licensesClientService; + private final LicenseeRegistry licenseeRegistry; private final LicenseEventsNotifier notifier; - private final Collection expirationLoggers; - private final LicensesClientService.AcknowledgementCallback acknowledgementCallback; - private boolean enabled = false; + private volatile LicenseState state = LicenseState.DISABLED; @Inject - public LicenseService(Settings settings, LicensesClientService licensesClientService, LicenseEventsNotifier notifier) { + public LicenseService(Settings settings, LicenseeRegistry licenseeRegistry, LicenseEventsNotifier notifier) { super(settings); - this.licensesClientService = licensesClientService; + this.licenseeRegistry = licenseeRegistry; this.notifier = notifier; - this.expirationLoggers = Arrays.asList( - new LicensesClientService.ExpirationCallback.Pre(days(7), days(30), days(1)) { - @Override - public void on(License license, LicensesClientService.ExpirationStatus status) { - logger.error("\n" + - "#\n" + - "# Shield license will expire on [{}]. Cluster health, cluster stats and indices stats operations are\n" + - "# blocked on Shield license expiration. All data operations (read and write) continue to work. If you\n" + - "# have a new license, please update it. Otherwise, please reach out to your support contact.\n" + - "#", DATE_FORMATTER.printer().print(license.expiryDate())); - } - }, - new LicensesClientService.ExpirationCallback.Pre(days(0), days(7), minutes(10)) { - @Override - public void on(License license, LicensesClientService.ExpirationStatus status) { - logger.error("\n" + - "#\n" + - "# Shield license will expire on [{}]. Cluster health, cluster stats and indices stats operations are\n" + - "# blocked on Shield license expiration. All data operations (read and write) continue to work. If you\n" + - "# have a new license, please update it. Otherwise, please reach out to your support contact.\n" + - "#", DATE_FORMATTER.printer().print(license.expiryDate())); - } - }, - new LicensesClientService.ExpirationCallback.Post(days(0), null, minutes(10)) { - @Override - public void on(License license, LicensesClientService.ExpirationStatus status) { - logger.error("\n" + - "#\n" + - "# SHIELD LICENSE EXPIRED ON [{}]! CLUSTER HEALTH, CLUSTER STATS AND INDICES STATS OPERATIONS ARE\n" + - "# NOW BLOCKED. ALL DATA OPERATIONS (READ AND WRITE) CONTINUE TO WORK. IF YOU HAVE A NEW LICENSE, PLEASE\n" + - "# UPDATE IT. OTHERWISE, PLEASE REACH OUT TO YOUR SUPPORT CONTACT.\n" + - "#", DATE_FORMATTER.printer().print(license.expiryDate())); - } - } - ); - this.acknowledgementCallback = new LicensesClientService.AcknowledgementCallback() { - @Override - public List acknowledge(License currentLicense, License newLicense) { - // TODO: add messages to be acknowledged when installing newLicense from currentLicense - // NOTE: currentLicense can be null, as a license registration can happen before - // a trial license could be generated - return Collections.emptyList(); - } + } + + @Override + public String id() { + return FEATURE_NAME; + } + + @Override + public String[] expirationMessages() { + return new String[] { + "Cluster health, cluster stats and indices stats operations are blocked", + "All data operations (read and write) continue to work" }; } - public synchronized boolean enabled() { - return enabled; + @Override + public String[] acknowledgmentMessages(License currentLicense, License newLicense) { + switch (newLicense.operationMode()) { + case BASIC: + if (currentLicense != null) { + switch (currentLicense.operationMode()) { + case TRIAL: + case GOLD: + case PLATINUM: + return new String[] { "The following Shield functionality will be disabled: authentication, authorization, ip filtering, auditing, SSL will be disabled on node restart. Please restart your node after applying the license." }; + } + } + break; + case GOLD: + if (currentLicense != null) { + switch (currentLicense.operationMode()) { + case TRIAL: + case BASIC: + case PLATINUM: + return new String[] { + "Field and document level access control will be disabled" + }; + } + } + break; + } + return Strings.EMPTY_ARRAY; + } + + @Override + public void onChange(License license, LicenseState state) { + synchronized (this) { + this.state = state; + notifier.notify(state); + } + } + public LicenseState state() { + return state; } @Override protected void doStart() throws ElasticsearchException { if (settings.getGroups("tribe", true).isEmpty()) { - licensesClientService.register(FEATURE_NAME, TRIAL_LICENSE_OPTIONS, expirationLoggers, acknowledgementCallback, new InternalListener()); + licenseeRegistry.register(this); } else { //TODO currently we disable licensing on tribe node. remove this once es core supports merging cluster - new InternalListener().onEnabled(null); + onChange(null, LicenseState.ENABLED); } } @@ -109,34 +105,4 @@ public class LicenseService extends AbstractLifecycleComponent { @Override protected void doClose() throws ElasticsearchException { } - - static TimeValue days(int days) { - return TimeValue.timeValueHours(days * 24); - } - - static TimeValue minutes(int minutes) { - return TimeValue.timeValueMinutes(minutes); - } - - class InternalListener implements LicensesClientService.Listener { - - @Override - public void onEnabled(License license) { - synchronized (LicenseService.this) { - logger.info("enabling license for [{}]", FEATURE_NAME); - enabled = true; - notifier.notifyEnabled(); - } - } - - @Override - public void onDisabled(License license) { - synchronized (LicenseService.this) { - logger.info("DISABLING LICENSE FOR [{}]", FEATURE_NAME); - enabled = false; - notifier.notifyDisabled(); - } - } - } - } diff --git a/shield/src/main/java/org/elasticsearch/shield/rest/action/RestShieldInfoAction.java b/shield/src/main/java/org/elasticsearch/shield/rest/action/RestShieldInfoAction.java index 1a94e529b6b..86dd59d3137 100644 --- a/shield/src/main/java/org/elasticsearch/shield/rest/action/RestShieldInfoAction.java +++ b/shield/src/main/java/org/elasticsearch/shield/rest/action/RestShieldInfoAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.internal.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.license.plugin.core.LicenseState; import org.elasticsearch.rest.*; import org.elasticsearch.shield.ShieldBuild; import org.elasticsearch.shield.ShieldPlugin; @@ -71,7 +72,7 @@ public class RestShieldInfoAction extends BaseRestHandler { private Status resolveStatus() { if (shieldEnabled) { - if (licenseService.enabled()) { + if (licenseService.state() != LicenseState.DISABLED) { return Status.ENABLED; } return Status.UNLICENSED; diff --git a/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java b/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java index a351116ea38..1be68833056 100644 --- a/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java +++ b/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java @@ -19,7 +19,9 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.core.License; -import org.elasticsearch.license.plugin.core.LicensesClientService; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.license.plugin.core.Licensee; +import org.elasticsearch.license.plugin.core.LicenseeRegistry; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.shield.license.LicenseService; @@ -169,13 +171,13 @@ public class LicensingTests extends ShieldIntegTestCase { } public static void disableLicensing() { - for (InternalLicensesClientService service : internalCluster().getInstances(InternalLicensesClientService.class)) { + for (InternalLicenseeRegistry service : internalCluster().getInstances(InternalLicenseeRegistry.class)) { service.disable(); } } public static void enableLicensing() { - for (InternalLicensesClientService service : internalCluster().getInstances(InternalLicensesClientService.class)) { + for (InternalLicenseeRegistry service : internalCluster().getInstances(InternalLicenseeRegistry.class)) { service.enable(); } } @@ -203,49 +205,47 @@ public class LicensingTests extends ShieldIntegTestCase { public static class InternalLicenseModule extends AbstractModule { @Override protected void configure() { - bind(InternalLicensesClientService.class).asEagerSingleton(); - bind(LicensesClientService.class).to(InternalLicensesClientService.class); + bind(InternalLicenseeRegistry.class).asEagerSingleton(); + bind(LicenseeRegistry.class).to(InternalLicenseeRegistry.class); } } - public static class InternalLicensesClientService extends AbstractComponent implements LicensesClientService { + public static class InternalLicenseeRegistry extends AbstractComponent implements LicenseeRegistry { - private final List listeners = new ArrayList<>(); + private final List licensees = new ArrayList<>(); static final License DUMMY_LICENSE = License.builder() - .feature(LicenseService.FEATURE_NAME) .expiryDate(System.currentTimeMillis()) .issueDate(System.currentTimeMillis()) .issuedTo("LicensingTests") .issuer("test") .maxNodes(Integer.MAX_VALUE) .signature("_signature") - .type("test_license_for_shield") - .subscriptionType("all_is_good") + .type("basic") .uid(String.valueOf(randomLong()) + System.identityHashCode(LicensingTests.class)) .build(); @Inject - public InternalLicensesClientService(Settings settings) { + public InternalLicenseeRegistry(Settings settings) { super(settings); enable(); } @Override - public void register(String s, LicensesClientService.TrialLicenseOptions trialLicenseOptions, Collection collection, AcknowledgementCallback acknowledgementCallback, Listener listener) { - listeners.add(listener); + public void register(Licensee licensee) { + licensees.add(licensee); enable(); } void enable() { - for (Listener listener : listeners) { - listener.onEnabled(DUMMY_LICENSE); + for (Licensee licensee : licensees) { + licensee.onChange(DUMMY_LICENSE, LicenseState.ENABLED); } } void disable() { - for (Listener listener : listeners) { - listener.onDisabled(DUMMY_LICENSE); + for (Licensee licensee : licensees) { + licensee.onChange(DUMMY_LICENSE, LicenseState.DISABLED); } } } diff --git a/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionFilterTests.java b/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionFilterTests.java index f5992c5bd8e..99d0d9e7192 100644 --- a/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionFilterTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionFilterTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.support.ActionFilterChain; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.plugin.core.LicenseState; import org.elasticsearch.shield.User; import org.elasticsearch.shield.action.interceptor.RequestInterceptor; import org.elasticsearch.shield.audit.AuditTrail; @@ -112,7 +113,7 @@ public class ShieldActionFilterTests extends ESTestCase { private class MockLicenseEventsNotifier extends LicenseEventsNotifier { @Override public void register(MockLicenseEventsNotifier.Listener listener) { - listener.enabled(); + listener.notify(LicenseState.ENABLED); } } } diff --git a/watcher/src/main/java/org/elasticsearch/watcher/license/LicenseService.java b/watcher/src/main/java/org/elasticsearch/watcher/license/LicenseService.java index 41463a8e865..f19c22b1773 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/license/LicenseService.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/license/LicenseService.java @@ -6,93 +6,70 @@ package org.elasticsearch.watcher.license; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.joda.FormatDateTimeFormatter; -import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.license.core.License; -import org.elasticsearch.license.plugin.core.LicensesClientService; -import org.elasticsearch.license.plugin.core.LicensesService; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.license.plugin.core.Licensee; +import org.elasticsearch.license.plugin.core.LicenseeRegistry; import org.elasticsearch.watcher.WatcherPlugin; -import java.util.*; - /** * */ -public class LicenseService extends AbstractLifecycleComponent { +public class LicenseService extends AbstractLifecycleComponent implements Licensee { public static final String FEATURE_NAME = WatcherPlugin.NAME; - private static final LicensesClientService.TrialLicenseOptions TRIAL_LICENSE_OPTIONS = - new LicensesClientService.TrialLicenseOptions(TimeValue.timeValueHours(30 * 24), 1000); - - private static final FormatDateTimeFormatter DATE_FORMATTER = Joda.forPattern("EEEE, MMMMM dd, yyyy", Locale.ROOT); - - private final LicensesClientService clientService; - private final Collection expirationLoggers; - private final LicensesClientService.AcknowledgementCallback acknowledgementCallback; - - private volatile boolean enabled; + private final LicenseeRegistry clientService; + private volatile LicenseState state; @Inject - public LicenseService(Settings settings, LicensesClientService clientService) { + public LicenseService(Settings settings, LicenseeRegistry clientService) { super(settings); this.clientService = clientService; - this.expirationLoggers = Arrays.asList( - new LicensesService.ExpirationCallback.Pre(days(7), days(30), days(1)) { - @Override - public void on(License license, LicensesService.ExpirationStatus status) { - logger.error("\n" + - "#\n" + - "# Watcher license will expire on [{}]. All configured actions on\n" + - "# all registered watches are throttled (not executed) on Watcher license expiration. \n" + - "# Watches will continue be evaluated and watch history will continue being recorded.\n" + - "# Have a new license? please update it. Otherwise, please reach out to your support contact.\n" + - "#", DATE_FORMATTER.printer().print(license.expiryDate())); - } - }, - new LicensesService.ExpirationCallback.Pre(days(0), days(7), minutes(10)) { - @Override - public void on(License license, LicensesService.ExpirationStatus status) { - logger.error("\n" + - "#\n" + - "# Watcher license will expire on [{}]. All configured actions on\n" + - "# all registered watches are throttled (not executed) on Watcher license expiration. \n" + - "# Watches will continue be evaluated and watch history will continue being recorded.\n" + - "# Have a new license? please update it. Otherwise, please reach out to your support contact.\n" + - "#", DATE_FORMATTER.printer().print(license.expiryDate())); - } - }, - new LicensesService.ExpirationCallback.Post(days(0), null, minutes(10)) { - @Override - public void on(License license, LicensesService.ExpirationStatus status) { - logger.error("\n" + - "#\n" + - "# WATCHER LICENSE WAS EXPIRED ON [{}]. ALL CONFIGURED ACTIONS ON\n" + - "# ALL REGISTERED WATCHES ARE THROTTLED (NOT EXECUTED) ON WATCHER LICENSE EXPIRATION. \n" + - "# WATCHES WILL CONTINUE BE EVALUATED AND WATCH HISTORY WILL CONTINUE BEING RECORDED.\n" + - "# HAVE A NEW LICENSE? PLEASE UPDATE IT. OTHERWISE, PLEASE REACH OUT TO YOUR SUPPORT CONTACT.\n" + - "#", DATE_FORMATTER.printer().print(license.expiryDate())); + } + + @Override + public String id() { + return FEATURE_NAME; + } + + @Override + public String[] expirationMessages() { + // TODO add messages to be logged around license expiry + return new String[0]; + } + + @Override + public String[] acknowledgmentMessages(License currentLicense, License newLicense) { + switch (newLicense.operationMode()) { + case BASIC: + if (currentLicense != null) { + switch (currentLicense.operationMode()) { + case TRIAL: + case GOLD: + case PLATINUM: + return new String[] { "Watcher will be disabled" }; } } - ); - this.acknowledgementCallback = new LicensesClientService.AcknowledgementCallback() { - @Override - public List acknowledge(License currentLicense, License newLicense) { - // TODO: add messages to be acknowledged when installing newLicense from currentLicense - // NOTE: currentLicense can be null, as a license registration can happen before - // a trial license could be generated - return Collections.emptyList(); - } - }; + break; + } + return Strings.EMPTY_ARRAY; + } + + @Override + public void onChange(License license, LicenseState state) { + synchronized (this) { + this.state = state; + } } @Override protected void doStart() throws ElasticsearchException { - clientService.register(FEATURE_NAME, TRIAL_LICENSE_OPTIONS, expirationLoggers, acknowledgementCallback, new InternalListener(this)); + clientService.register(this); } @Override @@ -104,33 +81,6 @@ public class LicenseService extends AbstractLifecycleComponent { } public boolean enabled() { - return enabled; - } - - static TimeValue days(int days) { - return TimeValue.timeValueHours(days * 24); - } - - static TimeValue minutes(int minutes) { - return TimeValue.timeValueMinutes(minutes); - } - - class InternalListener implements LicensesClientService.Listener { - - private final LicenseService service; - - public InternalListener(LicenseService service) { - this.service = service; - } - - @Override - public void onEnabled(License license) { - service.enabled = true; - } - - @Override - public void onDisabled(License license) { - service.enabled = false; - } + return state != LicenseState.DISABLED; } } diff --git a/watcher/src/test/java/org/elasticsearch/watcher/license/LicenseIntegrationTests.java b/watcher/src/test/java/org/elasticsearch/watcher/license/LicenseIntegrationTests.java index 41d0853440b..cd5cc13fdd8 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/license/LicenseIntegrationTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/license/LicenseIntegrationTests.java @@ -14,8 +14,9 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.core.License; -import org.elasticsearch.license.plugin.core.LicensesClientService; -import org.elasticsearch.license.plugin.core.LicensesService; +import org.elasticsearch.license.plugin.core.LicenseState; +import org.elasticsearch.license.plugin.core.Licensee; +import org.elasticsearch.license.plugin.core.LicenseeRegistry; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.watcher.actions.ActionStatus; @@ -46,15 +47,13 @@ import static org.hamcrest.Matchers.*; public class LicenseIntegrationTests extends AbstractWatcherIntegrationTestCase { static final License DUMMY_LICENSE = License.builder() - .feature(LicenseService.FEATURE_NAME) .expiryDate(System.currentTimeMillis()) .issueDate(System.currentTimeMillis()) .issuedTo("LicensingTests") .issuer("test") .maxNodes(Integer.MAX_VALUE) .signature("_signature") - .type("test_license_for_watcher") - .subscriptionType("all_is_good") + .type("basic") .uid(String.valueOf(RandomizedTest.systemPropertyAsInt(SysGlobals.CHILDVM_SYSPROP_JVM_ID, 0)) + System.identityHashCode(LicenseIntegrationTests.class)) .build(); @@ -301,13 +300,13 @@ public class LicenseIntegrationTests extends AbstractWatcherIntegrationTestCase @Override protected void configure() { bind(MockLicenseService.class).asEagerSingleton(); - bind(LicensesClientService.class).to(MockLicenseService.class); + bind(LicenseeRegistry.class).to(MockLicenseService.class); } } - public static class MockLicenseService extends AbstractComponent implements LicensesClientService { + public static class MockLicenseService extends AbstractComponent implements LicenseeRegistry { - private final List listeners = new ArrayList<>(); + private final List licensees = new ArrayList<>(); @Inject public MockLicenseService(Settings settings) { @@ -316,23 +315,23 @@ public class LicenseIntegrationTests extends AbstractWatcherIntegrationTestCase } @Override - public void register(String s, LicensesService.TrialLicenseOptions trialLicenseOptions, Collection collection, AcknowledgementCallback acknowledgementCallback, Listener listener) { - listeners.add(listener); + public void register(Licensee licensee) { + licensees.add(licensee); enable(); } public void enable() { // enabled all listeners (incl. shield) - for (Listener listener : listeners) { - listener.onEnabled(DUMMY_LICENSE); + for (Licensee licensee : licensees) { + licensee.onChange(DUMMY_LICENSE, LicenseState.ENABLED); } } public void disable() { // only disable watcher listener (we need shield to work) - for (Listener listener : listeners) { - if (listener instanceof LicenseService.InternalListener) { - listener.onDisabled(DUMMY_LICENSE); + for (Licensee licensee : licensees) { + if (licensee instanceof LicenseService) { + licensee.onChange(DUMMY_LICENSE, LicenseState.DISABLED); } } } From 0ebc6198ac1b06c684e4aeed753f11ecc7e7d115 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 7 Oct 2015 11:43:37 +0200 Subject: [PATCH 11/12] Marvel: Update Shard mapping Closes elastic/elasticsearch#751 Original commit: elastic/x-pack-elasticsearch@af4276785c6411116f735dbfce5dce0ece923b48 --- .../main/resources/marvel_index_template.json | 25 ++++++++++- .../agent/renderer/shards/ShardsIT.java | 44 ++++++++++++++++++- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/marvel/src/main/resources/marvel_index_template.json b/marvel/src/main/resources/marvel_index_template.json index 672f82884ac..51db87307c8 100644 --- a/marvel/src/main/resources/marvel_index_template.json +++ b/marvel/src/main/resources/marvel_index_template.json @@ -252,7 +252,30 @@ "index": "not_analyzed" }, "shard": { - "type": "object" + "properties": { + "state": { + "type": "string", + "index": "not_analyzed" + }, + "primary": { + "type": "boolean" + }, + "index": { + "type": "string", + "index": "not_analyzed" + }, + "relocating_node": { + "type": "string", + "index": "not_analyzed" + }, + "shard": { + "type": "long" + }, + "node": { + "type": "string", + "index": "not_analyzed" + } + } } } } diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsIT.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsIT.java index 02f1e42286a..ba2416203c0 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsIT.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsIT.java @@ -6,27 +6,37 @@ package org.elasticsearch.marvel.agent.renderer.shards; import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.marvel.agent.collector.shards.ShardsCollector; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.junit.Test; import java.util.Map; -import static org.hamcrest.Matchers.greaterThan; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.*; @AwaitsFix(bugUrl="https://github.com/elastic/x-plugins/issues/729") public class ShardsIT extends MarvelIntegTestCase { + private static final String INDEX_PREFIX = "test-shards-"; + @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(MarvelSettings.INTERVAL, "3s") .put(MarvelSettings.COLLECTORS, ShardsCollector.NAME) + .put(MarvelSettings.INDICES, INDEX_PREFIX + "*") .build(); } @@ -34,7 +44,7 @@ public class ShardsIT extends MarvelIntegTestCase { public void testShards() throws Exception { logger.debug("--> creating some indices so that shards collector reports data"); for (int i = 0; i < randomIntBetween(1, 5); i++) { - client().prepareIndex("test-" + i, "foo").setRefresh(true).setSource("field1", "value1").get(); + client().prepareIndex(INDEX_PREFIX + i, "foo").setRefresh(true).setSource("field1", "value1").get(); } awaitMarvelDocsCount(greaterThan(0L), ShardsCollector.TYPE); @@ -55,4 +65,34 @@ public class ShardsIT extends MarvelIntegTestCase { logger.debug("--> shards successfully collected"); } + + /** + * This test uses a terms aggregation to check that the "not_analyzed" + * fields of the "shards" document type are indeed not analyzed + */ + @Test + public void testNotAnalyzedFields() throws Exception { + final String indexName = INDEX_PREFIX + randomInt(); + assertAcked(prepareCreate(indexName).setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); + + awaitMarvelDocsCount(greaterThan(0L), ShardsCollector.TYPE); + + SearchRequestBuilder searchRequestBuilder = client() + .prepareSearch() + .setTypes(ShardsCollector.TYPE) + .setQuery(QueryBuilders.termQuery("shard.index", indexName)); + + String[] notAnalyzedFields = {"state_uuid", "shard.state", "shard.index", "shard.node"}; + for (String field : notAnalyzedFields) { + searchRequestBuilder.addAggregation(AggregationBuilders.terms("agg_" + field.replace('.', '_')).field(field)); + } + + SearchResponse response = searchRequestBuilder.get(); + assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1L)); + + for (Aggregation aggregation : response.getAggregations()) { + assertThat(aggregation, instanceOf(StringTerms.class)); + assertThat(((StringTerms) aggregation).getBuckets().size(), equalTo(1)); + } + } } From 2f1c88a6331cd9e618e9e66beffa808f7e8706f5 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 6 Oct 2015 20:12:56 +0200 Subject: [PATCH 12/12] Marvel: Use mock web server in HttpExporterTests Original commit: elastic/x-pack-elasticsearch@b69b28af903eeb302b983cf230a60f03856ada7f --- marvel/pom.xml | 7 + .../agent/exporter/http/HttpExporter.java | 11 +- .../exporter/http/HttpExporterTests.java | 405 ++++++++++-------- 3 files changed, 245 insertions(+), 178 deletions(-) diff --git a/marvel/pom.xml b/marvel/pom.xml index e6f244edb62..7b846a967c2 100644 --- a/marvel/pom.xml +++ b/marvel/pom.xml @@ -42,6 +42,13 @@ provided + + com.squareup.okhttp + mockwebserver + 2.3.0 + test + + diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java index f04dcca8cb5..43eed3af50f 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java @@ -54,6 +54,7 @@ public class HttpExporter extends Exporter { public static final String HOST_SETTING = "host"; public static final String CONNECTION_TIMEOUT_SETTING = "connection.timeout"; public static final String CONNECTION_READ_TIMEOUT_SETTING = "connection.read_timeout"; + public static final String CONNECTION_KEEP_ALIVE_SETTING = "connection.keep_alive"; public static final String AUTH_USERNAME_SETTING = "auth.username"; public static final String AUTH_PASSWORD_SETTING = "auth.password"; @@ -91,6 +92,7 @@ public class HttpExporter extends Exporter { /** Version of the built-in template **/ final Version templateVersion; + boolean keepAlive; final ConnectionKeepAliveWorker keepAliveWorker; Thread keepAliveThread; @@ -117,6 +119,7 @@ public class HttpExporter extends Exporter { String templateCheckTimeoutValue = config.settings().get(TEMPLATE_CHECK_TIMEOUT_SETTING, null); templateCheckTimeout = TimeValue.parseTimeValue(templateCheckTimeoutValue, null, settingFQN(TEMPLATE_CHECK_TIMEOUT_SETTING)); + keepAlive = config.settings().getAsBoolean(CONNECTION_KEEP_ALIVE_SETTING, true); keepAliveWorker = new ConnectionKeepAliveWorker(); sslSocketFactory = createSSLSocketFactory(config.settings().getAsSettings(SSL_SETTING)); @@ -511,9 +514,11 @@ public class HttpExporter extends Exporter { } protected void initKeepAliveThread() { - keepAliveThread = new Thread(keepAliveWorker, "marvel-exporter[" + config.name() + "][keep_alive]"); - keepAliveThread.setDaemon(true); - keepAliveThread.start(); + if (keepAlive) { + keepAliveThread = new Thread(keepAliveWorker, "marvel-exporter[" + config.name() + "][keep_alive]"); + keepAliveThread.setDaemon(true); + keepAliveThread.start(); + } } diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java index 0ff26b351b1..f95f8b3f30b 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java @@ -5,134 +5,114 @@ */ package org.elasticsearch.marvel.agent.exporter.http; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.MockWebServer; +import com.squareup.okhttp.mockwebserver.QueueDispatcher; +import com.squareup.okhttp.mockwebserver.RecordedRequest; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateMarvelDoc; import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryCollector; import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryMarvelDoc; import org.elasticsearch.marvel.agent.exporter.Exporters; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; +import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.test.MarvelIntegTestCase; -import org.elasticsearch.node.Node; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.ESIntegTestCase.SuppressLocalMode; -import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.hamcrest.Matchers; import org.joda.time.format.DateTimeFormat; import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.net.BindException; import java.util.Collections; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; +import java.util.Map; -import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; - - -// Transport Client instantiation also calls the marvel plugin, which then fails to find modules -@SuppressLocalMode -@ClusterScope(scope = TEST, transportClientRatio = 0.0, numDataNodes = 0, numClientNodes = 0) -@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/729") +@ESIntegTestCase.ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class HttpExporterTests extends MarvelIntegTestCase { - final static AtomicLong timeStampGenerator = new AtomicLong(); - - @Override - protected boolean enableShield() { - return false; - } + private int webPort; + private MockWebServer webServer; @Before - public void init() throws Exception { - startCollection(); + public void startWebservice() throws Exception { + for (webPort = 9250; webPort < 9300; webPort++) { + try { + webServer = new MockWebServer(); + QueueDispatcher dispatcher = new QueueDispatcher(); + dispatcher.setFailFast(true); + webServer.setDispatcher(dispatcher); + webServer.start(webPort); + return; + } catch (BindException be) { + logger.warn("port [{}] was already in use trying next port", webPort); + } + } + throw new ElasticsearchException("unable to find open port between 9200 and 9300"); } @After public void cleanup() throws Exception { stopCollection(); - } - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(Node.HTTP_ENABLED, true) - .put("shield.enabled", false) - .build(); + webServer.shutdown(); } @Test - public void testSimpleExport() throws Exception { - TargetNode target = TargetNode.start(internalCluster()); + public void testExport() throws Exception { + enqueueGetClusterVersionResponse(Version.CURRENT); + enqueueResponse(404, "marvel template does not exist"); + enqueueResponse(201, "marvel template created"); + enqueueResponse(200, "successful bulk request "); Settings.Builder builder = Settings.builder() + .put(MarvelSettings.INTERVAL, "-1") .put("marvel.agent.exporters._http.type", "http") - .put("marvel.agent.exporters._http.host", target.httpAddress); + .put("marvel.agent.exporters._http.host", webServer.getHostName() + ":" + webServer.getPort()) + .put("marvel.agent.exporters._http.connection.keep_alive", false); + String agentNode = internalCluster().startNode(builder); - ensureGreen(); HttpExporter exporter = getExporter(agentNode); MarvelDoc doc = newRandomMarvelDoc(); exporter.export(Collections.singletonList(doc)); - flush(); - refresh(); + assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(4)); - SearchResponse response = client().prepareSearch(".marvel-es-*").setTypes(doc.type()).get(); - assertThat(response, notNullValue()); - assertThat(response.getHits().totalHits(), is(1L)); - } + RecordedRequest recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/")); - @Test - public void testTemplateAdditionDespiteOfLateClusterForming() throws Exception { + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/_template/marvel")); - TargetNode target = TargetNode.start(internalCluster()); + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("PUT")); + assertThat(recordedRequest.getPath(), equalTo("/_template/marvel")); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); - Settings.Builder builder = Settings.builder() - .put(MarvelSettings.STARTUP_DELAY, "200m") - .put(Node.HTTP_ENABLED, true) - .put("discovery.type", "zen") - .put("discovery.zen.ping_timeout", "1s") - .put("discovery.initial_state_timeout", "100ms") - .put("discovery.zen.minimum_master_nodes", 2) - .put("marvel.agent.exporters._http.type", "http") - .put("marvel.agent.exporters._http.host", target.httpAddress) - .put("marvel.agent.exporters._http." + HttpExporter.BULK_TIMEOUT_SETTING, "1s") - .put("marvel.agent.exporters._http." + HttpExporter.TEMPLATE_CHECK_TIMEOUT_SETTING, "1s"); - - String nodeName = internalCluster().startNode(builder); - - HttpExporter exporter = getExporter(nodeName); - logger.info("exporting events while there is no cluster"); - exporter.export(Collections.singletonList(newRandomMarvelDoc())); - - logger.info("bringing up a second node"); - internalCluster().startNode(builder); - ensureGreen(); - logger.info("exporting a second event"); - exporter.export(Collections.singletonList(newRandomMarvelDoc())); - - logger.info("verifying that template has been created"); - assertMarvelTemplateInstalled(); + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("POST")); + assertThat(recordedRequest.getPath(), equalTo("/_bulk")); } @Test public void testDynamicHostChange() { - // disable exporting to be able to use non valid hosts Settings.Builder builder = Settings.builder() .put(MarvelSettings.INTERVAL, "-1") @@ -159,144 +139,222 @@ public class HttpExporterTests extends MarvelIntegTestCase { @Test public void testHostChangeReChecksTemplate() throws Exception { - TargetNode targetNode = TargetNode.start(internalCluster()); - Settings.Builder builder = Settings.builder() - .put(MarvelSettings.STARTUP_DELAY, "200m") + .put(MarvelSettings.INTERVAL, "-1") .put("marvel.agent.exporters._http.type", "http") - .put("marvel.agent.exporters._http.host", targetNode.httpAddress); + .put("marvel.agent.exporters._http.host", webServer.getHostName() + ":" + webServer.getPort()) + .put("marvel.agent.exporters._http.connection.keep_alive", false); + + logger.info("--> starting node"); + + enqueueGetClusterVersionResponse(Version.CURRENT); + enqueueResponse(404, "marvel template does not exist"); + enqueueResponse(201, "marvel template created"); + enqueueResponse(200, "successful bulk request "); String agentNode = internalCluster().startNode(builder); + logger.info("--> exporting data"); HttpExporter exporter = getExporter(agentNode); - - logger.info("exporting an event"); exporter.export(Collections.singletonList(newRandomMarvelDoc())); - logger.info("removing the marvel template"); - assertAcked(client().admin().indices().prepareDeleteTemplate("marvel").get()); - assertMarvelTemplateMissing(); + assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(4)); - assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings( - Settings.builder().putArray("marvel.agent.exporters._http.host", exporter.hosts)).get()); + RecordedRequest recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/")); - // a new exporter is created on update, so we need to re-fetch it - exporter = getExporter(agentNode); + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/_template/marvel")); - logger.info("exporting a second event"); - exporter.export(Collections.singletonList(newRandomMarvelDoc())); + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("PUT")); + assertThat(recordedRequest.getPath(), equalTo("/_template/marvel")); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); - logger.info("verifying that template has been created"); - assertMarvelTemplateInstalled(); - } + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("POST")); + assertThat(recordedRequest.getPath(), equalTo("/_bulk")); - @Test - public void testHostFailureChecksTemplate() throws Exception { + logger.info("--> setting up another web server"); + MockWebServer secondWebServer = null; + int secondWebPort; - TargetNode target0 = TargetNode.start(internalCluster()); - assertThat(target0.name, is(internalCluster().getMasterName())); - - TargetNode target1 = TargetNode.start(internalCluster()); - - // lets start node0 & node1 first, such that node0 will be the master (it's first to start) - final String node0 = internalCluster().startNode(Settings.builder() - .put(MarvelSettings.STARTUP_DELAY, "200m") - .put("marvel.agent.exporters._http.type", "http") - .putArray("marvel.agent.exporters._http.host", target0.httpAddress, target1.httpAddress)); - - HttpExporter exporter = getExporter(node0); - - logger.info("--> exporting events to have new settings take effect"); - exporter.export(Collections.singletonList(newRandomMarvelDoc())); - - logger.info("verifying that template has been created"); - assertMarvelTemplateInstalled(); - - logger.info("--> removing the marvel template"); - assertAcked(client().admin().indices().prepareDeleteTemplate("marvel").get()); - assertMarvelTemplateMissing(); - - logger.info("--> shutting down target0"); - assertThat(target0.name, is(internalCluster().getMasterName())); // just to be sure it's still the master - internalCluster().stopCurrentMasterNode(); - - // we use assert busy node because url caching may cause the node failure to be only detected while sending the event - assertBusy(new Runnable() { - @Override - public void run() { + try { + for (secondWebPort = 9250; secondWebPort < 9300; secondWebPort++) { try { - logger.info("--> exporting events from node0"); - getExporter(node0).export(Collections.singletonList(newRandomMarvelDoc())); - } catch (Exception e) { - e.printStackTrace(); - fail("failed to export event from node0"); + secondWebServer = new MockWebServer(); + QueueDispatcher dispatcher = new QueueDispatcher(); + dispatcher.setFailFast(true); + secondWebServer.setDispatcher(dispatcher); + secondWebServer.start(secondWebPort); + break; + } catch (BindException be) { + logger.warn("port [{}] was already in use trying next port", secondWebPort); } - logger.debug("--> checking for template"); - assertMarvelTemplateInstalled(); - logger.debug("--> template exists"); } - }, 30, TimeUnit.SECONDS); + + assertNotNull("Unable to start the second mock web server", secondWebServer); + + assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings( + Settings.builder().putArray("marvel.agent.exporters._http.host", secondWebServer.getHostName() + ":" + secondWebServer.getPort())).get()); + + // a new exporter is created on update, so we need to re-fetch it + exporter = getExporter(agentNode); + + enqueueGetClusterVersionResponse(secondWebServer, Version.CURRENT); + enqueueResponse(secondWebServer, 404, "marvel template does not exist"); + enqueueResponse(secondWebServer, 201, "marvel template created"); + enqueueResponse(secondWebServer, 200, "successful bulk request "); + + logger.info("--> exporting a second event"); + exporter.export(Collections.singletonList(newRandomMarvelDoc())); + + assertThat(secondWebServer.getRequestCount(), greaterThanOrEqualTo(4)); + + recordedRequest = secondWebServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/")); + + recordedRequest = secondWebServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/_template/marvel")); + + recordedRequest = secondWebServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("PUT")); + assertThat(recordedRequest.getPath(), equalTo("/_template/marvel")); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); + + recordedRequest = secondWebServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("POST")); + assertThat(recordedRequest.getPath(), equalTo("/_bulk")); + + } finally { + if (secondWebServer != null) { + secondWebServer.shutdown(); + } + } } @Test public void testDynamicIndexFormatChange() throws Exception { - TargetNode targetNode = TargetNode.start(internalCluster()); - Settings.Builder builder = Settings.builder() - .put(MarvelSettings.STARTUP_DELAY, "200m") + .put(MarvelSettings.INTERVAL, "-1") .put("marvel.agent.exporters._http.type", "http") - .put("marvel.agent.exporters._http.host", targetNode.httpAddress); + .put("marvel.agent.exporters._http.host", webServer.getHostName() + ":" + webServer.getPort()) + .put("marvel.agent.exporters._http.connection.keep_alive", false); String agentNode = internalCluster().startNode(builder); - logger.info("exporting a first event"); + logger.info("--> exporting a first event"); + + enqueueGetClusterVersionResponse(Version.CURRENT); + enqueueResponse(404, "marvel template does not exist"); + enqueueResponse(201, "marvel template created"); + enqueueResponse(200, "successful bulk request "); + HttpExporter exporter = getExporter(agentNode); + MarvelDoc doc = newRandomMarvelDoc(); exporter.export(Collections.singletonList(doc)); + assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(4)); + + RecordedRequest recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/")); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/_template/marvel")); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("PUT")); + assertThat(recordedRequest.getPath(), equalTo("/_template/marvel")); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("POST")); + assertThat(recordedRequest.getPath(), equalTo("/_bulk")); + String indexName = exporter.indexNameResolver().resolve(doc); - logger.info("checks that the index [{}] is created", indexName); - assertTrue(client().admin().indices().prepareExists(indexName).get().isExists()); + logger.info("--> checks that the document in the bulk request is indexed in [{}]", indexName); + + byte[] bytes = recordedRequest.getBody().readByteArray(); + Map data = XContentHelper.convertToMap(new BytesArray(bytes), false).v2(); + Map index = (Map) data.get("index"); + assertThat(index.get("_index"), equalTo(indexName)); String newTimeFormat = randomFrom("YY", "YYYY", "YYYY.MM", "YYYY-MM", "MM.YYYY", "MM"); - logger.info("updating index time format setting to {}", newTimeFormat); + logger.info("--> updating index time format setting to {}", newTimeFormat); assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder() .put("marvel.agent.exporters._http.index.name.time_format", newTimeFormat))); - exporter = getExporter(agentNode); - logger.info("exporting a second event"); + logger.info("--> exporting a second event"); + + enqueueGetClusterVersionResponse(Version.CURRENT); + enqueueResponse(404, "marvel template does not exist"); + enqueueResponse(201, "marvel template created"); + enqueueResponse(200, "successful bulk request "); + doc = newRandomMarvelDoc(); + exporter = getExporter(agentNode); exporter.export(Collections.singletonList(doc)); String expectedMarvelIndex = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(newTimeFormat).withZoneUTC().print(doc.timestamp()); - logger.info("checks that the index [{}] is created", expectedMarvelIndex); - assertTrue(client().admin().indices().prepareExists(expectedMarvelIndex).get().isExists()); + assertThat(webServer.getRequestCount(), greaterThanOrEqualTo(4)); - logger.info("verifying that template has been created"); - assertMarvelTemplateInstalled(); + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/")); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/_template/marvel")); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("PUT")); + assertThat(recordedRequest.getPath(), equalTo("/_template/marvel")); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("POST")); + assertThat(recordedRequest.getPath(), equalTo("/_bulk")); + + logger.info("--> checks that the document in the bulk request is indexed in [{}]", expectedMarvelIndex); + + bytes = recordedRequest.getBody().readByteArray(); + data = XContentHelper.convertToMap(new BytesArray(bytes), false).v2(); + index = (Map) data.get("index"); + assertThat(index.get("_index"), equalTo(expectedMarvelIndex)); } @Test - public void testLoadRemoteClusterVersion() { - - TargetNode targetNode = TargetNode.start(internalCluster()); + public void testLoadRemoteClusterVersion() throws IOException { + final String host = webServer.getHostName() + ":" + webServer.getPort(); Settings.Builder builder = Settings.builder() - .put(MarvelSettings.STARTUP_DELAY, "200m") + .put(MarvelSettings.INTERVAL, "-1") .put("marvel.agent.exporters._http.type", "http") - .put("marvel.agent.exporters._http.host", targetNode.httpAddress); + .put("marvel.agent.exporters._http.host", host) + .put("marvel.agent.exporters._http.connection.keep_alive", false); String agentNode = internalCluster().startNode(builder); - HttpExporter exporter = getExporter(agentNode); - logger.info("--> loading remote cluster version"); - Version resolved = exporter.loadRemoteClusterVersion(targetNode.httpAddress); + enqueueGetClusterVersionResponse(Version.CURRENT); + Version resolved = exporter.loadRemoteClusterVersion(host); assertTrue(resolved.equals(Version.CURRENT)); + + final Version expected = randomFrom(Version.CURRENT, Version.V_0_18_0, Version.V_1_1_0, Version.V_1_2_5, Version.V_1_4_5, Version.V_1_6_0); + enqueueGetClusterVersionResponse(expected); + resolved = exporter.loadRemoteClusterVersion(host); + assertTrue(resolved.equals(expected)); } private HttpExporter getExporter(String nodeName) { @@ -307,29 +365,26 @@ public class HttpExporterTests extends MarvelIntegTestCase { private MarvelDoc newRandomMarvelDoc() { if (randomBoolean()) { return new IndexRecoveryMarvelDoc(internalCluster().getClusterName(), - IndexRecoveryCollector.TYPE, timeStampGenerator.incrementAndGet(), new RecoveryResponse()); + IndexRecoveryCollector.TYPE, System.currentTimeMillis(), new RecoveryResponse()); } else { return new ClusterStateMarvelDoc(internalCluster().getClusterName(), - ClusterStateCollector.TYPE, timeStampGenerator.incrementAndGet(), ClusterState.PROTO, ClusterHealthStatus.GREEN); + ClusterStateCollector.TYPE, System.currentTimeMillis(), ClusterState.PROTO, ClusterHealthStatus.GREEN); } } - static class TargetNode { + private void enqueueGetClusterVersionResponse(Version v) throws IOException { + enqueueGetClusterVersionResponse(webServer, v); + } - private final String name; - private final TransportAddress address; - private final String httpAddress; - private final Client client; + private void enqueueGetClusterVersionResponse(MockWebServer mockWebServer, Version v) throws IOException { + mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody(jsonBuilder().startObject().startObject("version").field("number", v.number()).endObject().endObject().bytes().toUtf8())); + } - private TargetNode(InternalTestCluster cluster) { - name = cluster.startNode(Settings.builder().put(Node.HTTP_ENABLED, true)); - address = cluster.getInstance(HttpServerTransport.class, name).boundAddress().publishAddress(); - httpAddress = address.getHost() + ":" + address.getPort(); - this.client = cluster.client(name); - } + private void enqueueResponse(int responseCode, String body) throws IOException { + enqueueResponse(webServer, responseCode, body); + } - static TargetNode start(InternalTestCluster cluster) { - return new TargetNode(cluster); - } + private void enqueueResponse(MockWebServer mockWebServer, int responseCode, String body) throws IOException { + mockWebServer.enqueue(new MockResponse().setResponseCode(responseCode).setBody(body)); } }