Marvel: Add Licenses collector to ship licensing information into a dedicated index

Closes elastic/elasticsearch#369

squash

Original commit: elastic/x-pack-elasticsearch@6d6d5c08d2

marvel/src/main/java/org/elasticsearch/marvel/agent/AgentService.java

@@ -13,6 +13,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.marvel.agent.collector.Collector;
+import org.elasticsearch.marvel.agent.collector.licenses.LicensesCollector;
 import org.elasticsearch.marvel.agent.exporter.Exporter;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
@@ -57,11 +58,8 @@ public class AgentService extends AbstractLifecycleComponent<AgentService> imple
         for (Collector collector : collectors) {
             if (Regex.simpleMatch(filters, collector.name().toLowerCase(Locale.ROOT))) {
                 list.add(collector);
-
-            /* TODO Always add license collector
             } else if (collector instanceof LicensesCollector) {
                 list.add(collector);
-             */
             }
         }
         return list;

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/CollectorModule.java

@@ -11,6 +11,7 @@ import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector;
 import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector;
 import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryCollector;
 import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector;
+import org.elasticsearch.marvel.agent.collector.licenses.LicensesCollector;
 import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
 
 import java.util.HashSet;
@@ -22,6 +23,7 @@ public class CollectorModule extends AbstractModule {
 
     public CollectorModule() {
         // Registers default collectors
+        registerCollector(LicensesCollector.class);
         registerCollector(IndexStatsCollector.class);
         registerCollector(ClusterStatsCollector.class);
         registerCollector(ClusterStateCollector.class);

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/licenses/LicensesCollector.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.agent.collector.licenses;
+
+import com.google.common.collect.ImmutableList;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.core.License;
+import org.elasticsearch.marvel.agent.collector.AbstractCollector;
+import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
+import org.elasticsearch.marvel.agent.settings.MarvelSettings;
+import org.elasticsearch.marvel.license.LicenseService;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Collector for registered licenses.
+ * <p/>
+ * This collector runs on the master node and collect data about all
+ * known licenses that are currently registered. Each license is
+ * collected as a {@link LicensesMarvelDoc} document.
+ */
+public class LicensesCollector extends AbstractCollector<LicensesMarvelDoc> {
+
+    public static final String NAME = "licenses-collector";
+    public static final String TYPE = "cluster_licenses";
+
+    private final LicenseService licenseService;
+
+    @Inject
+    public LicensesCollector(Settings settings, ClusterService clusterService, ClusterName clusterName,
+                             MarvelSettings marvelSettings,
+                             LicenseService licenseService) {
+        super(settings, NAME, clusterService, clusterName, marvelSettings);
+        this.licenseService = licenseService;
+    }
+
+    @Override
+    protected Collection<MarvelDoc> doCollect() throws Exception {
+        ImmutableList.Builder<MarvelDoc> results = ImmutableList.builder();
+
+        List<License> licenses = licenseService.licenses();
+        if (licenses != null) {
+            results.add(LicensesMarvelDoc.createMarvelDoc(MarvelSettings.MARVEL_DATA_INDEX_NAME, TYPE, clusterName.value(), clusterName.value(),
+                    System.currentTimeMillis(), Version.CURRENT.toString(), licenses));
+        }
+        return results.build();
+    }
+}

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/licenses/LicensesMarvelDoc.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.agent.collector.licenses;
+
+import org.elasticsearch.license.core.License;
+import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
+
+import java.util.List;
+
+public class LicensesMarvelDoc extends MarvelDoc<LicensesMarvelDoc.Payload> {
+
+    private final Payload payload;
+
+    LicensesMarvelDoc(String index, String type, String id, String clusterName, long timestamp, Payload payload) {
+        super(index, type, id, clusterName, timestamp);
+        this.payload = payload;
+    }
+
+    @Override
+    public LicensesMarvelDoc.Payload payload() {
+        return payload;
+    }
+
+    public static LicensesMarvelDoc createMarvelDoc(String index, String type, String id, String clusterName, long timestamp, String version, List<License> licenses) {
+        return new LicensesMarvelDoc(index, type, id, clusterName, timestamp, new Payload(version, licenses));
+    }
+
+    public static class Payload {
+
+        private final String version;
+        private final List<License> licenses;
+
+        public Payload(String version, List<License> licenses) {
+            this.version = version;
+            this.licenses = licenses;
+        }
+
+        public String getVersion() {
+            return version;
+        }
+
+        public List<License> getLicenses() {
+            return licenses;
+        }
+    }
+}

marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/HttpESExporter.java

@@ -195,7 +195,17 @@ public class HttpESExporter extends AbstractExporter<HttpESExporter> implements
 
             // Builds the bulk action metadata line
             builder.startObject();
-            builder.startObject("index").field("_type", marvelDoc.type()).endObject();
+            builder.startObject("index");
+            if (marvelDoc.index() != null) {
+                builder.field("_index", marvelDoc.index());
+            }
+            if (marvelDoc.type() != null) {
+                builder.field("_type", marvelDoc.type());
+            }
+            if (marvelDoc.id() != null) {
+                builder.field("_id", marvelDoc.id());
+            }
+            builder.endObject();
             builder.endObject();
 
             // Adds action metadata line bulk separator

marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelDoc.java

@@ -7,24 +7,41 @@ package org.elasticsearch.marvel.agent.exporter;
 
 public abstract class MarvelDoc<T> {
 
-    private final String clusterName;
+    private final String index;
     private final String type;
+    private final String id;
+
+    private final String clusterName;
     private final long timestamp;
 
-    public MarvelDoc(String clusterName, String type, long timestamp) {
+    public MarvelDoc(String index, String type, String id, String clusterName, long timestamp) {
-        this.clusterName = clusterName;
+        this.index = index;
         this.type = type;
+        this.id = id;
+        this.clusterName = clusterName;
         this.timestamp = timestamp;
     }
 
+    public MarvelDoc(String clusterName, String type, long timestamp) {
+        this(null, type, null, clusterName, timestamp);
+    }
+
     public String clusterName() {
         return clusterName;
     }
 
+    public String index() {
+        return index;
+    }
+
     public String type() {
         return type;
     }
 
+    public String id() {
+        return id;
+    }
+
     public long timestamp() {
         return timestamp;
     }

marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java

@@ -11,11 +11,13 @@ import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector;
 import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector;
 import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryCollector;
 import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector;
+import org.elasticsearch.marvel.agent.collector.licenses.LicensesCollector;
 import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
 import org.elasticsearch.marvel.agent.renderer.cluster.ClusterStateRenderer;
 import org.elasticsearch.marvel.agent.renderer.cluster.ClusterStatsRenderer;
 import org.elasticsearch.marvel.agent.renderer.indices.IndexRecoveryRenderer;
 import org.elasticsearch.marvel.agent.renderer.indices.IndexStatsRenderer;
+import org.elasticsearch.marvel.agent.renderer.licenses.LicensesRenderer;
 import org.elasticsearch.marvel.agent.renderer.node.NodeStatsRenderer;
 
 import java.util.HashMap;
@@ -34,6 +36,9 @@ public class RendererModule extends AbstractModule {
         MapBinder<String, Renderer> mbinder = MapBinder.newMapBinder(binder(), String.class, Renderer.class);
 
         // Bind default renderers
+        bind(LicensesRenderer.class).asEagerSingleton();
+        mbinder.addBinding(LicensesCollector.TYPE).to(LicensesRenderer.class);
+
         bind(IndexStatsRenderer.class).asEagerSingleton();
         mbinder.addBinding(IndexStatsCollector.TYPE).to(IndexStatsRenderer.class);
 

marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/licenses/LicensesRenderer.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.agent.renderer.licenses;
+
+import com.google.common.base.Charsets;
+import com.google.common.hash.Hashing;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
+import org.elasticsearch.license.core.License;
+import org.elasticsearch.marvel.agent.collector.licenses.LicensesMarvelDoc;
+import org.elasticsearch.marvel.agent.renderer.AbstractRenderer;
+
+import java.io.IOException;
+import java.util.List;
+
+public class LicensesRenderer extends AbstractRenderer<LicensesMarvelDoc> {
+
+    public LicensesRenderer() {
+        super(Strings.EMPTY_ARRAY, false);
+    }
+
+    @Override
+    protected void doRender(LicensesMarvelDoc marvelDoc, XContentBuilder builder, ToXContent.Params params) throws IOException {
+        LicensesMarvelDoc.Payload payload = marvelDoc.payload();
+
+        if (payload != null) {
+            builder.field(Fields.VERSION, payload.getVersion());
+            builder.startArray(Fields.LICENSES);
+
+            List<License> licenses = payload.getLicenses();
+            if (licenses != null) {
+                for (License license : licenses) {
+                    builder.startObject();
+                    builder.field(Fields.STATUS, status(license));
+                    builder.field(Fields.UID, license.uid());
+                    builder.field(Fields.TYPE, license.type());
+                    builder.dateValueField(Fields.ISSUE_DATE_IN_MILLIS, Fields.ISSUE_DATE, license.issueDate());
+                    builder.field(Fields.FEATURE, license.feature());
+                    builder.dateValueField(Fields.EXPIRY_DATE_IN_MILLIS, Fields.EXPIRY_DATE, license.expiryDate());
+                    builder.field(Fields.MAX_NODES, license.maxNodes());
+                    builder.field(Fields.ISSUED_TO, license.issuedTo());
+                    builder.field(Fields.ISSUER, license.issuer());
+                    builder.field(Fields.HKEY, hash(license, marvelDoc.clusterName()));
+                    builder.endObject();
+                }
+            }
+            builder.endArray();
+        }
+    }
+
+    // TODO (tlrx): move status as a calculated getter in License class then remove this method
+    public static String status(License license) {
+        String status = "active";
+        long now = System.currentTimeMillis();
+        if (license.issueDate() > now) {
+            status = "invalid";
+        } else if (license.expiryDate() < now) {
+            status = "expired";
+        }
+        return status;
+    }
+
+    public static String hash(License license, String clusterName) {
+        String toHash = status(license) + license.uid() + license.type() + String.valueOf(license.expiryDate()) + clusterName;
+        return Hashing.sha256().hashString(toHash, Charsets.UTF_8).toString();
+    }
+
+    static final class Fields {
+        static final XContentBuilderString LICENSES = new XContentBuilderString("licenses");
+        static final XContentBuilderString VERSION = new XContentBuilderString("version");
+
+        static final XContentBuilderString HKEY = new XContentBuilderString("hkey");
+
+        static final XContentBuilderString STATUS = new XContentBuilderString("status");
+        static final XContentBuilderString UID = new XContentBuilderString("uid");
+        static final XContentBuilderString TYPE = new XContentBuilderString("type");
+        static final XContentBuilderString FEATURE = new XContentBuilderString("feature");
+        static final XContentBuilderString ISSUE_DATE_IN_MILLIS = new XContentBuilderString("issue_date_in_millis");
+        static final XContentBuilderString ISSUE_DATE = new XContentBuilderString("issue_date");
+        static final XContentBuilderString EXPIRY_DATE_IN_MILLIS = new XContentBuilderString("expiry_date_in_millis");
+        static final XContentBuilderString EXPIRY_DATE = new XContentBuilderString("expiry_date");
+        static final XContentBuilderString MAX_NODES = new XContentBuilderString("max_nodes");
+        static final XContentBuilderString ISSUED_TO = new XContentBuilderString("issued_to");
+        static final XContentBuilderString ISSUER = new XContentBuilderString("issuer");
+    }
+}

marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java

@@ -21,6 +21,8 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
 
     private static final String PREFIX = MarvelPlugin.NAME + ".agent.";
 
+    public static final String MARVEL_DATA_INDEX_NAME = ".marvel-data";
+
     public static final String INTERVAL                     = PREFIX + "interval";
     public static final String STARTUP_DELAY                = PREFIX + "startup.delay";
     public static final String INDEX_STATS_TIMEOUT          = PREFIX + "index.stats.timeout";

marvel/src/main/java/org/elasticsearch/marvel/license/LicenseService.java

@@ -14,12 +14,14 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.license.core.License;
 import org.elasticsearch.license.plugin.core.LicensesClientService;
+import org.elasticsearch.license.plugin.core.LicensesManagerService;
 import org.elasticsearch.license.plugin.core.LicensesService;
 import org.elasticsearch.marvel.MarvelPlugin;
 import org.elasticsearch.marvel.mode.Mode;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 import java.util.Locale;
 
 public class LicenseService extends AbstractLifecycleComponent<LicenseService> {
@@ -31,14 +33,16 @@ public class LicenseService extends AbstractLifecycleComponent<LicenseService> {
 
     private static final FormatDateTimeFormatter DATE_FORMATTER = Joda.forPattern("EEEE, MMMMM dd, yyyy", Locale.ROOT);
 
+    private final LicensesManagerService managerService;
     private final LicensesClientService clientService;
     private final Collection<LicensesService.ExpirationCallback> expirationLoggers;
 
     private volatile Mode mode;
 
     @Inject
-    public LicenseService(Settings settings, LicensesClientService clientService) {
+    public LicenseService(Settings settings, LicensesClientService clientService, LicensesManagerService managerService) {
         super(settings);
+        this.managerService = managerService;
         this.clientService = clientService;
         this.mode = Mode.LITE;
         this.expirationLoggers = Arrays.asList(
@@ -103,6 +107,13 @@ public class LicenseService extends AbstractLifecycleComponent<LicenseService> {
         return mode;
     }
 
+    /**
+     * @return all registered licenses
+     */
+    public List<License> licenses() {
+        return managerService.getLicenses();
+    }
+
     class InternalListener implements LicensesClientService.Listener {
 
         private final LicenseService service;

marvel/src/main/resources/marvel_index_template.json

@@ -152,6 +152,9 @@
         }
       }
     },
+    "cluster_licenses": {
+      "enabled": false
+    },
     "marvel_node_stats": {
       "properties": {
         "node_stats": {

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/licenses/LicensesRendererIT.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.agent.renderer.licenses;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.license.core.License;
+import org.elasticsearch.license.plugin.LicensePlugin;
+import org.elasticsearch.marvel.MarvelPlugin;
+import org.elasticsearch.marvel.agent.collector.licenses.LicensesCollector;
+import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
+import org.elasticsearch.marvel.agent.settings.MarvelSettings;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.*;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, transportClientRatio = 0.0)
+public class LicensesRendererIT extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put("plugin.types", MarvelPlugin.class.getName() + "," + LicensePlugin.class.getName())
+                .put(Node.HTTP_ENABLED, true)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(MarvelSettings.STARTUP_DELAY, "1s")
+                .put(MarvelSettings.COLLECTORS, LicensesCollector.NAME)
+                .build();
+    }
+
+    @Test
+    public void testLicenses() throws Exception {
+        logger.debug("--> waiting for licenses collector to collect data (ie, the trial marvel license)");
+        GetResponse response = assertBusy(new Callable<GetResponse>() {
+            @Override
+            public GetResponse call() throws Exception {
+                // Checks if the marvel data index exists (it should have been created by the LicenseCollector)
+                assertTrue(client().admin().indices().prepareExists(MarvelSettings.MARVEL_DATA_INDEX_NAME).get().isExists());
+                ensureYellow(MarvelSettings.MARVEL_DATA_INDEX_NAME);
+
+                GetResponse response = client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, LicensesCollector.TYPE, cluster().getClusterName()).get();
+                assertTrue(response.isExists());
+                return response;
+            }
+        });
+
+        logger.debug("--> checking that the document contains license information");
+        assertThat(response.getIndex(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_NAME));
+        assertThat(response.getType(), equalTo(LicensesCollector.TYPE));
+        assertThat(response.getId(), equalTo(cluster().getClusterName()));
+
+        Map<String, Object> source = response.getSource();
+        assertThat((String) source.get(LicensesRenderer.Fields.VERSION.underscore().toString()), equalTo(Version.CURRENT.toString()));
+
+        Object licensesList = source.get(LicensesRenderer.Fields.LICENSES.underscore().toString());
+        assertThat(licensesList, instanceOf(List.class));
+
+        List licenses = (List) licensesList;
+        assertThat(licenses.size(), equalTo(1));
+
+        Map license = (Map) licenses.iterator().next();
+        assertThat(license, instanceOf(Map.class));
+
+        String uid = (String) ((Map) license).get(LicensesRenderer.Fields.UID.underscore().toString());
+        assertThat(uid, not(isEmptyOrNullString()));
+
+        String type = (String) ((Map) license).get(LicensesRenderer.Fields.TYPE.underscore().toString());
+        assertThat(type, not(isEmptyOrNullString()));
+
+        String status = (String) ((Map) license).get(LicensesRenderer.Fields.STATUS.underscore().toString());
+        assertThat(status, not(isEmptyOrNullString()));
+
+        Long expiryDate = (Long) ((Map) license).get(LicensesRenderer.Fields.EXPIRY_DATE_IN_MILLIS.underscore().toString());
+        assertThat(expiryDate, greaterThan(0L));
+
+        // We basically recompute the hash here
+        String hkey = (String) ((Map) license).get(LicensesRenderer.Fields.HKEY.underscore().toString());
+        String recalculated = LicensesRenderer.hash(License.builder().uid(uid).type(type).expiryDate(expiryDate).build(), cluster().getClusterName());
+        assertThat(hkey, equalTo(recalculated));
+
+        assertThat((String) ((Map) license).get(LicensesRenderer.Fields.FEATURE.underscore().toString()), not(isEmptyOrNullString()));
+        assertThat((String) ((Map) license).get(LicensesRenderer.Fields.ISSUER.underscore().toString()), not(isEmptyOrNullString()));
+        assertThat((String) ((Map) license).get(LicensesRenderer.Fields.ISSUED_TO.underscore().toString()), not(isEmptyOrNullString()));
+        assertThat((Long) ((Map) license).get(LicensesRenderer.Fields.ISSUE_DATE_IN_MILLIS.underscore().toString()), greaterThan(0L));
+        assertThat((Integer) ((Map) license).get(LicensesRenderer.Fields.MAX_NODES.underscore().toString()), greaterThan(0));
+
+
+        logger.debug("--> check that the cluster_licenses is not indexed");
+        refresh();
+        assertHitCount(client().prepareCount()
+                .setIndices(MarvelSettings.MARVEL_DATA_INDEX_NAME)
+                .setTypes(LicensesCollector.TYPE)
+                .setQuery(QueryBuilders.boolQuery()
+                        .should(QueryBuilders.matchQuery(LicensesRenderer.Fields.STATUS.underscore().toString(), "active"))
+                        .should(QueryBuilders.matchQuery(LicensesRenderer.Fields.STATUS.underscore().toString(), "inactive"))
+                        .should(QueryBuilders.matchQuery(LicensesRenderer.Fields.STATUS.underscore().toString(), "expired"))
+                        .minimumNumberShouldMatch(1)
+                ).get(), 0L);
+
+    }
+}