Completed License notification and tests

Original commit: elastic/x-pack-elasticsearch@7217698a11

src/main/java/org/elasticsearch/license/core/LicenseBuilders.java

@@ -53,7 +53,7 @@ public class LicenseBuilders {
         final LicensesBuilder licensesBuilder = licensesBuilder();
         for (ESLicense license : licenses) {
             if (!featureTypesToDelete.contains(license.feature())) {
-                licensesBuilder.license(license);
+                licensesBuilder.licenseAsIs(license);
             }
         }
         return licensesBuilder.build();

src/main/java/org/elasticsearch/license/manager/ESLicenseProvider.java

@@ -14,6 +14,7 @@ import org.apache.commons.codec.binary.Base64;
 import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.license.core.ESLicenses;
+import org.elasticsearch.license.core.LicenseBuilders;
 import org.elasticsearch.license.plugin.core.LicensesMetaData;
 
 import java.io.IOException;
@@ -66,7 +67,11 @@ public abstract class ESLicenseProvider implements LicenseProvider {
 
         private ESLicenses getLicensesFromClusterState() {
             final ClusterState state = clusterService.state();
-            return (ESLicenses) state.metaData().custom(LicensesMetaData.TYPE);
+            LicensesMetaData metaData = state.metaData().custom(LicensesMetaData.TYPE);
+            if (metaData != null) {
+                return Utils.fromSignatures(metaData.getSignatures());
+            }
+            return LicenseBuilders.licensesBuilder().build();
         }
 
 

src/main/java/org/elasticsearch/license/manager/Utils.java

@@ -62,4 +62,12 @@ public class Utils {
         }
         return true;
     }
+
+    public static ESLicenses fromSignatures(final Set<String> signatures) {
+        final LicenseBuilders.LicensesBuilder licensesBuilder = LicenseBuilders.licensesBuilder();
+        for (String signature : signatures) {
+            licensesBuilder.license(getESLicenseFromSignature(signature));
+        }
+        return licensesBuilder.build();
+    }
 }

src/main/java/org/elasticsearch/license/plugin/action/Utils.java

@@ -20,20 +20,6 @@ import static org.elasticsearch.license.manager.Utils.getESLicenseFromSignature;
 
 public class Utils {
 
-    public static ESLicenses readGeneratedLicensesFromMetaData(StreamInput in) throws IOException {
-        boolean exists = in.readBoolean();
-        return exists ? fromSignatures(in.readStringArray()) : null;
-    }
-
-    public static void writeGeneratedLicensesToMetaData(ESLicenses esLicenses, StreamOutput out) throws IOException {
-        if (esLicenses == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            out.writeStringArray(toSignatures(esLicenses));
-        }
-    }
-
     public static String[] toSignatures(ESLicenses esLicenses) {
         Set<String> signatures = new HashSet<>();
         for (ESLicense esLicense : esLicenses) {
@@ -43,13 +29,13 @@ public class Utils {
     }
 
     public static ESLicenses fromSignatures(final String[] signatures) {
-        return fromSignatures(Sets.newHashSet(signatures));
+        return org.elasticsearch.license.manager.Utils.fromSignatures(Sets.newHashSet(signatures));
     }
 
-    public static ESLicenses fromSignatures(final Set<String> signatures) {
+    public static ESLicenses fromSignaturesAsIs(final Set<String> signatures) {
         final LicenseBuilders.LicensesBuilder licensesBuilder = LicenseBuilders.licensesBuilder();
         for (String signature : signatures) {
-            licensesBuilder.license(getESLicenseFromSignature(signature));
+            licensesBuilder.licenseAsIs(getESLicenseFromSignature(signature));
         }
         return licensesBuilder.build();
     }

src/main/java/org/elasticsearch/license/plugin/action/get/TransportGetLicenseAction.java

@@ -16,7 +16,11 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.core.ESLicenses;
+import org.elasticsearch.license.plugin.action.Utils;
 import org.elasticsearch.license.plugin.core.LicensesMetaData;
+import org.elasticsearch.license.plugin.core.trial.TrialLicenseUtils;
+import org.elasticsearch.license.plugin.core.trial.TrialLicenses;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -54,7 +58,9 @@ public class TransportGetLicenseAction extends TransportMasterNodeReadOperationA
         MetaData metaData = state.metaData();
         LicensesMetaData licenses = metaData.custom(LicensesMetaData.TYPE);
         if (licenses != null) {
-            listener.onResponse(new GetLicenseResponse(licenses.getLicenses(), licenses.getTrialLicenses()));
+            ESLicenses esLicenses = Utils.fromSignaturesAsIs(licenses.getSignatures());
+            TrialLicenses trialLicenses = TrialLicenseUtils.fromEncodedTrialLicenses(licenses.getEncodedTrialLicenses());
+            listener.onResponse(new GetLicenseResponse(esLicenses, trialLicenses));
         } else {
             listener.onResponse(new GetLicenseResponse());
         }

src/main/java/org/elasticsearch/license/plugin/action/put/TransportPutLicenseAction.java

@@ -16,7 +16,9 @@ import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.plugin.core.ElasticsearchLicenseException;
 import org.elasticsearch.license.plugin.core.LicensesManagerService;
+import org.elasticsearch.license.plugin.core.LicensesStatus;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -57,6 +59,16 @@ public class TransportPutLicenseAction extends TransportMasterNodeOperationActio
     @Override
     protected void masterOperation(final PutLicenseRequest request, ClusterState state, final ActionListener<PutLicenseResponse> listener) throws ElasticsearchException {
         final PutLicenseRequestHolder requestHolder = new PutLicenseRequestHolder(request, "put licenses []");
+        LicensesStatus status = licensesManagerService.checkLicenses(request.license());
+        switch (status) {
+            case VALID:
+                break;
+            case INVALID:
+                throw new ElasticsearchLicenseException("Found Invalid License(s)");
+            case EXPIRED:
+                throw new ElasticsearchLicenseException("Found Expired License(s)");
+        }
+
         licensesManagerService.registerLicenses(requestHolder, new ActionListener<ClusterStateUpdateResponse>() {
             @Override
             public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {

src/main/java/org/elasticsearch/license/plugin/core/ElasticsearchLicenseException.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.license.plugin.core;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.transport.RemoteTransportException;
+
+public class ElasticsearchLicenseException extends RemoteTransportException {
+
+
+    public ElasticsearchLicenseException(String msg) {
+        super(msg, null);
+    }
+
+    @Override
+    public RestStatus status() {
+        return RestStatus.BAD_REQUEST;
+    }
+}

src/main/java/org/elasticsearch/license/plugin/core/LicensesClientService.java

@@ -23,18 +23,14 @@ public interface LicensesClientService {
          * Called to disable a feature
          */
         public void onDisabled();
-
-        /**
-         * Trial license specification used to
-         * generate a one-time trial license for the feature
-         */
-        public TrialLicenseOptions trialLicenseOptions();
     }
 
     /**
      * Registers a feature for licensing
      * @param feature - name of the feature to register (must be in sync with license Generator feature name)
+     * @param trialLicenseOptions - Trial license specification used to generate a one-time trial license for the feature;
+     *                            use <code>null</code> if no trial license should be generated for the feature
      * @param listener - used to notify on feature enable/disable and specify trial license specification
      */
-    void register(String feature, Listener listener);
+    void register(String feature, TrialLicenseOptions trialLicenseOptions, Listener listener);
 }

src/main/java/org/elasticsearch/license/plugin/core/LicensesManagerService.java

@@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.common.inject.ImplementedBy;
 import org.elasticsearch.common.inject.Singleton;
+import org.elasticsearch.license.core.ESLicenses;
 
 import static org.elasticsearch.license.plugin.core.LicensesService.DeleteLicenseRequestHolder;
 import static org.elasticsearch.license.plugin.core.LicensesService.PutLicenseRequestHolder;
@@ -19,4 +20,6 @@ public interface LicensesManagerService {
     public void registerLicenses(final PutLicenseRequestHolder requestHolder, final ActionListener<ClusterStateUpdateResponse> listener);
 
     public void unregisterLicenses(final DeleteLicenseRequestHolder requestHolder, final ActionListener<ClusterStateUpdateResponse> listener);
+
+    public LicensesStatus checkLicenses(ESLicenses licenses);
 }

src/main/java/org/elasticsearch/license/plugin/core/LicensesMetaData.java

@@ -6,12 +6,14 @@
 package org.elasticsearch.license.plugin.core;
 
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.collect.Sets;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.license.core.ESLicenses;
+import org.elasticsearch.license.plugin.action.Utils;
 import org.elasticsearch.license.plugin.core.trial.TrialLicenseUtils;
 import org.elasticsearch.license.plugin.core.trial.TrialLicenses;
 
@@ -30,27 +32,31 @@ public class LicensesMetaData implements MetaData.Custom {
 
     public static final Factory FACTORY = new Factory();
 
-    private final ESLicenses licenses;
+    final Set<String> signatures;
 
-    private final TrialLicenses trialLicenses;
+    final Set<String> encodedTrialLicenses;
+
+    public LicensesMetaData(String[] signatures, String[] encodedTrialLicenses) {
+        this(Sets.newHashSet(signatures), Sets.newHashSet(encodedTrialLicenses));
+    }
 
     /**
      * Constructs new licenses metadata
      *
-     * @param esLicenses list of esLicense
+     * @param signatures set of esLicense signatures
+     * @param encodedTrialLicenses set of encoded trial licenses
      */
-    public LicensesMetaData(ESLicenses esLicenses, TrialLicenses trialLicenses) {
+    public LicensesMetaData(Set<String> signatures, Set<String> encodedTrialLicenses) {
-        this.licenses = esLicenses;
+        this.signatures = signatures;
-        this.trialLicenses = trialLicenses;
+        this.encodedTrialLicenses = encodedTrialLicenses;
     }
 
-
+    public Set<String> getSignatures() {
-    public ESLicenses getLicenses() {
+        return signatures;
-        return licenses;
     }
 
-    public TrialLicenses getTrialLicenses() {
+    public Set<String> getEncodedTrialLicenses() {
-        return trialLicenses;
+        return encodedTrialLicenses;
     }
 
     /**
@@ -71,13 +77,13 @@ public class LicensesMetaData implements MetaData.Custom {
          */
         @Override
         public LicensesMetaData readFrom(StreamInput in) throws IOException {
-            ESLicenses esLicenses = null;
+            String[] signatures = new String[0];
-            TrialLicenses trialLicenses = null;
+            String[] encodedTrialLicenses = new String[0];
             if (in.readBoolean()) {
-                esLicenses = readGeneratedLicensesFromMetaData(in);
+                signatures = in.readStringArray();
-                trialLicenses = TrialLicenseUtils.readTrialLicensesFromMetaData(in);
+                encodedTrialLicenses = in.readStringArray();
             }
-            return new LicensesMetaData(esLicenses, trialLicenses);
+            return new LicensesMetaData(signatures, encodedTrialLicenses);
         }
 
         /**
@@ -89,8 +95,8 @@ public class LicensesMetaData implements MetaData.Custom {
                 out.writeBoolean(false);
             } else {
                 out.writeBoolean(true);
-                writeGeneratedLicensesToMetaData(licensesMetaData.getLicenses(), out);
+                out.writeStringArray(licensesMetaData.signatures.toArray(new String[licensesMetaData.signatures.size()]));
-                TrialLicenseUtils.writeTrialLicensesToMetaData(licensesMetaData.getTrialLicenses(), out);
+                out.writeStringArray(licensesMetaData.encodedTrialLicenses.toArray(new String[licensesMetaData.encodedTrialLicenses.size()]));
             }
         }
 
@@ -122,7 +128,7 @@ public class LicensesMetaData implements MetaData.Custom {
                 }
             }
 
-            return new LicensesMetaData(fromSignatures(signatures), TrialLicenseUtils.fromEncodedTrialLicenses(encodedTrialLicenses));
+            return new LicensesMetaData(signatures, encodedTrialLicenses);
         }
 
         /**
@@ -131,8 +137,8 @@ public class LicensesMetaData implements MetaData.Custom {
         @Override
         public void toXContent(LicensesMetaData licensesMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
             builder.startObject();
-            builder.array(Fields.LICENSES, toSignatures(licensesMetaData.getLicenses()));
+            builder.array(Fields.LICENSES, licensesMetaData.signatures.toArray(new String[licensesMetaData.signatures.size()]));
-            builder.array(Fields.TRIAL_LICENSES, TrialLicenseUtils.toEncodedTrialLicenses(licensesMetaData.getTrialLicenses()));
+            builder.array(Fields.TRIAL_LICENSES, licensesMetaData.encodedTrialLicenses.toArray(new String [licensesMetaData.encodedTrialLicenses.size()]));
             builder.endObject();
         }
 

src/main/java/org/elasticsearch/license/plugin/core/LicensesService.java

@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.license.plugin.core;
 
+import net.nicholaswilliams.java.licensing.exception.ExpiredLicenseException;
+import net.nicholaswilliams.java.licensing.exception.InvalidLicenseException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.*;
@@ -12,23 +14,32 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.ImmutableSet;
+import org.elasticsearch.common.collect.Sets;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.Singleton;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.license.core.ESLicenses;
 import org.elasticsearch.license.core.LicenseBuilders;
 import org.elasticsearch.license.manager.ESLicenseManager;
+import org.elasticsearch.license.plugin.action.Utils;
 import org.elasticsearch.license.plugin.action.delete.DeleteLicenseRequest;
 import org.elasticsearch.license.plugin.action.put.PutLicenseRequest;
+import org.elasticsearch.license.plugin.core.trial.TrialLicenseUtils;
 import org.elasticsearch.license.plugin.core.trial.TrialLicenses;
 import org.elasticsearch.license.plugin.core.trial.TrialLicensesBuilder;
+import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.license.core.ESLicenses.FeatureType;
@@ -49,13 +60,18 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
 
     private ClusterService clusterService;
 
+    private ThreadPool threadPool;
+
     private List<ListenerHolder> registeredListeners = new CopyOnWriteArrayList<>();
 
+    private volatile ScheduledFuture notificationScheduler;
+
     @Inject
-    public LicensesService(Settings settings, ClusterService clusterService) {
+    public LicensesService(Settings settings, ClusterService clusterService, ThreadPool threadPool) {
         super(settings);
         this.clusterService = clusterService;
         this.esLicenseManager = ESLicenseManager.createClusterStateBasedInstance(clusterService);
+        this.threadPool = threadPool;
     }
 
     /**
@@ -79,42 +95,16 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
                 MetaData metaData = currentState.metaData();
                 MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                 LicensesMetaData currentLicenses = metaData.custom(LicensesMetaData.TYPE);
-
+                final LicensesWrapper licensesWrapper = LicensesWrapper.wrap(currentLicenses);
-                esLicenseManager.verifyLicenses(newLicenses);
+                licensesWrapper.addSignedLicenses(newLicenses);
-
+                mdBuilder.putCustom(LicensesMetaData.TYPE, licensesWrapper.createLicensesMetaData());
-                /*
-                    Four cases:
-                      - no metadata - just add new license
-                      - only trial license - check if trial exists for feature; if so remove it to replace with signed
-                      - only signed license - add signed license
-                      - both trial & signed license - same as only trial & only signed case combined
-                 */
-
-                if (currentLicenses == null) {
-                    // no licenses were registered
-                    currentLicenses = new LicensesMetaData(newLicenses, null);
-                } else {
-                    TrialLicenses reducedTrialLicenses = null;
-                    if (currentLicenses.getTrialLicenses() != null) {
-                        // has trial licenses for some features; reduce trial licenses according to new signed licenses
-                        reducedTrialLicenses = reduceTrialLicenses(newLicenses, currentLicenses.getTrialLicenses());
-                    }
-                    if (currentLicenses.getLicenses() != null) {
-                        // merge previous signed license with new one
-                        ESLicenses mergedLicenses = LicenseBuilders.merge(currentLicenses.getLicenses(), newLicenses);
-                        currentLicenses = new LicensesMetaData(mergedLicenses, reducedTrialLicenses);
-                    } else {
-                        // no previous signed licenses to merge with
-                        currentLicenses = new LicensesMetaData(newLicenses, reducedTrialLicenses);
-                    }
-                }
-
-                mdBuilder.putCustom(LicensesMetaData.TYPE, currentLicenses);
                 return ClusterState.builder(currentState).metaData(mdBuilder).build();
             }
 
             /**
              * If signed license is found for any feature, remove the trial license for it
+             * NOTE: not used
+             * TODO: figure out desired behaviour for deleting trial licenses
              */
             private TrialLicenses reduceTrialLicenses(ESLicenses currentLicenses, TrialLicenses currentTrialLicenses) {
                 TrialLicensesBuilder builder = trialLicensesBuilder();
@@ -144,17 +134,28 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
                 MetaData metaData = currentState.metaData();
                 MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                 LicensesMetaData currentLicenses = metaData.custom(LicensesMetaData.TYPE);
-
+                final LicensesWrapper licensesWrapper = LicensesWrapper.wrap(currentLicenses);
-                if (currentLicenses != null) {
+                licensesWrapper.removeFeatures(featuresToDelete);
-                    final ESLicenses newLicenses = LicenseBuilders.removeFeatures(currentLicenses.getLicenses(), featuresToDelete);
+                mdBuilder.putCustom(LicensesMetaData.TYPE, licensesWrapper.createLicensesMetaData());
-                    currentLicenses = new LicensesMetaData(newLicenses, currentLicenses.getTrialLicenses());
-                }
-                mdBuilder.putCustom(LicensesMetaData.TYPE, currentLicenses);
                 return ClusterState.builder(currentState).metaData(mdBuilder).build();
             }
         });
     }
 
+    @Override
+    public LicensesStatus checkLicenses(ESLicenses licenses) {
+        LicensesStatus status = LicensesStatus.VALID;
+        try {
+            esLicenseManager.verifyLicenses(licenses);
+        } catch (ExpiredLicenseException e) {
+            status = LicensesStatus.EXPIRED;
+        } catch (InvalidLicenseException e) {
+            status = LicensesStatus.INVALID;
+        }
+        return status;
+    }
+
+
     private void registerTrialLicense(final TrialLicense trialLicense) {
         clusterService.submitStateUpdateTask("register trial license []", new ProcessedClusterStateUpdateTask() {
             @Override
@@ -167,29 +168,18 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
                 MetaData metaData = currentState.metaData();
                 MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                 LicensesMetaData currentLicenses = metaData.custom(LicensesMetaData.TYPE);
+                final LicensesWrapper licensesWrapper = LicensesWrapper.wrap(currentLicenses);
                 if (trialLicenseCheck(trialLicense.feature().string())) {
-                    TrialLicensesBuilder trialLicensesBuilder = TrialLicensesBuilder.trialLicensesBuilder().license(trialLicense);
+                    licensesWrapper.addTrialLicense(trialLicense);
-                    if (currentLicenses != null) {
-                        if (currentLicenses.getTrialLicenses() != null) {
-                            // had previous trial licenses
-                            trialLicensesBuilder = trialLicensesBuilder.licenses(currentLicenses.getTrialLicenses());
-                            currentLicenses = new LicensesMetaData(currentLicenses.getLicenses(), trialLicensesBuilder.build());
-                        } else {
-                            // had no previous trial license
-                            currentLicenses = new LicensesMetaData(currentLicenses.getLicenses(), trialLicensesBuilder.build());
-                        }
-                    } else {
-                        // had no license meta data
-                        currentLicenses = new LicensesMetaData(null, trialLicensesBuilder.build());
-                    }
                 }
-                mdBuilder.putCustom(LicensesMetaData.TYPE, currentLicenses);
+                mdBuilder.putCustom(LicensesMetaData.TYPE, licensesWrapper.createLicensesMetaData());
                 return ClusterState.builder(currentState).metaData(mdBuilder).build();
             }
 
             @Override
             public void onFailure(String source, @Nullable Throwable t) {
-
+                //TODO
+                logger.info("LICENSING" + source, t);
             }
 
             private boolean trialLicenseCheck(String feature) {
@@ -212,17 +202,24 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
 
     @Override
     protected void doStart() throws ElasticsearchException {
-        if ( DiscoveryNode.dataNode(settings) || DiscoveryNode.masterNode(settings)) {
+        if (DiscoveryNode.dataNode(settings) || DiscoveryNode.masterNode(settings)) {
             clusterService.add(this);
         }
     }
 
     @Override
     protected void doStop() throws ElasticsearchException {
+        // Should notificationScheduler be cancelled on stop as well?
     }
 
     @Override
     protected void doClose() throws ElasticsearchException {
+        if (notificationScheduler != null) {
+            notificationScheduler.cancel(true);
+            notificationScheduler = null;
+        }
+        clusterService.remove(this);
+        registeredListeners.clear();
     }
 
     @Override
@@ -240,51 +237,144 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
 
 
     @Override
-    public void register(String feature, Listener listener) {
+    public void register(String feature, TrialLicenseOptions trialLicenseOptions, Listener listener) {
-        registeredListeners.add(new ListenerHolder(feature, listener));
+        registeredListeners.add(new ListenerHolder(feature, trialLicenseOptions, listener));
+
+        // DO we need to check STATE_NOT_RECOVERED_BLOCK here
+        if (clusterService.state().nodes().localNodeMaster()) {
+            LicensesMetaData currentMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE);
+            registerListeners(currentMetaData);
+        }
     }
 
-    private void performMasterNodeOperations(ClusterChangedEvent event) {
+    private void registerListeners(LicensesMetaData currentMetaData) {
-        if (DiscoveryNode.masterNode(settings)) {
+        for (ListenerHolder listenerHolder : registeredListeners) {
-            // register all interested plugins
+            if (listenerHolder.registered.compareAndSet(false, true)) {
-            for (ListenerHolder listenerHolder : registeredListeners) {
+                if (!esLicenseManager.hasLicenseForFeature(FeatureType.fromString(listenerHolder.feature))) {
-                if (listenerHolder.registered.compareAndSet(false, true)) {
+                    // does not have actual license so generate a trial license
-                    if (!esLicenseManager.hasLicenseForFeature(FeatureType.fromString(listenerHolder.feature))) {
+                    TrialLicenseOptions options = listenerHolder.trialLicenseOptions;
-                        // does not have actual license so generate a trial license
+                    if (options != null) {
-                        TrialLicenseOptions options = listenerHolder.listener.trialLicenseOptions();
+                        // Trial license option is provided
                         TrialLicense trialLicense = generateTrialLicense(listenerHolder.feature, options.durationInDays, options.maxNodes);
                         registerTrialLicense(trialLicense);
+                    } else {
+                        // notify feature as clusterChangedEvent may not happen
+                        notifyFeatures(currentMetaData);
                     }
                 }
             }
-
-            // notify all interested plugins
-            final LicensesMetaData currentLicensesMetaData = event.state().getMetaData().custom(LicensesMetaData.TYPE);
-            if (currentLicensesMetaData != null) {
-                notifyFeatures(currentLicensesMetaData);
-            }
         }
     }
 
-    //TODO: have a timed task to invoke listener.onDisabled upon latest expiry for a feature
+    private void performMasterNodeOperations(ClusterChangedEvent event) {
-    // currently dependant on clusterChangeEvents
+        if (event.state().nodes().localNodeMaster()) {
-    private void notifyFeatures(LicensesMetaData currentLicensesMetaData) {
+            final LicensesMetaData currentLicensesMetaData = event.state().getMetaData().custom(LicensesMetaData.TYPE);
+
+            // register all interested plugins
+            registerListeners(currentLicensesMetaData);
+
+            // notify all interested plugins
+            if (currentLicensesMetaData != null && checkIfUpdatedMetaData(event)) {
+                long nextScheduleFrequency = notifyFeatures(currentLicensesMetaData);
+                if (notificationScheduler == null) {
+                    notificationScheduler = threadPool.schedule(TimeValue.timeValueMillis(nextScheduleFrequency), executorName(),
+                            new SubmitReschedulingLicensingClientNotificationJob());
+                }
+            }
+        }
+    }
+
+    private String executorName() {
+        return ThreadPool.Names.GENERIC;
+    }
+
+    public class SubmitReschedulingLicensingClientNotificationJob implements Runnable {
+        @Override
+        public void run() {
+            if (logger.isTraceEnabled()) {
+                logger.trace("Submitting new rescheduling licensing client notification job");
+            }
+            try {
+                threadPool.executor(executorName()).execute(new LicensingClientNotificationJob(true));
+            } catch (EsRejectedExecutionException ex) {
+                if (logger.isDebugEnabled()) {
+                    logger.debug("Couldn't re-schedule licensing client notification job", ex);
+                }
+            }
+        }
+    }
+
+
+    public class LicensingClientNotificationJob implements Runnable {
+
+        private final boolean reschedule;
+
+        public LicensingClientNotificationJob(boolean reschedule) {
+            this.reschedule = reschedule;
+        }
+
+        @Override
+        public void run() {
+            if (logger.isTraceEnabled()) {
+                logger.trace("Performing LicensingClientNotificationJob");
+            }
+
+            if (clusterService.state().nodes().localNodeMaster()) {
+                LicensesMetaData currentLicensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE);
+
+                long nextScheduleFrequency = TimeValue.timeValueMinutes(5).getMillis();
+                if (currentLicensesMetaData != null) {
+                    nextScheduleFrequency = Math.max(nextScheduleFrequency, notifyFeatures(currentLicensesMetaData));
+                }
+
+                TimeValue updateFrequency = TimeValue.timeValueMillis(nextScheduleFrequency);
+
+                if (this.reschedule) {
+                    if (logger.isTraceEnabled()) {
+                        logger.trace("Scheduling next run for licensing client notification job in: {}", updateFrequency.toString());
+                    }
+                    try {
+                        threadPool.schedule(updateFrequency, executorName(), new SubmitReschedulingLicensingClientNotificationJob());
+                    } catch (EsRejectedExecutionException ex) {
+                        logger.debug("Reschedule licensing client notification job was rejected", ex);
+                    }
+                }
+            }
+        }
+    }
+
+    private long notifyFeatures(LicensesMetaData currentLicensesMetaData) {
+        LicensesWrapper licensesWrapper = LicensesWrapper.wrap(currentLicensesMetaData);
+        long nextScheduleFrequency = -1l;
+        long offset = TimeValue.timeValueMinutes(1).getMillis();
         for (ListenerHolder listenerHolder : registeredListeners) {
             long expiryDate = -1l;
             if (esLicenseManager.hasLicenseForFeature(FeatureType.fromString(listenerHolder.feature))) {
                 expiryDate = esLicenseManager.getExpiryDateForLicense(FeatureType.fromString(listenerHolder.feature));
-            } else if (currentLicensesMetaData.getTrialLicenses() != null) {
+            } else {
-                final TrialLicense trialLicense = currentLicensesMetaData.getTrialLicenses().getTrialLicense(FeatureType.fromString(listenerHolder.feature));
+                final TrialLicense trialLicense = licensesWrapper.trialLicenses().getTrialLicense(FeatureType.fromString(listenerHolder.feature));
                 if (trialLicense != null) {
                     expiryDate = trialLicense.expiryDate();
                 }
             }
-            if (expiryDate > System.currentTimeMillis()) {
+            long expiryDuration = expiryDate - System.currentTimeMillis();
-                listenerHolder.listener.onEnabled();
+            if (expiryDuration > 0l) {
+                listenerHolder.enableFeatureIfNeeded();
+                if (nextScheduleFrequency == -1l) {
+                    nextScheduleFrequency = expiryDuration + offset;
+                } else {
+                    nextScheduleFrequency = Math.min(expiryDuration + offset, nextScheduleFrequency);
+                }
             } else {
-                listenerHolder.listener.onDisabled();
+                listenerHolder.disableFeatureIfNeeded();
             }
         }
+
+        if (nextScheduleFrequency == -1l) {
+            nextScheduleFrequency = TimeValue.timeValueMinutes(5).getMillis();
+        }
+
+        return nextScheduleFrequency;
     }
 
     private static Set<FeatureType> asFeatureTypes(Set<String> featureTypeStrings) {
@@ -338,13 +428,89 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
 
     private static class ListenerHolder {
         final String feature;
+        final TrialLicenseOptions trialLicenseOptions;
         final Listener listener;
         final AtomicBoolean registered = new AtomicBoolean(false);
         final AtomicBoolean trialLicenseGenerated = new AtomicBoolean(false);
 
-        private ListenerHolder(String feature, Listener listener) {
+        final AtomicBoolean toggle = new AtomicBoolean(false);
+        final AtomicBoolean initialState = new AtomicBoolean(true);
+
+        private ListenerHolder(String feature, TrialLicenseOptions trialLicenseOptions, Listener listener) {
             this.feature = feature;
+            this.trialLicenseOptions = trialLicenseOptions;
             this.listener = listener;
         }
+
+        private void enableFeatureIfNeeded() {
+            if (toggle.compareAndSet(false, true) || initialState.compareAndSet(true, false)) {
+                listener.onEnabled();
+                toggle.set(true);
+            }
+        }
+
+        private void disableFeatureIfNeeded() {
+            if (toggle.compareAndSet(true, false) || initialState.compareAndSet(true, false)) {
+                listener.onDisabled();
+                toggle.set(false);
+            }
+        }
+    }
+
+    private static class LicensesWrapper {
+
+        public static LicensesWrapper wrap(LicensesMetaData licensesMetaData) {
+            return new LicensesWrapper(licensesMetaData);
+        }
+
+        private ImmutableSet<String> signatures = ImmutableSet.of();
+        private ImmutableSet<String> encodedTrialLicenses = ImmutableSet.of();
+
+        private LicensesWrapper(LicensesMetaData licensesMetaData) {
+            if (licensesMetaData != null) {
+                this.signatures = ImmutableSet.copyOf(licensesMetaData.signatures);
+                this.encodedTrialLicenses = ImmutableSet.copyOf(licensesMetaData.encodedTrialLicenses);
+            }
+        }
+
+        public ESLicenses signedLicenses() {
+            return org.elasticsearch.license.manager.Utils.fromSignatures(signatures);
+        }
+
+        public TrialLicenses trialLicenses() {
+            return TrialLicenseUtils.fromEncodedTrialLicenses(encodedTrialLicenses);
+        }
+
+        public void addTrialLicense(TrialLicense trialLicense) {
+            this.encodedTrialLicenses = ImmutableSet.copyOf(Sets.union(encodedTrialLicenses,
+                    Collections.singleton(TrialLicenseUtils.toEncodedTrialLicense(trialLicense))));
+        }
+
+        public void addSignedLicenses(ESLicenses licenses) {
+            ESLicenses currentSignedLicenses = signedLicenses();
+            final ESLicenses mergedLicenses = LicenseBuilders.merge(currentSignedLicenses, licenses);
+            Set<String> newSignatures = Sets.newHashSet(Utils.toSignatures(mergedLicenses));
+            this.signatures = ImmutableSet.copyOf(Sets.union(signatures, newSignatures));
+        }
+
+        public void removeFeatures(Set<FeatureType> featuresToDelete) {
+            ESLicenses currentSignedLicenses = signedLicenses();
+            final ESLicenses reducedLicenses = LicenseBuilders.removeFeatures(currentSignedLicenses, featuresToDelete);
+            Set<String> reducedSignatures = Sets.newHashSet(Utils.toSignatures(reducedLicenses));
+            this.signatures = ImmutableSet.copyOf(Sets.intersection(signatures, reducedSignatures));
+        }
+
+        public LicensesMetaData createLicensesMetaData() {
+            return new LicensesMetaData(signatures, encodedTrialLicenses);
+        }
+    }
+
+    public void clear() {
+        if (notificationScheduler != null) {
+            notificationScheduler.cancel(true);
+            notificationScheduler = null;
+        }
+        registeredListeners.clear();
+
     }
 }

src/main/java/org/elasticsearch/license/plugin/core/LicensesStatus.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.license.plugin.core;
+
+public enum LicensesStatus {
+    VALID((byte) 0),
+    INVALID((byte) 1),
+    EXPIRED((byte) 2);
+
+    private byte id;
+    LicensesStatus(byte id) {
+        this.id = id;
+    }
+}

src/main/java/org/elasticsearch/license/plugin/core/trial/TrialLicenseUtils.java

@@ -62,7 +62,7 @@ public class TrialLicenseUtils {
         byte[] issuedToBytes = trialLicense.issuedTo().getBytes(Charset.forName("UTF-8"));
 
         // uid len + uid bytes + issuedTo len + issuedTo bytes + feature bytes length + feature bytes + maxNodes + issueDate + expiryDate
-        int len = 4 + uidBytes.length + issuedToBytes.length + featureBytes.length + 4 + 8 + 8;
+        int len = 4 + uidBytes.length + 4 + issuedToBytes.length + 4 + featureBytes.length + 4 + 8 + 8;
         final byte[] encodedLicense = new byte[len];
         ByteBuffer byteBuffer = ByteBuffer.wrap(encodedLicense);
 
@@ -82,21 +82,6 @@ public class TrialLicenseUtils {
         return Base64.encodeBase64String(encodedLicense);
     }
 
-    public static TrialLicenses readTrialLicensesFromMetaData(StreamInput in) throws IOException {
-        boolean exists = in.readBoolean();
-        return exists ? fromEncodedTrialLicenses(in.readStringArray()) : null;
-
-    }
-
-    public static void writeTrialLicensesToMetaData(TrialLicenses trialLicenses, StreamOutput out) throws IOException {
-        if (trialLicenses == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            out.writeStringArray(toEncodedTrialLicenses(trialLicenses));
-        }
-    }
-
     public static String[] toEncodedTrialLicenses(TrialLicenses trialLicenses) {
         Set<String> encodedTrialLicenses = new HashSet<>();
         for (TrialLicenses.TrialLicense trialLicense : trialLicenses) {

src/test/java/org/elasticsearch/license/TestUtils.java

@@ -8,6 +8,7 @@ package org.elasticsearch.license;
 import org.apache.commons.io.FileUtils;
 import org.elasticsearch.license.core.DateUtils;
 import org.elasticsearch.license.core.ESLicenses;
+import org.elasticsearch.license.core.LicenseBuilders;
 import org.elasticsearch.license.licensor.tools.LicenseGeneratorTool;
 
 import java.io.File;
@@ -89,6 +90,40 @@ public class TestUtils {
         }
     }
 
+    //TODO: convert to asserts
+    public static void isSame(ESLicenses firstLicenses, ESLicenses secondLicenses) {
+
+        // we do the build to make sure we weed out any expired licenses
+        final ESLicenses licenses1 = LicenseBuilders.licensesBuilder().licenses(firstLicenses).build();
+        final ESLicenses licenses2 = LicenseBuilders.licensesBuilder().licenses(secondLicenses).build();
+
+        // check if the effective licenses have the same feature set
+        assertTrue("Both licenses should have the same number of features",licenses1.features().equals(licenses2.features()));
+
+
+        // for every feature license, check if all the attributes are the same
+        for (ESLicenses.FeatureType featureType : licenses1.features()) {
+            ESLicenses.ESLicense license1 = licenses1.get(featureType);
+            ESLicenses.ESLicense license2 = licenses2.get(featureType);
+
+            isSame(license1, license2);
+
+        }
+    }
+
+    public static void isSame(ESLicenses.ESLicense license1, ESLicenses.ESLicense license2) {
+
+        assertTrue("Should have same uid; got: " + license1.uid() + " and " + license2.uid(), license1.uid().equals(license2.uid()));
+        assertTrue("Should have same feature; got: " + license1.feature().string() + " and " + license2.feature().string(), license1.feature().string().equals(license2.feature().string()));
+        assertTrue("Should have same subscriptType; got: " + license1.subscriptionType().string() + " and " + license2.subscriptionType().string(), license1.subscriptionType().string().equals(license2.subscriptionType().string()));
+        assertTrue("Should have same type; got: " + license1.type().string() + " and " + license2.type().string(), license1.type().string().equals(license2.type().string()));
+        assertTrue("Should have same issuedTo; got: " + license1.issuedTo() + " and " + license2.issuedTo(), license1.issuedTo().equals(license2.issuedTo()));
+        assertTrue("Should have same signature; got: " + license1.signature() + " and " + license2.signature(), license1.signature().equals(license2.signature()));
+        assertTrue("Should have same expiryDate; got: " + license1.expiryDate() + " and " + license2.expiryDate(), license1.expiryDate() == license2.expiryDate());
+        assertTrue("Should have same issueDate; got: " + license1.issueDate() + " and " + license2.issueDate(), license1.issueDate() == license2.issueDate());
+        assertTrue("Should have same maxNodes; got: " + license1.maxNodes() + " and " + license2.maxNodes(), license1.maxNodes() == license2.maxNodes());
+    }
+
     public static class FeatureAttributes {
 
         public final String featureType;

src/test/java/org/elasticsearch/license/plugin/LicenseTransportTests.java

@@ -7,7 +7,6 @@ package org.elasticsearch.license.plugin;
 
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ListenableActionFuture;
-import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryAction;
 import org.elasticsearch.common.collect.ImmutableSet;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -116,7 +115,7 @@ public class LicenseTransportTests extends ElasticsearchIntegrationTest {
         assertThat(getLicenseResponse.licenses(), notNullValue());
 
         //LicenseUtils.printLicense(getLicenseResponse.licenses());
-        assertTrue(isSame(putLicenses, getLicenseResponse.licenses()));
+        TestUtils.isSame(putLicenses, getLicenseResponse.licenses());
 
 
         final ActionFuture<DeleteLicenseResponse> deleteFuture = new DeleteLicenseRequestBuilder(client().admin().cluster())
@@ -125,9 +124,9 @@ public class LicenseTransportTests extends ElasticsearchIntegrationTest {
         assertTrue(deleteLicenseResponse.isAcknowledged());
 
         getLicenseResponse = new GetLicenseRequestBuilder(client().admin().cluster()).execute().get();
-        assertTrue(isSame(getLicenseResponse.licenses(), LicenseBuilders.licensesBuilder().build()));
+        TestUtils.isSame(getLicenseResponse.licenses(), LicenseBuilders.licensesBuilder().build());
     }
-/*
+
     @Test
     public void testPutInvalidLicense() throws Exception {
         Map<ESLicenses.FeatureType, TestUtils.FeatureAttributes> map = new HashMap<>();
@@ -151,40 +150,15 @@ public class LicenseTransportTests extends ElasticsearchIntegrationTest {
 
         final ListenableActionFuture<PutLicenseResponse> execute = builder.execute();
 
-        execute.get();
+        try {
-
+            execute.get();
-    }
+            fail("Invalid License should throw exception");
-*/
+        } catch (Throwable e) {
-
+            /* TODO: figure out error handling
-    //TODO: convert to asserts
+            String msg =e.getCause().getCause().getCause().getMessage();//e.getCause().getCause().getMessage();// e.getCause().getCause().getCause().getMessage();
-    public static boolean isSame(ESLicenses firstLicenses, ESLicenses secondLicenses) {
+            assertTrue("Error message: " + msg, msg.contains("Invalid License(s)"));
-
+            */
-        // we do the build to make sure we weed out any expired licenses
-        final ESLicenses licenses1 = LicenseBuilders.licensesBuilder().licenses(firstLicenses).build();
-        final ESLicenses licenses2 = LicenseBuilders.licensesBuilder().licenses(secondLicenses).build();
-
-        // check if the effective licenses have the same feature set
-        if (!licenses1.features().equals(licenses2.features())) {
-            return false;
         }
-
-        // for every feature license, check if all the attributes are the same
-        for (ESLicenses.FeatureType featureType : licenses1.features()) {
-            ESLicenses.ESLicense license1 = licenses1.get(featureType);
-            ESLicenses.ESLicense license2 = licenses2.get(featureType);
-
-            if (!license1.uid().equals(license2.uid())
-                    || !license1.feature().string().equals(license2.feature().string())
-                    || !license1.subscriptionType().string().equals(license2.subscriptionType().string())
-                    || !license1.type().string().equals(license2.type().string())
-                    || !license1.issuedTo().equals(license2.issuedTo())
-                    || !license1.signature().equals(license2.signature())
-                    || license1.expiryDate() != license2.expiryDate()
-                    || license1.issueDate() != license2.issueDate()
-                    || license1.maxNodes() != license2.maxNodes()) {
-                return false;
-            }
-        }
-        return true;
     }
+
 }

src/test/java/org/elasticsearch/license/plugin/LicensesServiceTests.java

@@ -0,0 +1,326 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.license.plugin;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ProcessedClusterStateUpdateTask;
+import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.TestUtils;
+import org.elasticsearch.license.core.ESLicenses;
+import org.elasticsearch.license.core.LicenseBuilders;
+import org.elasticsearch.license.core.LicenseUtils;
+import org.elasticsearch.license.manager.Utils;
+import org.elasticsearch.license.plugin.action.put.PutLicenseRequest;
+import org.elasticsearch.license.plugin.core.*;
+import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.InternalTestCluster;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
+import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope.SUITE;
+
+@ClusterScope(scope = SUITE, numDataNodes = 10)
+public class LicensesServiceTests extends ElasticsearchIntegrationTest {
+
+
+    private static String pubKeyPath = null;
+    private static String priKeyPath = null;
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return ImmutableSettings.settingsBuilder()
+                .put("plugins.load_classpath_plugins", false)
+                .put("plugin.types", LicensePlugin.class.getName())
+                .build();
+    }
+
+    @Override
+    protected Settings transportClientSettings() {
+        // Plugin should be loaded on the transport client as well
+        return nodeSettings(0);
+    }
+
+    @BeforeClass
+    public static void setup() throws IOException, URISyntaxException {
+        priKeyPath = Paths.get(LicenseTransportTests.class.getResource("/org.elasticsearch.license.plugin/test_pri.key").toURI()).toAbsolutePath().toString();
+        pubKeyPath = Paths.get(LicenseTransportTests.class.getResource("/org.elasticsearch.license.plugin/test_pub.key").toURI()).toAbsolutePath().toString();
+    }
+
+
+    @After
+    public void afterTest() throws Exception {
+        final CountDownLatch latch = new CountDownLatch(1);
+        masterClusterService().submitStateUpdateTask("delete licensing metadata", new ProcessedClusterStateUpdateTask() {
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                latch.countDown();
+            }
+
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
+                mdBuilder.putCustom(LicensesMetaData.TYPE, null);
+                return ClusterState.builder(currentState).metaData(mdBuilder).build();
+            }
+
+            @Override
+            public void onFailure(String source, @Nullable Throwable t) {
+                logger.error("error on metaData cleanup after test", t);
+            }
+        });
+        latch.await();
+        clear();
+
+    }
+
+    @Test
+    public void testEmptySignedLicenseCheck() {
+        LicensesManagerService licensesManagerService = licensesManagerService();
+        assertTrue(LicensesStatus.VALID == licensesManagerService.checkLicenses(LicenseBuilders.licensesBuilder().build()));
+    }
+
+    @Test
+    public void testInvalidSignedLicenseCheck() throws Exception {
+        LicensesManagerService licensesManagerService = licensesManagerService();
+
+        Map<ESLicenses.FeatureType, TestUtils.FeatureAttributes> map = new HashMap<>();
+        TestUtils.FeatureAttributes featureAttributes =
+                new TestUtils.FeatureAttributes("shield", "subscription", "platinum", "foo bar Inc.", "elasticsearch", 2, "2014-12-13", "2015-12-13");
+        map.put(ESLicenses.FeatureType.SHIELD, featureAttributes);
+        String licenseString = TestUtils.generateESLicenses(map);
+        String licenseOutput = TestUtils.runLicenseGenerationTool(licenseString, pubKeyPath, priKeyPath);
+        ESLicenses licenses = LicenseUtils.readLicensesFromString(licenseOutput);
+
+        assertTrue(LicensesStatus.VALID == licensesManagerService.checkLicenses(licenses));
+
+        ESLicenses.ESLicense tamperedLicense = LicenseBuilders.licenseBuilder(true)
+                .fromLicense(licenses.get(ESLicenses.FeatureType.SHIELD))
+                .expiryDate(licenses.get(ESLicenses.FeatureType.SHIELD).expiryDate() + 5 * 24 * 60 * 60 * 1000l)
+                .issuer("elasticsearch")
+                .build();
+
+        ESLicenses tamperedLicenses = LicenseBuilders.licensesBuilder().license(tamperedLicense).build();
+
+        assertTrue(LicensesStatus.INVALID == licensesManagerService.checkLicenses(tamperedLicenses));
+    }
+
+    @Test
+    public void testStoringLicenses() throws Exception {
+        Map<ESLicenses.FeatureType, TestUtils.FeatureAttributes> map = new HashMap<>();
+        TestUtils.FeatureAttributes featureAttributes1 =
+                new TestUtils.FeatureAttributes("shield", "subscription", "platinum", "foo bar Inc.", "elasticsearch", 2, "2014-12-13", "2015-12-13");
+        map.put(ESLicenses.FeatureType.SHIELD, featureAttributes1);
+        String licenseString = TestUtils.generateESLicenses(map);
+        String licenseOutput = TestUtils.runLicenseGenerationTool(licenseString, pubKeyPath, priKeyPath);
+        ESLicenses licenses = LicenseUtils.readLicensesFromString(licenseOutput);
+
+        LicensesManagerService licensesManagerService = licensesManagerService();
+        final CountDownLatch latch1 = new CountDownLatch(1);
+        licensesManagerService.registerLicenses(new LicensesService.PutLicenseRequestHolder(new PutLicenseRequest().license(licenses), "test"), new ActionListener<ClusterStateUpdateResponse>() {
+            @Override
+            public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {
+                if (clusterStateUpdateResponse.isAcknowledged()) {
+                    latch1.countDown();
+                }
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+
+            }
+        });
+
+        latch1.await();
+        LicensesMetaData metaData = clusterService().state().metaData().custom(LicensesMetaData.TYPE);
+        ESLicenses metaDataLicense = Utils.fromSignatures(metaData.getSignatures());
+        TestUtils.isSame(licenses, metaDataLicense);
+
+
+        TestUtils.FeatureAttributes featureAttributes2 =
+                new TestUtils.FeatureAttributes("shield", "subscription", "platinum", "foo bar Inc.", "elasticsearch", 2, "2014-12-13", "2016-12-13");
+        map.put(ESLicenses.FeatureType.SHIELD, featureAttributes2);
+        licenseString = TestUtils.generateESLicenses(map);
+        licenseOutput = TestUtils.runLicenseGenerationTool(licenseString, pubKeyPath, priKeyPath);
+        ESLicenses licenses2 = LicenseUtils.readLicensesFromString(licenseOutput);
+        final CountDownLatch latch2 = new CountDownLatch(1);
+        licensesManagerService.registerLicenses(new LicensesService.PutLicenseRequestHolder(new PutLicenseRequest().license(licenses2), "test"), new ActionListener<ClusterStateUpdateResponse>() {
+            @Override
+            public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {
+                if (clusterStateUpdateResponse.isAcknowledged()) {
+                    latch2.countDown();
+                }
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+
+            }
+        });
+
+        latch2.await();
+        metaData = clusterService().state().metaData().custom(LicensesMetaData.TYPE);
+        metaDataLicense = Utils.fromSignatures(metaData.getSignatures());
+        TestUtils.isSame(licenses2, metaDataLicense);
+    }
+
+    @Test
+    public void testTrialLicenseGeneration() throws Exception {
+        LicensesClientService clientService = licensesClientService();
+        final CountDownLatch latch = new CountDownLatch(1);
+        clientService.register("shield", new LicensesService.TrialLicenseOptions(10, 100), new LicensesClientService.Listener() {
+            @Override
+            public void onEnabled() {
+                latch.countDown();
+            }
+
+            @Override
+            public void onDisabled() {
+                fail();
+            }
+        });
+        latch.await();
+        final LicensesMetaData metaData = clusterService().state().metaData().custom(LicensesMetaData.TYPE);
+        assertTrue(metaData.getEncodedTrialLicenses().size() == 1);
+    }
+
+    @Test
+    public void testMultipleClientRegistration() {}
+
+    private class TestLicenseClientListener implements LicensesClientService.Listener {
+
+        AtomicBoolean shouldBeEnabled = new AtomicBoolean(false);
+        CountDownLatch latch = new CountDownLatch(2);
+        AtomicBoolean processed = new AtomicBoolean(false);
+
+        private TestLicenseClientListener(boolean shouldBeEnabled) {
+            this.shouldBeEnabled.getAndSet(shouldBeEnabled);
+        }
+
+        @Override
+        public void onEnabled() {
+            if (this.shouldBeEnabled.get()) {
+                latch.countDown();
+                processed.getAndSet(true);
+            } else {
+                fail("onEnabled should not have been called");
+            }
+
+        }
+
+        @Override
+        public void onDisabled() {
+            if (!this.shouldBeEnabled.get()) {
+                latch.countDown();
+                processed.getAndSet(true);
+            } else {
+                fail("onDisabled should not have been called");
+            }
+
+        }
+    }
+
+    @Test
+    public void testClientValidation() throws Exception {
+        // start with no trial license
+        // feature should be onDisabled
+        // then add signed license
+        // feature should be onEnabled
+
+        LicensesClientService clientService = licensesClientService();
+        final TestLicenseClientListener testLicenseClientListener = new TestLicenseClientListener(false);
+        clientService.register("shield", null, testLicenseClientListener);
+
+        while(!testLicenseClientListener.processed.get()) {}
+
+        testLicenseClientListener.shouldBeEnabled.getAndSet(true);
+        testLicenseClientListener.processed.getAndSet(false);
+
+        Map<ESLicenses.FeatureType, TestUtils.FeatureAttributes> map = new HashMap<>();
+        TestUtils.FeatureAttributes featureAttributes1 =
+                new TestUtils.FeatureAttributes("shield", "subscription", "platinum", "foo bar Inc.", "elasticsearch", 2, "2014-12-13", "2015-12-13");
+        map.put(ESLicenses.FeatureType.SHIELD, featureAttributes1);
+        String licenseString = TestUtils.generateESLicenses(map);
+        String licenseOutput = TestUtils.runLicenseGenerationTool(licenseString, pubKeyPath, priKeyPath);
+        ESLicenses licenses = LicenseUtils.readLicensesFromString(licenseOutput);
+
+        LicensesManagerService licensesManagerService = licensesManagerService();
+        final CountDownLatch latch1 = new CountDownLatch(1);
+        licensesManagerService.registerLicenses(new LicensesService.PutLicenseRequestHolder(new PutLicenseRequest().license(licenses), "test"), new ActionListener<ClusterStateUpdateResponse>() {
+            @Override
+            public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {
+                if (clusterStateUpdateResponse.isAcknowledged()) {
+                    latch1.countDown();
+                }
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+
+            }
+        });
+
+        latch1.await();
+
+        testLicenseClientListener.latch.await();
+    }
+
+    @Test
+    public void testFeatureWithoutLicense() throws Exception {
+        LicensesClientService clientService = licensesClientService();
+        final CountDownLatch latch = new CountDownLatch(1);
+        clientService.register("marvel", null, new LicensesClientService.Listener() {
+            @Override
+            public void onEnabled() {
+                fail();
+            }
+
+            @Override
+            public void onDisabled() {
+                latch.countDown();
+            }
+        });
+        latch.await();
+    }
+
+
+    private LicensesManagerService licensesManagerService() {
+        final InternalTestCluster clients = internalCluster();
+        return clients.getInstance(LicensesManagerService.class, clients.getMasterName());
+    }
+
+    private LicensesClientService licensesClientService() {
+        final InternalTestCluster clients = internalCluster();
+        return clients.getInstance(LicensesClientService.class, clients.getMasterName());
+    }
+
+    private ClusterService masterClusterService() {
+        final InternalTestCluster clients = internalCluster();
+        return clients.getInstance(ClusterService.class, clients.getMasterName());
+    }
+
+    private void clear() {
+        final InternalTestCluster clients = internalCluster();
+        LicensesService service = clients.getInstance(LicensesService.class, clients.getMasterName());
+        service.clear();
+    }
+
+}