LicensesService:

- Ensure license registration can be called through any stage of consumer plugin life cycle
 - Added randomized plugin integration tests
 - Run test on plugins with different license registration scheme

Original commit: elastic/x-pack-elasticsearch@1cee85943d

src/main/java/org/elasticsearch/license/plugin/core/LicensesService.java

@@ -18,6 +18,7 @@ import org.elasticsearch.common.collect.ImmutableMap;
 import org.elasticsearch.common.collect.ImmutableSet;
 import org.elasticsearch.common.collect.Sets;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.Singleton;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -50,24 +51,24 @@ import static org.elasticsearch.license.core.ESLicenses.reduceAndMap;
  * - LicensesManagerService - responsible for managing signed and one-time-trial licenses
  * - LicensesClientService - responsible for feature registration and notification to consumer plugin(s)
  * <p/>
- *
+ * <p/>
  * Notification Scheme:
- *
+ * <p/>
  * All registered feature(s) are notified using {@link #notifyFeatures(LicensesMetaData)} (depends on the current
  * {@link #registeredListeners}). It is idempotent with respect to all the feature listeners.
- *
+ * <p/>
  * The notification scheduling is done by {@link #notifyFeaturesAndScheduleNotification(LicensesMetaData)} which does the following:
  * - calls {@link #notifyFeatures(LicensesMetaData)} to notify all registered feature(s)
  * - if there is any license(s) with a future expiry date in the current cluster state:
  * - schedules a delayed {@link LicensingClientNotificationJob} on the MIN of all the expiry dates of all the registered feature(s)
- *
+ * <p/>
  * The {@link LicensingClientNotificationJob} calls {@link #notifyFeaturesAndScheduleNotification(LicensesMetaData)} to schedule
  * another delayed {@link LicensingClientNotificationJob} as stated above. It is a no-op in case of a global block on
  * {@link org.elasticsearch.gateway.GatewayService#STATE_NOT_RECOVERED_BLOCK}
- *
+ * <p/>
  * Upon successful registration of a new feature:
  * - {@link #notifyFeaturesAndScheduleNotification(LicensesMetaData)} is called
- *
+ * <p/>
  * Upon clusterChanged():
  * - {@link #notifyFeaturesAndScheduleNotification(LicensesMetaData)} is called if:
  * - new trial/signed license(s) are found in the cluster state meta data
@@ -386,38 +387,29 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
      */
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
-        final ClusterState currentClusterState = event.state();
         final ClusterState previousClusterState = event.previousState();
+        final ClusterState currentClusterState = event.state();
         if (!currentClusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
-            LicensesMetaData oldLicensesMetaData = previousClusterState.getMetaData().custom(LicensesMetaData.TYPE);
+            final LicensesMetaData oldLicensesMetaData = previousClusterState.getMetaData().custom(LicensesMetaData.TYPE);
-            LicensesMetaData currentLicensesMetaData = currentClusterState.getMetaData().custom(LicensesMetaData.TYPE);
+            final LicensesMetaData currentLicensesMetaData = currentClusterState.getMetaData().custom(LicensesMetaData.TYPE);
             logLicenseMetaDataStats("old", oldLicensesMetaData);
             logLicenseMetaDataStats("new", currentLicensesMetaData);
 
-            // Check pending feature registrations and try to complete registrations
-            boolean addedNewRegisteredListener = false;
             if (!pendingListeners.isEmpty()) {
-                ListenerHolder pendingRegistrationLister;
+                ListenerHolder pendingRegistrationListener;
-                while ((pendingRegistrationLister = pendingListeners.poll()) != null) {
+                while ((pendingRegistrationListener = pendingListeners.poll()) != null) {
-                    boolean masterAvailable = registerListener(pendingRegistrationLister);
+                    boolean masterAvailable = registerListener(pendingRegistrationListener);
-                    logger.debug("trying to register pending listener for " + pendingRegistrationLister.feature + " masterAvailable: " + masterAvailable);
+                    logger.debug("trying to register pending listener for " + pendingRegistrationListener.feature + " masterAvailable: " + masterAvailable);
                     if (!masterAvailable) {
                         // if the master is not available do not, break out of trying pendingListeners
-                        pendingListeners.add(pendingRegistrationLister);
+                        pendingListeners.add(pendingRegistrationListener);
                         break;
-                    } else {
-                        logger.debug("successfully registered listener for: " + pendingRegistrationLister.feature);
-                        registeredListeners.add(pendingRegistrationLister);
-                        // make sure to notify new registered feature
-                        // notifications could have been scheduled for it before it was registered
-                        addedNewRegisteredListener = true;
                     }
                 }
             }
 
             // notify all interested plugins
-            // Change to debug
+            if (previousClusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
-            if (previousClusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK) || addedNewRegisteredListener) {
                 logger.debug("calling notifyFeaturesAndScheduleNotification from clusterChanged");
                 notifyFeaturesAndScheduleNotification(currentLicensesMetaData);
             } else {
@@ -433,7 +425,7 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
      * Calls {@link #notifyFeaturesAndScheduleNotification(LicensesMetaData)} with <code>currentLicensesMetaData</code>
      * if it was not already notified on
      */
-    private void notifyFeaturesAndScheduleNotificationIfNeeded(LicensesMetaData currentLicensesMetaData) {
+    private void notifyFeaturesAndScheduleNotificationIfNeeded(final LicensesMetaData currentLicensesMetaData) {
         final LicensesMetaData lastNotifiedLicensesMetaData = lastObservedLicensesState.get();
         if (lastNotifiedLicensesMetaData != null && lastNotifiedLicensesMetaData.equals(currentLicensesMetaData)) {
             logger.debug("currentLicensesMetaData has been already notified on");
@@ -446,7 +438,7 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
      * Calls {@link #notifyFeatures(LicensesMetaData)} with <code>currentLicensesMetaData</code>
      * and schedules the earliest expiry (if any) notification for registered feature(s)
      */
-    private void notifyFeaturesAndScheduleNotification(LicensesMetaData currentLicensesMetaData) {
+    private void notifyFeaturesAndScheduleNotification(final LicensesMetaData currentLicensesMetaData) {
         long nextScheduleFrequency = notifyFeatures(currentLicensesMetaData);
         if (nextScheduleFrequency != -1l) {
             scheduleNextNotification(nextScheduleFrequency);
@@ -463,7 +455,7 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
      * returns the minimum of the expiry times of all the registered feature(s) to
      * schedule an expiry notification
      */
-    private long notifyFeatures(LicensesMetaData currentLicensesMetaData) {
+    private long notifyFeatures(final LicensesMetaData currentLicensesMetaData) {
         long nextScheduleFrequency = -1l;
         for (ListenerHolder listenerHolder : registeredListeners) {
             long expiryDate = expiryDateForFeature(listenerHolder.feature, currentLicensesMetaData);
@@ -496,8 +488,6 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
             }
         }
 
-        lastObservedLicensesState.set(currentLicensesMetaData);
-
         if (logger.isDebugEnabled()) {
             logLicenseMetaDataStats("Setting last observed metaData", currentLicensesMetaData);
             if (nextScheduleFrequency == -1l) {
@@ -507,9 +497,11 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
             }
         }
 
+        lastObservedLicensesState.set(currentLicensesMetaData);
         return nextScheduleFrequency;
 
     }
+
     private void logLicenseMetaDataStats(String prefix, LicensesMetaData licensesMetaData) {
         if (licensesMetaData != null) {
             logger.debug(prefix + " LicensesMetaData: signedLicenses: " + licensesMetaData.getSignatures().size() + " trialLicenses: " + licensesMetaData.getEncodedTrialLicenses().size());
@@ -524,23 +516,33 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
     @Override
     public void register(String feature, TrialLicenseOptions trialLicenseOptions, Listener listener) {
         final ListenerHolder listenerHolder = new ListenerHolder(feature, trialLicenseOptions, listener);
-       logger.debug("add listener for: " + listenerHolder.feature + " to pending registration queue");
+        // don't trust the clusterState for blocks just yet!
+        final Lifecycle.State state = clusterService.lifecycleState();
+        if (state != Lifecycle.State.STARTED) {
             pendingListeners.add(listenerHolder);
+        } else {
+            if (!registerListener(listenerHolder)) {
+                pendingListeners.add(listenerHolder);
+            }
+        }
     }
 
     /**
      * Notifies new feature listener if it already has a signed license
      * if new feature has a non-null trial license option, a master node request is made to generate the trial license
-     * if no trial license option is specified for the feature and no signed license is found,
+     * then notifies features if needed
-     * then notifies features to be disabled
-     * then notifies features to be disabled
      *
      * @param listenerHolder of the feature to register
-     * @return true if registration has been completed, false otherwise (if masterNode is not available & trail license spec is provided
+     * @return true if registration has been completed, false otherwise (if masterNode is not available & trail license spec is provided)
+     * or if there is a global block on {@link org.elasticsearch.gateway.GatewayService#STATE_NOT_RECOVERED_BLOCK}
      */
     private boolean registerListener(final ListenerHolder listenerHolder) {
         logger.debug("Registering listener for " + listenerHolder.feature);
-        ClusterState currentState = clusterService.state();
+        final ClusterState currentState = clusterService.state();
+        if (currentState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            return false;
+        }
+
         LicensesMetaData currentMetaData = currentState.metaData().custom(LicensesMetaData.TYPE);
         if (expiryDateForFeature(listenerHolder.feature, currentMetaData) == -1l) {
             // does not have any license so generate a trial license
@@ -564,17 +566,20 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
                         return false;
                     }
                 }
+                registeredListeners.add(listenerHolder);
             } else {
                 // notify feature as clusterChangedEvent may not happen
                 // as no trial or signed license has been found for feature
                 // Change to debug
                 logger.debug("Calling notifyFeaturesAndScheduleNotification [no trial license spec provided]");
+                registeredListeners.add(listenerHolder);
                 notifyFeaturesAndScheduleNotification(currentMetaData);
             }
         } else {
             // signed license already found for the new registered
             // feature, notify feature on registration
             logger.debug("Calling notifyFeaturesAndScheduleNotification [signed/trial license available]");
+            registeredListeners.add(listenerHolder);
             notifyFeaturesAndScheduleNotification(currentMetaData);
         }
         return true;
@@ -584,8 +589,6 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
         final Map<String, ESLicense> effectiveLicenses = getEffectiveLicenses(currentLicensesMetaData);
         ESLicense featureLicense;
         if ((featureLicense = effectiveLicenses.get(feature)) != null) {
-            logger.debug("effective license for " + feature + " relative expiry: " +
-                    TimeValue.timeValueMillis(effectiveLicenses.get(feature).expiryDate() - System.currentTimeMillis()));
             return featureLicense.expiryDate();
         }
         logger.debug("no effective license for " + feature);
@@ -648,7 +651,8 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
      */
     private class LicensingClientNotificationJob implements Runnable {
 
-        public LicensingClientNotificationJob() {}
+        public LicensingClientNotificationJob() {
+        }
 
         @Override
         public void run() {
@@ -766,7 +770,8 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
         private String feature;
         private TimeValue duration;
 
-        private RegisterTrialLicenseRequest() {}
+        private RegisterTrialLicenseRequest() {
+        }
 
         private RegisterTrialLicenseRequest(String feature, TimeValue duration, int maxNodes) {
             this.maxNodes = maxNodes;

src/test/java/org/elasticsearch/license/plugin/AbstractLicensesIntegrationTests.java

@@ -16,14 +16,13 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.license.core.ESLicense;
 import org.elasticsearch.license.licensor.ESLicenseSigner;
-import org.elasticsearch.license.plugin.consumer.TestPluginService1;
+import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService;
-import org.elasticsearch.license.plugin.consumer.TestPluginService2;
+import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationPluginService;
 import org.elasticsearch.license.plugin.consumer.TestPluginServiceBase;
 import org.elasticsearch.license.plugin.core.LicensesManagerService;
 import org.elasticsearch.license.plugin.core.LicensesMetaData;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import org.elasticsearch.test.InternalTestCluster;
-import org.hamcrest.Matchers;
 
 import java.util.*;
 import java.util.concurrent.CountDownLatch;
@@ -115,30 +114,30 @@ public abstract class AbstractLicensesIntegrationTests extends ElasticsearchInte
         }, 2, TimeUnit.SECONDS), equalTo(true));
     }
 
-    protected void assertConsumerPlugin1DisableNotification(int timeoutInSec) throws InterruptedException {
+    protected void assertEagerConsumerPluginDisableNotification(int timeoutInSec) throws InterruptedException {
-        assertConsumerPlugin1Notification(false, timeoutInSec);
+        assertEagerConsumerPluginNotification(false, timeoutInSec);
     }
 
-    protected void assertConsumerPlugin1EnableNotification(int timeoutInSec) throws InterruptedException {
+    protected void assertEagerConsumerPluginEnableNotification(int timeoutInSec) throws InterruptedException {
-        assertConsumerPlugin1Notification(true, timeoutInSec);
+        assertEagerConsumerPluginNotification(true, timeoutInSec);
     }
 
-    protected void assertConsumerPlugin2DisableNotification(int timeoutInSec) throws InterruptedException {
+    protected void assertLazyConsumerPluginDisableNotification(int timeoutInSec) throws InterruptedException {
-        assertConsumerPlugin2Notification(false, timeoutInSec);
+        assertLazyConsumerPluginNotification(false, timeoutInSec);
     }
 
-    protected void assertConsumerPlugin2EnableNotification(int timeoutInSec) throws InterruptedException {
+    protected void assertLazyConsumerPluginEnableNotification(int timeoutInSec) throws InterruptedException {
-        assertConsumerPlugin2Notification(true, timeoutInSec);
+        assertLazyConsumerPluginNotification(true, timeoutInSec);
     }
 
-    protected void assertConsumerPlugin2Notification(final boolean expectedEnabled, int timeoutInSec) throws InterruptedException {
+    protected void assertLazyConsumerPluginNotification(final boolean expectedEnabled, int timeoutInSec) throws InterruptedException {
-        final List<TestPluginServiceBase> consumerPluginServices = consumerPlugin2Services();
+        final List<TestPluginServiceBase> consumerPluginServices = consumerLazyPluginServices();
         assertThat("At least one instance has to be present", consumerPluginServices.size(), greaterThan(0));
         assertConsumerPluginNotification(consumerPluginServices, expectedEnabled, timeoutInSec);
     }
 
-    protected void assertConsumerPlugin1Notification(final boolean expectedEnabled, int timeoutInSec) throws InterruptedException {
+    protected void assertEagerConsumerPluginNotification(final boolean expectedEnabled, int timeoutInSec) throws InterruptedException {
-        final List<TestPluginServiceBase> consumerPluginServices = consumerPlugin1Services();
+        final List<TestPluginServiceBase> consumerPluginServices = consumerEagerPluginServices();
         assertThat("At least one instance has to be present", consumerPluginServices.size(), greaterThan(0));
         assertConsumerPluginNotification(consumerPluginServices, expectedEnabled, timeoutInSec);
     }
@@ -158,19 +157,19 @@ public abstract class AbstractLicensesIntegrationTests extends ElasticsearchInte
 
     }
 
-    private List<TestPluginServiceBase> consumerPlugin2Services() {
+    private List<TestPluginServiceBase> consumerLazyPluginServices() {
         final InternalTestCluster clients = internalCluster();
         List<TestPluginServiceBase> consumerPluginServices = new ArrayList<>();
-        for (TestPluginServiceBase service : clients.getDataNodeInstances(TestPluginService2.class)) {
+        for (TestPluginServiceBase service : clients.getDataNodeInstances(LazyLicenseRegistrationPluginService.class)) {
             consumerPluginServices.add(service);
         }
         return consumerPluginServices;
     }
 
-    private List<TestPluginServiceBase> consumerPlugin1Services() {
+    private List<TestPluginServiceBase> consumerEagerPluginServices() {
         final InternalTestCluster clients = internalCluster();
         List<TestPluginServiceBase> consumerPluginServices = new ArrayList<>();
-        for (TestPluginServiceBase service : clients.getDataNodeInstances(TestPluginService1.class)) {
+        for (TestPluginServiceBase service : clients.getDataNodeInstances(EagerLicenseRegistrationPluginService.class)) {
             consumerPluginServices.add(service);
         }
         return consumerPluginServices;

src/test/java/org/elasticsearch/license/plugin/LicensesPluginIntegrationTests.java

@@ -5,15 +5,19 @@
  */
 package org.elasticsearch.license.plugin;
 
+import org.elasticsearch.common.base.Predicate;
 import org.elasticsearch.common.collect.Lists;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.license.core.ESLicense;
-import org.elasticsearch.license.plugin.consumer.TestConsumerPlugin1;
+import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationConsumerPlugin;
-import org.elasticsearch.license.plugin.consumer.TestPluginService1;
+import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService;
 import org.elasticsearch.license.plugin.action.put.PutLicenseRequestBuilder;
 import org.elasticsearch.license.plugin.action.put.PutLicenseResponse;
+import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationConsumerPlugin;
+import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationPluginService;
 import org.elasticsearch.license.plugin.core.LicensesStatus;
 import org.junit.After;
 import org.junit.Test;
@@ -25,100 +29,134 @@ import static org.hamcrest.CoreMatchers.equalTo;
 @ClusterScope(scope = TEST, numDataNodes = 10, numClientNodes = 0)
 public class LicensesPluginIntegrationTests extends AbstractLicensesIntegrationTests {
 
-    private final int trialLicenseDurationInSeconds = 2;
+    private final boolean useEagerLicenseRegistrationPlugin = randomBoolean();
 
-    private final String FEATURE_NAME = TestPluginService1.FEATURE_NAME;
+    private final int trialLicenseDurationInSeconds = 2;
     
     protected Settings nodeSettings(int nodeOrdinal) {
         return ImmutableSettings.settingsBuilder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(TestConsumerPlugin1.NAME + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
+                .put(((useEagerLicenseRegistrationPlugin) ? EagerLicenseRegistrationConsumerPlugin.NAME : LazyLicenseRegistrationConsumerPlugin.NAME)
-                .put("plugin.types", LicensePlugin.class.getName() + "," + TestConsumerPlugin1.class.getName())
+                        + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
+                .putArray("plugin.types", LicensePlugin.class.getName(),
+                        (useEagerLicenseRegistrationPlugin) ? EagerLicenseRegistrationConsumerPlugin.class.getName() : LazyLicenseRegistrationConsumerPlugin.class.getName())
                 .build();
     }
 
     @After
-    public void beforeTest() throws Exception {
+    public void afterTest() throws Exception {
         wipeAllLicenses();
+        assertThat(awaitBusy(new Predicate<Object>() {
+            @Override
+            public boolean apply(Object o) {
+                return !clusterService().state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK);
+            }
+        }), equalTo(true));
     }
 
     @Test
     public void testTrialLicenseAndSignedLicenseNotification() throws Exception {
+        logger.info("using "+ ((useEagerLicenseRegistrationPlugin) ? "eager" : "lazy") + " consumer plugin");
         logger.info(" --> trial license generated");
         // managerService should report feature to be enabled on all data nodes
-        assertLicenseManagerEnabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerEnabledFeatureFor(getCurrentFeatureName());
         // consumer plugin service should return enabled on all data nodes
-        assertConsumerPlugin1EnableNotification(1);
+        assertConsumerPluginEnabledNotification(1);
 
         logger.info(" --> check trial license expiry notification");
         // consumer plugin should notify onDisabled on all data nodes (expired trial license)
-        assertConsumerPlugin1DisableNotification(trialLicenseDurationInSeconds * 2);
+        assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds * 2);
-        assertLicenseManagerDisabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerDisabledFeatureFor(getCurrentFeatureName());
 
         logger.info(" --> put signed license");
-        ESLicense license = generateSignedLicense(FEATURE_NAME, TimeValue.timeValueSeconds(trialLicenseDurationInSeconds));
+        ESLicense license = generateSignedLicense(getCurrentFeatureName(), TimeValue.timeValueSeconds(trialLicenseDurationInSeconds));
         final PutLicenseResponse putLicenseResponse = new PutLicenseRequestBuilder(client().admin().cluster()).setLicense(Lists.newArrayList(license)).get();
         assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
 
         logger.info(" --> check signed license enabled notification");
         // consumer plugin should notify onEnabled on all data nodes (signed license)
-        assertConsumerPlugin1EnableNotification(1);
+        assertConsumerPluginEnabledNotification(1);
-        assertLicenseManagerEnabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerEnabledFeatureFor(getCurrentFeatureName());
 
         logger.info(" --> check signed license expiry notification");
         // consumer plugin should notify onDisabled on all data nodes (expired signed license)
-        assertConsumerPlugin1DisableNotification(trialLicenseDurationInSeconds * 2);
+        assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds * 2);
-        assertLicenseManagerDisabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerDisabledFeatureFor(getCurrentFeatureName());
     }
 
     @Test
     public void testTrialLicenseNotification() throws Exception {
         logger.info(" --> check onEnabled for trial license");
         // managerService should report feature to be enabled on all data nodes
-        assertLicenseManagerEnabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerEnabledFeatureFor(getCurrentFeatureName());
         // consumer plugin service should return enabled on all data nodes
-        assertConsumerPlugin1EnableNotification(1);
+        assertConsumerPluginEnabledNotification(1);
 
         logger.info(" --> sleep for rest of trailLicense duration");
         Thread.sleep(trialLicenseDurationInSeconds * 1000l);
 
         logger.info(" --> check trial license expiry notification");
         // consumer plugin should notify onDisabled on all data nodes (expired signed license)
-        assertConsumerPlugin1DisableNotification(trialLicenseDurationInSeconds);
+        assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds);
-        assertLicenseManagerDisabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerDisabledFeatureFor(getCurrentFeatureName());
     }
 
     @Test
     public void testOverlappingTrialAndSignedLicenseNotification() throws Exception {
         logger.info(" --> check onEnabled for trial license");
         // managerService should report feature to be enabled on all data nodes
-        assertLicenseManagerEnabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerEnabledFeatureFor(getCurrentFeatureName());
         // consumer plugin service should return enabled on all data nodes
-        assertConsumerPlugin1EnableNotification(1);
+        assertConsumerPluginEnabledNotification(1);
 
         logger.info(" --> put signed license while trial license is in effect");
-        ESLicense license = generateSignedLicense(FEATURE_NAME, TimeValue.timeValueSeconds(trialLicenseDurationInSeconds * 2));
+        ESLicense license = generateSignedLicense(getCurrentFeatureName(), TimeValue.timeValueSeconds(trialLicenseDurationInSeconds * 2));
         final PutLicenseResponse putLicenseResponse = new PutLicenseRequestBuilder(client().admin().cluster()).setLicense(Lists.newArrayList(license)).get();
         assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
 
         logger.info(" --> check signed license enabled notification");
         // consumer plugin should notify onEnabled on all data nodes (signed license)
-        assertConsumerPlugin1EnableNotification(1);
+        assertConsumerPluginEnabledNotification(1);
-        assertLicenseManagerEnabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerEnabledFeatureFor(getCurrentFeatureName());
 
         logger.info(" --> sleep for rest of trailLicense duration");
         Thread.sleep(trialLicenseDurationInSeconds * 1000l);
 
         logger.info(" --> check consumer is still enabled [signed license]");
         // consumer plugin should notify onEnabled on all data nodes (signed license)
-        assertConsumerPlugin1EnableNotification(1);
+        assertConsumerPluginEnabledNotification(1);
-        assertLicenseManagerEnabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerEnabledFeatureFor(getCurrentFeatureName());
 
         logger.info(" --> check signed license expiry notification");
         // consumer plugin should notify onDisabled on all data nodes (expired signed license)
-        assertConsumerPlugin1DisableNotification(trialLicenseDurationInSeconds * 2 * 2);
+        assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds * 2 * 2);
-        assertLicenseManagerDisabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerDisabledFeatureFor(getCurrentFeatureName());
     }
+
+    private String getCurrentFeatureName() {
+        if (useEagerLicenseRegistrationPlugin) {
+            return EagerLicenseRegistrationPluginService.FEATURE_NAME;
+        } else {
+            return LazyLicenseRegistrationPluginService.FEATURE_NAME;
+        }
+    }
+
+    private void assertConsumerPluginEnabledNotification(int timeoutInSec) throws InterruptedException {
+        if (useEagerLicenseRegistrationPlugin) {
+            assertEagerConsumerPluginEnableNotification(timeoutInSec);
+        } else {
+            assertLazyConsumerPluginEnableNotification(timeoutInSec);
+        }
+    }
+
+    private void assertConsumerPluginDisabledNotification(int timeoutInSec) throws InterruptedException {
+        if (useEagerLicenseRegistrationPlugin) {
+            assertEagerConsumerPluginDisableNotification(timeoutInSec);
+        } else {
+            assertLazyConsumerPluginDisableNotification(timeoutInSec);
+        }
+    }
+
 }

src/test/java/org/elasticsearch/license/plugin/LicensesPluginsIntegrationTests.java

@@ -5,17 +5,19 @@
  */
 package org.elasticsearch.license.plugin;
 
+import org.elasticsearch.common.base.Predicate;
 import org.elasticsearch.common.collect.Lists;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.license.core.ESLicense;
 import org.elasticsearch.license.plugin.action.put.PutLicenseRequestBuilder;
 import org.elasticsearch.license.plugin.action.put.PutLicenseResponse;
-import org.elasticsearch.license.plugin.consumer.TestConsumerPlugin1;
+import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationConsumerPlugin;
-import org.elasticsearch.license.plugin.consumer.TestConsumerPlugin2;
+import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService;
-import org.elasticsearch.license.plugin.consumer.TestPluginService1;
+import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationConsumerPlugin;
-import org.elasticsearch.license.plugin.consumer.TestPluginService2;
+import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationPluginService;
 import org.elasticsearch.license.plugin.core.LicensesStatus;
 import org.junit.After;
 import org.junit.Test;
@@ -24,29 +26,30 @@ import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope.TEST;
 import static org.hamcrest.CoreMatchers.equalTo;
 
+//@Ignore
 @ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0)
 public class LicensesPluginsIntegrationTests extends AbstractLicensesIntegrationTests {
 
     private final int trialLicenseDurationInSeconds = 2;
     
-    private final String FEATURE_NAME_1 = TestPluginService1.FEATURE_NAME;
+    private final String FEATURE_NAME_1 = EagerLicenseRegistrationPluginService.FEATURE_NAME;
-    private final String FEATURE_NAME_2 = TestPluginService2.FEATURE_NAME;
+    private final String FEATURE_NAME_2 = LazyLicenseRegistrationPluginService.FEATURE_NAME;
 
     protected Settings nodeSettings(int nodeOrdinal) {
         return ImmutableSettings.settingsBuilder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(TestConsumerPlugin1.NAME + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
+                .put(EagerLicenseRegistrationConsumerPlugin.NAME + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
-                .put(TestConsumerPlugin2.NAME + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
+                .put(LazyLicenseRegistrationConsumerPlugin.NAME + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
-                .putArray("plugin.types", LicensePlugin.class.getName(), TestConsumerPlugin1.class.getName(), TestConsumerPlugin2.class.getName())
+                .putArray("plugin.types", LicensePlugin.class.getName(), EagerLicenseRegistrationConsumerPlugin.class.getName(), LazyLicenseRegistrationConsumerPlugin.class.getName())
                 .build();
     }
 
     private Settings nodeSettingsWithConsumerPlugin(int consumer1TrialLicenseDuration, int consumer2TrialLicenseDuration) {
         return ImmutableSettings.settingsBuilder()
                 .put(super.nodeSettings(0))
-                .put(TestConsumerPlugin1.NAME + ".trial_license_duration_in_seconds", consumer1TrialLicenseDuration)
+                .put(EagerLicenseRegistrationConsumerPlugin.NAME + ".trial_license_duration_in_seconds", consumer1TrialLicenseDuration)
-                .put(TestConsumerPlugin2.NAME + ".trial_license_duration_in_seconds", consumer2TrialLicenseDuration)
+                .put(LazyLicenseRegistrationConsumerPlugin.NAME + ".trial_license_duration_in_seconds", consumer2TrialLicenseDuration)
-                .putArray("plugin.types", LicensePlugin.class.getName(), TestConsumerPlugin1.class.getName(), TestConsumerPlugin2.class.getName())
+                .putArray("plugin.types", LicensePlugin.class.getName(), EagerLicenseRegistrationConsumerPlugin.class.getName(), LazyLicenseRegistrationConsumerPlugin.class.getName())
                 .build();
 
     }
@@ -61,8 +64,8 @@ public class LicensesPluginsIntegrationTests extends AbstractLicensesIntegration
         int nNodes = randomIntBetween(2, 10);
         String[] nodes = startNodesWithConsumerPlugins(nNodes, -1, -1);
 
-        assertConsumerPlugin1DisableNotification(trialLicenseDurationInSeconds * 2);
+        assertEagerConsumerPluginDisableNotification(trialLicenseDurationInSeconds * 2);
-        assertConsumerPlugin2DisableNotification(trialLicenseDurationInSeconds * 2);
+        assertLazyConsumerPluginDisableNotification(trialLicenseDurationInSeconds * 2);
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_1);
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_2);
     }
@@ -78,8 +81,8 @@ public class LicensesPluginsIntegrationTests extends AbstractLicensesIntegration
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_1);
         assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_2);
         // consumer plugin service should return enabled on all data nodes
-        assertConsumerPlugin1DisableNotification(1);
+        assertEagerConsumerPluginDisableNotification(1);
-        assertConsumerPlugin2EnableNotification(1);
+        assertLazyConsumerPluginEnableNotification(1);
 
         logger.info(" --> put signed license for " + FEATURE_NAME_1);
         ESLicense license1 = generateSignedLicense(FEATURE_NAME_1, TimeValue.timeValueSeconds(consumer2TrialLicenseDuration));
@@ -88,15 +91,15 @@ public class LicensesPluginsIntegrationTests extends AbstractLicensesIntegration
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
 
         logger.info(" --> check that both " + FEATURE_NAME_1 + " and " + FEATURE_NAME_2 + " are enabled");
-        assertConsumerPlugin1EnableNotification(1);
+        assertEagerConsumerPluginEnableNotification(1);
-        assertConsumerPlugin2EnableNotification(1);
+        assertLazyConsumerPluginEnableNotification(1);
         assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_1);
         assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_2);
 
         logger.info(" --> check signed license expiry notification");
         // consumer plugin should notify onDisabled on all data nodes (expired signed license)
-        assertConsumerPlugin1DisableNotification(consumer2TrialLicenseDuration * 2);
+        assertEagerConsumerPluginDisableNotification(consumer2TrialLicenseDuration * 2);
-        assertConsumerPlugin2DisableNotification(consumer2TrialLicenseDuration * 2);
+        assertLazyConsumerPluginDisableNotification(consumer2TrialLicenseDuration * 2);
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_1);
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_2);
     }
@@ -106,19 +109,30 @@ public class LicensesPluginsIntegrationTests extends AbstractLicensesIntegration
 
         int nNodes = randomIntBetween(2, 10);
         String[] nodes = startNodesWithConsumerPlugins(nNodes, trialLicenseDurationInSeconds, trialLicenseDurationInSeconds);
-
+        waitUntilClusterRecovered();
+        assertThat(awaitBusy(new Predicate<Object>() {
+            @Override
+            public boolean apply(Object o) {
+                for (LazyLicenseRegistrationPluginService service : internalCluster().getDataNodeInstances(LazyLicenseRegistrationPluginService.class)) {
+                    if (!service.registered.get()) {
+                        return false;
+                    }
+                }
+                return true;
+            }
+        }), equalTo(true));
         logger.info(" --> trial license generated");
         // managerService should report feature to be enabled on all data nodes
         assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_1);
         assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_2);
         // consumer plugin service should return enabled on all data nodes
-        assertConsumerPlugin1EnableNotification(1);
+        assertEagerConsumerPluginEnableNotification(1);
-        assertConsumerPlugin2EnableNotification(1);
+        assertLazyConsumerPluginEnableNotification(1);
 
         logger.info(" --> check trial license expiry notification");
         // consumer plugin should notify onDisabled on all data nodes (expired trial license)
-        assertConsumerPlugin1DisableNotification(trialLicenseDurationInSeconds * 2);
+        assertEagerConsumerPluginDisableNotification(trialLicenseDurationInSeconds * 2);
-        assertConsumerPlugin2DisableNotification(trialLicenseDurationInSeconds * 2);
+        assertLazyConsumerPluginDisableNotification(trialLicenseDurationInSeconds * 2);
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_1);
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_2);
 
@@ -128,15 +142,15 @@ public class LicensesPluginsIntegrationTests extends AbstractLicensesIntegration
 
         logger.info(" --> check signed license enabled notification");
         // consumer plugin should notify onEnabled on all data nodes (signed license)
-        assertConsumerPlugin1EnableNotification(1);
+        assertEagerConsumerPluginEnableNotification(1);
-        assertConsumerPlugin2EnableNotification(1);
+        assertLazyConsumerPluginEnableNotification(1);
         assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_1);
         assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_2);
 
         logger.info(" --> check signed license expiry notification");
         // consumer plugin should notify onDisabled on all data nodes (expired signed license)
-        assertConsumerPlugin1DisableNotification(trialLicenseDurationInSeconds * 2);
+        assertEagerConsumerPluginDisableNotification(trialLicenseDurationInSeconds * 2);
-        assertConsumerPlugin2DisableNotification(trialLicenseDurationInSeconds * 2);
+        assertLazyConsumerPluginDisableNotification(trialLicenseDurationInSeconds * 2);
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_1);
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_2);
     }
@@ -150,33 +164,33 @@ public class LicensesPluginsIntegrationTests extends AbstractLicensesIntegration
         String[] nodes = startNodesWithConsumerPlugins(nNodes, trialLicenseDuration1, trialLicenseDuration2);
 
         if (trialLicenseDuration1 != -1) {
-            assertConsumerPlugin1EnableNotification(1);
+            assertEagerConsumerPluginEnableNotification(1);
             assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_1);
         } else {
-            assertConsumerPlugin1DisableNotification(1);
+            assertEagerConsumerPluginDisableNotification(1);
             assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_1);
             putLicense(FEATURE_NAME_1, TimeValue.timeValueMillis(300 * 2));
         }
 
         if (trialLicenseDuration2 != -1) {
-            assertConsumerPlugin2EnableNotification(1);
+            assertLazyConsumerPluginEnableNotification(1);
             assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_2);
         } else {
-            assertConsumerPlugin2DisableNotification(1);
+            assertLazyConsumerPluginDisableNotification(1);
             assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_2);
             putLicense(FEATURE_NAME_2, TimeValue.timeValueMillis(300 * 2));
         }
 
         logger.info(" --> check license enabled notification");
-        assertConsumerPlugin1EnableNotification(1);
+        assertEagerConsumerPluginEnableNotification(1);
-        assertConsumerPlugin2EnableNotification(1);
+        assertLazyConsumerPluginEnableNotification(1);
         assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_1);
         assertLicenseManagerEnabledFeatureFor(FEATURE_NAME_2);
 
         logger.info(" --> check license expiry notification");
         // consumer plugin should notify onDisabled on all data nodes (expired signed license)
-        assertConsumerPlugin1DisableNotification(2 * 2);
+        assertEagerConsumerPluginDisableNotification(2 * 2);
-        assertConsumerPlugin2DisableNotification(2 * 2);
+        assertLazyConsumerPluginDisableNotification(2 * 2);
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_1);
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME_2);
 
@@ -196,4 +210,13 @@ public class LicensesPluginsIntegrationTests extends AbstractLicensesIntegration
         assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
     }
+
+    private void waitUntilClusterRecovered() throws InterruptedException {
+        assertThat(awaitBusy(new Predicate<Object>() {
+            @Override
+            public boolean apply(Object o) {
+                return !clusterService().state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK);
+            }
+        }), equalTo(true));
+    }
 }

src/test/java/org/elasticsearch/license/plugin/LicensesServiceClusterTest.java

@@ -6,22 +6,17 @@
 package org.elasticsearch.license.plugin;
 
 import org.elasticsearch.client.ClusterAdminClient;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.common.base.Predicate;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.DiscoverySettings;
-import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.license.TestUtils;
 import org.elasticsearch.license.core.ESLicense;
 import org.elasticsearch.license.plugin.action.get.GetLicenseRequestBuilder;
 import org.elasticsearch.license.plugin.action.get.GetLicenseResponse;
 import org.elasticsearch.license.plugin.action.put.PutLicenseRequestBuilder;
 import org.elasticsearch.license.plugin.action.put.PutLicenseResponse;
-import org.elasticsearch.license.plugin.consumer.TestConsumerPlugin1;
+import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationConsumerPlugin;
-import org.elasticsearch.license.plugin.consumer.TestPluginService1;
+import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService;
 import org.elasticsearch.license.plugin.core.LicensesStatus;
 import org.elasticsearch.node.internal.InternalNode;
 import org.junit.Test;
@@ -37,7 +32,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 @ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0, transportClientRatio = 0)
 public class LicensesServiceClusterTest extends AbstractLicensesIntegrationTests {
 
-    private final String FEATURE_NAME = TestPluginService1.FEATURE_NAME;
+    private final String FEATURE_NAME = EagerLicenseRegistrationPluginService.FEATURE_NAME;
 
     private final int trialLicenseDurationInSeconds = 2;
     
@@ -57,8 +52,8 @@ public class LicensesServiceClusterTest extends AbstractLicensesIntegrationTests
                 .put("plugins.load_classpath_plugins", false)
                 .put("node.data", true)
                 .put("format", "json")
-                .put(TestConsumerPlugin1.NAME + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
+                .put(EagerLicenseRegistrationConsumerPlugin.NAME + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
-                .putArray("plugin.types", LicensePlugin.class.getName(), TestConsumerPlugin1.class.getName())
+                .putArray("plugin.types", LicensePlugin.class.getName(), EagerLicenseRegistrationConsumerPlugin.class.getName())
                 .put(InternalNode.HTTP_ENABLED, true);
     }
 
@@ -91,12 +86,12 @@ public class LicensesServiceClusterTest extends AbstractLicensesIntegrationTests
         logger.info("--> start one master out of two [recovery state]");
         internalCluster().startNode(nodeSettingsBuilder(0).put("discovery.zen.minimum_master_nodes", 2).put("node.master", true));
         assertLicenseManagerDisabledFeatureFor(FEATURE_NAME);
-        assertConsumerPlugin1DisableNotification(1);
+        assertEagerConsumerPluginDisableNotification(1);
 
         logger.info("--> start second master out of two [recovered state]");
         internalCluster().startNode(nodeSettingsBuilder(1).put("discovery.zen.minimum_master_nodes", 2).put("node.master", true));
         assertLicenseManagerEnabledFeatureFor(FEATURE_NAME);
-        assertConsumerPlugin1EnableNotification(1);
+        assertEagerConsumerPluginEnableNotification(1);
     }
 
     private List<ESLicense> generateAndPutLicense() throws Exception {

src/test/java/org/elasticsearch/license/plugin/LicensesServiceNodeTests.java

@@ -8,8 +8,8 @@ package org.elasticsearch.license.plugin;
 import org.elasticsearch.common.base.Predicate;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.license.plugin.consumer.TestConsumerPlugin1;
+import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationConsumerPlugin;
-import org.elasticsearch.license.plugin.consumer.TestPluginService1;
+import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService;
 import org.elasticsearch.node.internal.InternalNode;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -29,8 +29,8 @@ public class LicensesServiceNodeTests extends AbstractLicensesIntegrationTests {
     protected Settings nodeSettings(int nodeOrdinal) {
         return ImmutableSettings.settingsBuilder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(TestConsumerPlugin1.NAME + ".trial_license_duration_in_seconds", 5)
+                .put(EagerLicenseRegistrationConsumerPlugin.NAME + ".trial_license_duration_in_seconds", 5)
-                .putArray("plugin.types", LicensePlugin.class.getName(), TestConsumerPlugin1.class.getName())
+                .putArray("plugin.types", LicensePlugin.class.getName(), EagerLicenseRegistrationConsumerPlugin.class.getName())
                 .put(InternalNode.HTTP_ENABLED, true)
                 .build();
     }
@@ -38,11 +38,11 @@ public class LicensesServiceNodeTests extends AbstractLicensesIntegrationTests {
     @Test
     @TestLogging("_root:DEBUG")
     public void testPluginStatus() throws Exception {
-        final Iterable<TestPluginService1> testPluginServices = internalCluster().getDataNodeInstances(TestPluginService1.class);
+        final Iterable<EagerLicenseRegistrationPluginService> testPluginServices = internalCluster().getDataNodeInstances(EagerLicenseRegistrationPluginService.class);
         assertThat(awaitBusy(new Predicate<Object>() {
             @Override
             public boolean apply(Object o) {
-                for (TestPluginService1 pluginService : testPluginServices) {
+                for (EagerLicenseRegistrationPluginService pluginService : testPluginServices) {
                     if (!pluginService.enabled()) {
                         return false;
                     }

src/test/java/org/elasticsearch/license/plugin/LicensesServiceTests.java

@@ -13,6 +13,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.base.Predicate;
 import org.elasticsearch.common.collect.ImmutableSet;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.license.TestUtils;
 import org.elasticsearch.license.core.ESLicense;
 import org.elasticsearch.license.manager.ESLicenseManager;
@@ -224,13 +225,13 @@ public class LicensesServiceTests extends AbstractLicensesIntegrationTests {
     }
 
     @Test
-    public void testRandomMultipleClientMultipleFeature() throws Exception {
+    public void testRandomActionSequenceMultipleFeature() throws Exception {
         LicensesService licensesService = randomLicensesService();
         LicensesManagerService masterLicensesManagerService = masterLicensesManagerService();
         Map<TestTrackingClientListener, List<Action>> clientListenersWithActions = new HashMap<>();
 
         TimeValue expiryDuration = TimeValue.timeValueSeconds(0);
-        for (int i = 0; i < randomIntBetween(3, 10); i++) {
+        for (int i = 0; i < randomIntBetween(10, 20); i++) {
             final TestTrackingClientListener clientListener = new TestTrackingClientListener();
             String feature = randomRealisticUnicodeOfCodepointLengthBetween(2, 10);
             expiryDuration = TimeValue.timeValueMillis(randomIntBetween(1, 5) * 100l + expiryDuration.millis());
@@ -313,6 +314,17 @@ public class LicensesServiceTests extends AbstractLicensesIntegrationTests {
 
                 // invoke clusterChanged event to flush out pendingRegistration
                 LicensesService licensesService = (LicensesService) clientService;
+                /*
+                try {
+                    assertThat(awaitBusy(new Predicate<Object>() {
+                                  @Override
+                                  public boolean apply(Object o) {
+                                      return !clusterService().state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK);
+                                  }
+                              }), equalTo(true));
+                } catch (InterruptedException e) {
+                    logger.error("Exception while trying to registerWithTrialLicense", e);
+                }*/
                 ClusterChangedEvent event = new ClusterChangedEvent("", clusterService().state(), clusterService().state());
                 licensesService.clusterChanged(event);
             }

src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationConsumerPlugin.java

@@ -9,18 +9,24 @@ import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 
-public class TestConsumerPlugin1 extends TestConsumerPluginBase {
+/**
+ * Registers licenses upon the start of the service lifecycle
+ * see {@link org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService}
+ *
+ * License registration might happen before clusterService start()
+ */
+public class EagerLicenseRegistrationConsumerPlugin extends TestConsumerPluginBase {
 
     public final static String NAME = "test_consumer_plugin_1";
 
     @Inject
-    public TestConsumerPlugin1(Settings settings) {
+    public EagerLicenseRegistrationConsumerPlugin(Settings settings) {
         super(settings);
     }
 
     @Override
     protected Class<? extends LifecycleComponent> service() {
-        return TestPluginService1.class;
+        return EagerLicenseRegistrationPluginService.class;
     }
 
     @Override

src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationPluginService.java

@@ -11,13 +11,13 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.plugin.core.LicensesClientService;
 
 @Singleton
-public class TestPluginService1 extends TestPluginServiceBase {
+public class EagerLicenseRegistrationPluginService extends TestPluginServiceBase {
 
     public static String FEATURE_NAME = "feature1";
 
     @Inject
-    public TestPluginService1(Settings settings, LicensesClientService licensesClientService) {
+    public EagerLicenseRegistrationPluginService(Settings settings, LicensesClientService licensesClientService) {
-        super(settings, licensesClientService);
+        super(true, settings, licensesClientService, null);
     }
 
     @Override
@@ -27,6 +27,6 @@ public class TestPluginService1 extends TestPluginServiceBase {
 
     @Override
     public String settingPrefix() {
-        return TestConsumerPlugin1.NAME;
+        return EagerLicenseRegistrationConsumerPlugin.NAME;
     }
 }

src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationConsumerPlugin.java

@@ -9,18 +9,24 @@ import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 
-public class TestConsumerPlugin2 extends TestConsumerPluginBase {
+/**
+ * Registers licenses only after cluster has recovered
+ * see {@link org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationPluginService}
+ *
+ * License registration happens after clusterservice start()
+ */
+public class LazyLicenseRegistrationConsumerPlugin extends TestConsumerPluginBase {
 
     public static String NAME = "test_consumer_plugin_2";
 
     @Inject
-    public TestConsumerPlugin2(Settings settings) {
+    public LazyLicenseRegistrationConsumerPlugin(Settings settings) {
         super(settings);
     }
 
     @Override
     protected Class<? extends LifecycleComponent> service() {
-        return TestPluginService2.class;
+        return LazyLicenseRegistrationPluginService.class;
     }
 
     @Override

src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationPluginService.java

@@ -5,20 +5,21 @@
  */
 package org.elasticsearch.license.plugin.consumer;
 
+import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.Singleton;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.plugin.core.LicensesClientService;
 
 @Singleton
-public class TestPluginService2 extends TestPluginServiceBase {
+public class LazyLicenseRegistrationPluginService extends TestPluginServiceBase {
 
 
     public static String FEATURE_NAME = "feature2";
 
     @Inject
-    public TestPluginService2(Settings settings, LicensesClientService licensesClientService) {
+    public LazyLicenseRegistrationPluginService(Settings settings, LicensesClientService licensesClientService, ClusterService clusterService) {
-        super(settings, licensesClientService);
+        super(false, settings, licensesClientService, clusterService);
     }
 
     @Override
@@ -28,6 +29,6 @@ public class TestPluginService2 extends TestPluginServiceBase {
 
     @Override
     public String settingPrefix() {
-        return TestConsumerPlugin2.NAME;
+        return LazyLicenseRegistrationConsumerPlugin.NAME;
     }
 }

src/test/java/org/elasticsearch/license/plugin/consumer/TestPluginServiceBase.java

@@ -6,26 +6,36 @@
 package org.elasticsearch.license.plugin.consumer;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.license.plugin.core.LicensesClientService;
 import org.elasticsearch.license.plugin.core.LicensesService;
 
 import java.util.concurrent.atomic.AtomicBoolean;
 
-public abstract class TestPluginServiceBase extends AbstractLifecycleComponent<TestPluginServiceBase> {
+public abstract class TestPluginServiceBase extends AbstractLifecycleComponent<TestPluginServiceBase> implements ClusterStateListener {
 
     private LicensesClientService licensesClientService;
 
+    private final ClusterService clusterService;
     // specify the trial license spec for the feature
     // example: 30 day trial on 1000 nodes
     final LicensesService.TrialLicenseOptions trialLicenseOptions;
 
+    final boolean eagerLicenseRegistration;
+
+    public final AtomicBoolean registered = new AtomicBoolean(false);
+
     private AtomicBoolean enabled = new AtomicBoolean(false);
 
-    public TestPluginServiceBase(Settings settings, LicensesClientService licensesClientService) {
+    public TestPluginServiceBase(boolean eagerLicenseRegistration, Settings settings, LicensesClientService licensesClientService, ClusterService clusterService) {
         super(settings);
+        this.eagerLicenseRegistration = eagerLicenseRegistration;
         this.licensesClientService = licensesClientService;
         int durationInSec = settings.getAsInt(settingPrefix() + ".trial_license_duration_in_seconds", -1);
         if (durationInSec == -1) {
@@ -33,6 +43,12 @@ public abstract class TestPluginServiceBase extends AbstractLifecycleComponent<T
         } else {
             this.trialLicenseOptions = new LicensesService.TrialLicenseOptions(TimeValue.timeValueSeconds(durationInSec), 1000);
         }
+        if (!eagerLicenseRegistration) {
+            this.clusterService = clusterService;
+            clusterService.add(this);
+        } else {
+            this.clusterService = null;
+        }
     }
 
     // should be the same string used by the license Manger to generate
@@ -46,14 +62,34 @@ public abstract class TestPluginServiceBase extends AbstractLifecycleComponent<T
         return enabled.get();
     }
 
-    protected void doStart() throws ElasticsearchException {
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        if (!eagerLicenseRegistration && !event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            if (registered.compareAndSet(false, true)) {
+                logger.info("Registering to licensesService [lazy]");
                 licensesClientService.register(featureName(),
                         trialLicenseOptions,
                         new LicensingClientListener());
             }
+        }
+    }
+
+    protected void doStart() throws ElasticsearchException {
+        if (eagerLicenseRegistration) {
+            if (registered.compareAndSet(false, true)) {
+                logger.info("Registering to licensesService [eager]");
+                licensesClientService.register(featureName(),
+                        trialLicenseOptions,
+                        new LicensingClientListener());
+            }
+        }
+    }
 
     @Override
     protected void doStop() throws ElasticsearchException {
+        if (clusterService != null) {
+            clusterService.remove(this);
+        }
     }
 
     @Override