[TEST & DOCS] Added PluginIntegrationTestFramework & license plugin test doc

Original commit: elastic/x-pack-elasticsearch@f65e4b3823

src/test/java/org/elasticsearch/license/AbstractLicensingTestBase.java

@@ -15,6 +15,7 @@ import org.elasticsearch.license.core.ESLicense;
 import org.elasticsearch.license.licensor.ESLicenseSigner;
 import org.elasticsearch.license.manager.ESLicenseManager;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.runner.RunWith;
 
 import java.io.IOException;
@@ -34,7 +35,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.IsEqual.equalTo;
 
 @RunWith(value = com.carrotsearch.randomizedtesting.RandomizedRunner.class)
-public class AbstractLicensingTestBase {
+public abstract class AbstractLicensingTestBase {
 
     protected static String pubKeyPath = null;
     protected static String priKeyPath = null;

src/test/java/org/elasticsearch/license/plugin/AbstractLicensesConsumerPluginIntegrationTests.java

@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.license.plugin;
+
+import org.elasticsearch.common.base.Predicate;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.license.plugin.consumer.*;
+import org.elasticsearch.test.InternalTestCluster;
+import org.junit.After;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
+import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope.TEST;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+/**
+ * Framework to test licensing plugin integration for existing/new consumer plugins
+ * see {@link LicensesEagerConsumerPluginIntegrationTests} and {@link LicensesLazyConsumerPluginIntegrationTests}
+ * for example usage
+ */
+@ClusterScope(scope = TEST, numDataNodes = 10, numClientNodes = 0, transportClientRatio = 0.0)
+public abstract class AbstractLicensesConsumerPluginIntegrationTests extends AbstractLicensesIntegrationTests {
+
+    protected final TestConsumerPluginBase consumerPlugin;
+
+    public AbstractLicensesConsumerPluginIntegrationTests(TestConsumerPluginBase consumerPlugin) {
+        this.consumerPlugin = consumerPlugin;
+    }
+
+    private final int trialLicenseDurationInSeconds = 5;
+
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return ImmutableSettings.settingsBuilder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put(consumerPlugin.name()
+                        + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
+                .putArray("plugin.types", LicensePlugin.class.getName(), consumerPlugin.getClass().getName())
+                .build();
+    }
+
+    @After
+    public void afterTest() throws Exception {
+        wipeAllLicenses();
+        assertThat(awaitBusy(new Predicate<Object>() {
+            @Override
+            public boolean apply(Object o) {
+                return !clusterService().state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK);
+            }
+        }), equalTo(true));
+    }
+
+    @Test
+    public void testTrialLicenseAndSignedLicenseNotification() throws Exception {
+        logger.info("using " + consumerPlugin.getClass().getName() + " consumer plugin");
+        logger.info(" --> trial license generated");
+        // managerService should report feature to be enabled on all data nodes
+        assertLicenseManagerEnabledFeatureFor(consumerPlugin.featureName());
+        // consumer plugin service should return enabled on all data nodes
+        assertConsumerPluginEnabledNotification(1);
+
+        logger.info(" --> check trial license expiry notification");
+        // consumer plugin should notify onDisabled on all data nodes (expired trial license)
+        assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds * 2);
+        assertLicenseManagerDisabledFeatureFor(consumerPlugin.featureName());
+
+        logger.info(" --> put signed license");
+        putLicense(consumerPlugin.featureName(), TimeValue.timeValueSeconds(trialLicenseDurationInSeconds));
+
+        logger.info(" --> check signed license enabled notification");
+        // consumer plugin should notify onEnabled on all data nodes (signed license)
+        assertConsumerPluginEnabledNotification(1);
+        assertLicenseManagerEnabledFeatureFor(consumerPlugin.featureName());
+
+        logger.info(" --> check signed license expiry notification");
+        // consumer plugin should notify onDisabled on all data nodes (expired signed license)
+        assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds * 2);
+        assertLicenseManagerDisabledFeatureFor(consumerPlugin.featureName());
+    }
+
+    @Test
+    public void testTrialLicenseNotification() throws Exception {
+        logger.info(" --> check onEnabled for trial license");
+        // managerService should report feature to be enabled on all data nodes
+        assertLicenseManagerEnabledFeatureFor(consumerPlugin.featureName());
+        // consumer plugin service should return enabled on all data nodes
+        assertConsumerPluginEnabledNotification(1);
+
+        logger.info(" --> sleep for rest of trailLicense duration");
+        Thread.sleep(trialLicenseDurationInSeconds * 1000l);
+
+        logger.info(" --> check trial license expiry notification");
+        // consumer plugin should notify onDisabled on all data nodes (expired signed license)
+        assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds);
+        assertLicenseManagerDisabledFeatureFor(consumerPlugin.featureName());
+    }
+
+    @Test
+    public void testOverlappingTrialAndSignedLicenseNotification() throws Exception {
+        logger.info(" --> check onEnabled for trial license");
+        // managerService should report feature to be enabled on all data nodes
+        assertLicenseManagerEnabledFeatureFor(consumerPlugin.featureName());
+        // consumer plugin service should return enabled on all data nodes
+        assertConsumerPluginEnabledNotification(1);
+
+        logger.info(" --> put signed license while trial license is in effect");
+        putLicense(consumerPlugin.featureName(), TimeValue.timeValueSeconds(trialLicenseDurationInSeconds * 2));
+
+        logger.info(" --> check signed license enabled notification");
+        // consumer plugin should notify onEnabled on all data nodes (signed license)
+        assertConsumerPluginEnabledNotification(1);
+        assertLicenseManagerEnabledFeatureFor(consumerPlugin.featureName());
+
+        logger.info(" --> sleep for rest of trailLicense duration");
+        Thread.sleep(trialLicenseDurationInSeconds * 1000l);
+
+        logger.info(" --> check consumer is still enabled [signed license]");
+        // consumer plugin should notify onEnabled on all data nodes (signed license)
+        assertConsumerPluginEnabledNotification(1);
+        assertLicenseManagerEnabledFeatureFor(consumerPlugin.featureName());
+
+        logger.info(" --> check signed license expiry notification");
+        // consumer plugin should notify onDisabled on all data nodes (expired signed license)
+        assertConsumerPluginDisabledNotification(trialLicenseDurationInSeconds * 2 * 2);
+        assertLicenseManagerDisabledFeatureFor(consumerPlugin.featureName());
+    }
+
+    private void assertConsumerPluginEnabledNotification(int timeoutInSec) throws InterruptedException {
+        assertConsumerPluginNotification(consumerPluginServices(), true, timeoutInSec);
+    }
+
+    private void assertConsumerPluginDisabledNotification(int timeoutInSec) throws InterruptedException {
+        assertConsumerPluginNotification(consumerPluginServices(), false, timeoutInSec);
+    }
+
+
+    private List<TestPluginServiceBase> consumerPluginServices() {
+        final InternalTestCluster clients = internalCluster();
+        List<TestPluginServiceBase> consumerPluginServices = new ArrayList<>();
+        for (TestPluginServiceBase service : clients.getDataNodeInstances(consumerPlugin.service())) {
+            consumerPluginServices.add(service);
+        }
+        return consumerPluginServices;
+    }
+
+}

src/test/java/org/elasticsearch/license/plugin/AbstractLicensesIntegrationTests.java

@@ -27,6 +27,7 @@ import org.elasticsearch.license.plugin.core.LicensesMetaData;
 import org.elasticsearch.license.plugin.core.LicensesStatus;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import org.elasticsearch.test.InternalTestCluster;
+import org.junit.Ignore;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -39,8 +40,6 @@ import static org.elasticsearch.license.AbstractLicensingTestBase.getTestPubKeyP
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
-/**
- */
 public abstract class AbstractLicensesIntegrationTests extends ElasticsearchIntegrationTest {
 
     @Override
@@ -146,14 +145,17 @@ public abstract class AbstractLicensesIntegrationTests extends ElasticsearchInte
 
     protected void assertLazyConsumerPluginNotification(final boolean expectedEnabled, int timeoutInSec) throws InterruptedException {
         final List<TestPluginServiceBase> consumerPluginServices = consumerLazyPluginServices();
-        assertThat("At least one instance has to be present", consumerPluginServices.size(), greaterThan(0));
+        assertConsumerPluginNotification(consumerPluginServices, expectedEnabled, timeoutInSec);
-        assertConsumerPluginNotification("LazyConsumer should have license status of: " + expectedEnabled, consumerPluginServices, expectedEnabled, timeoutInSec);
     }
 
     protected void assertEagerConsumerPluginNotification(final boolean expectedEnabled, int timeoutInSec) throws InterruptedException {
         final List<TestPluginServiceBase> consumerPluginServices = consumerEagerPluginServices();
+        assertConsumerPluginNotification(consumerPluginServices, expectedEnabled, timeoutInSec);
+    }
+
+    protected void assertConsumerPluginNotification(final List<TestPluginServiceBase> consumerPluginServices, final boolean expectedEnabled, int timeoutInSec) throws InterruptedException {
         assertThat("At least one instance has to be present", consumerPluginServices.size(), greaterThan(0));
-        assertConsumerPluginNotification("EagerConsumer should have license status of: " + expectedEnabled, consumerPluginServices, expectedEnabled, timeoutInSec);
+        assertConsumerPluginNotification(consumerPluginServices.get(0).getClass().getName() + " should have license status of: " + expectedEnabled, consumerPluginServices, expectedEnabled, timeoutInSec);
     }
 
     private void assertConsumerPluginNotification(String msg, final Iterable<TestPluginServiceBase> consumerPluginServices, final boolean expectedEnabled, int timeoutInSec) throws InterruptedException {

src/test/java/org/elasticsearch/license/plugin/AbstractLicensesServiceTests.java

@@ -19,6 +19,7 @@ import org.elasticsearch.license.plugin.core.LicensesService;
 import org.elasticsearch.license.plugin.core.LicensesStatus;
 import org.elasticsearch.test.InternalTestCluster;
 import org.junit.Before;
+import org.junit.Ignore;
 
 import java.util.HashSet;
 import java.util.List;
@@ -29,7 +30,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import static org.elasticsearch.license.plugin.core.LicensesService.LicensesUpdateResponse;
 import static org.hamcrest.Matchers.equalTo;
 
-public class AbstractLicensesServiceTests extends AbstractLicensesIntegrationTests {
+public abstract class AbstractLicensesServiceTests extends AbstractLicensesIntegrationTests {
 
     private static String node = null;
     private static String[] nodes;

src/test/java/org/elasticsearch/license/plugin/LicensesEagerConsumerPluginIntegrationTests.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.license.plugin;
+
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.license.plugin.consumer.*;
+
+public class LicensesEagerConsumerPluginIntegrationTests extends AbstractLicensesConsumerPluginIntegrationTests {
+
+    public LicensesEagerConsumerPluginIntegrationTests() {
+        super(new EagerLicenseRegistrationConsumerPlugin(ImmutableSettings.EMPTY));
+    }
+}

src/test/java/org/elasticsearch/license/plugin/LicensesLazyConsumerPluginIntegrationTests.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.license.plugin;
+
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationConsumerPlugin;
+
+public class LicensesLazyConsumerPluginIntegrationTests extends AbstractLicensesConsumerPluginIntegrationTests {
+
+    public LicensesLazyConsumerPluginIntegrationTests() {
+        super(new LazyLicenseRegistrationConsumerPlugin(ImmutableSettings.EMPTY));
+    }
+}

src/test/java/org/elasticsearch/license/plugin/LicensesServiceClusterTest.java

@@ -17,11 +17,14 @@ import org.elasticsearch.license.plugin.action.put.PutLicenseRequestBuilder;
 import org.elasticsearch.license.plugin.action.put.PutLicenseResponse;
 import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationConsumerPlugin;
 import org.elasticsearch.license.plugin.consumer.EagerLicenseRegistrationPluginService;
+import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationConsumerPlugin;
+import org.elasticsearch.license.plugin.consumer.LazyLicenseRegistrationPluginService;
+import org.elasticsearch.license.plugin.core.LicensesMetaData;
 import org.elasticsearch.license.plugin.core.LicensesStatus;
 import org.elasticsearch.node.internal.InternalNode;
 import org.junit.Test;
 
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.List;
 
 import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
@@ -32,7 +35,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 @ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0, transportClientRatio = 0)
 public class LicensesServiceClusterTest extends AbstractLicensesIntegrationTests {
 
-    private final String FEATURE_NAME = EagerLicenseRegistrationPluginService.FEATURE_NAME;
+    private final String[] FEATURES = {EagerLicenseRegistrationPluginService.FEATURE_NAME, LazyLicenseRegistrationPluginService.FEATURE_NAME};
 
     private final int trialLicenseDurationInSeconds = 2;
 
@@ -53,7 +56,8 @@ public class LicensesServiceClusterTest extends AbstractLicensesIntegrationTests
                 .put("node.data", true)
                 .put("format", "json")
                 .put(EagerLicenseRegistrationConsumerPlugin.NAME + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
-                .putArray("plugin.types", LicensePlugin.class.getName(), EagerLicenseRegistrationConsumerPlugin.class.getName())
+                .put(LazyLicenseRegistrationConsumerPlugin.NAME + ".trial_license_duration_in_seconds", trialLicenseDurationInSeconds)
+                .putArray("plugin.types", LicensePlugin.class.getName(), EagerLicenseRegistrationConsumerPlugin.class.getName(), LazyLicenseRegistrationConsumerPlugin.class.getName())
                 .put(InternalNode.HTTP_ENABLED, true);
     }
 
@@ -69,14 +73,14 @@ public class LicensesServiceClusterTest extends AbstractLicensesIntegrationTests
         ensureGreen();
 
         logger.info("--> put signed license");
-        final ESLicense esLicense = generateAndPutLicense();
+        final List<ESLicense> licenses = generateAndPutLicenses();
-        getAndCheckLicense(esLicense);
+        getAndCheckLicense(licenses);
         logger.info("--> restart all nodes");
         internalCluster().fullRestart();
         ensureYellow();
 
         logger.info("--> get and check signed license");
-        getAndCheckLicense(esLicense);
+        getAndCheckLicense(licenses);
 
         wipeAllLicenses();
     }
@@ -85,35 +89,81 @@ public class LicensesServiceClusterTest extends AbstractLicensesIntegrationTests
     public void testClusterNotRecovered() throws Exception {
         logger.info("--> start one master out of two [recovery state]");
         internalCluster().startNode(nodeSettingsBuilder(0).put("discovery.zen.minimum_master_nodes", 2).put("node.master", true));
-        assertLicenseManagerDisabledFeatureFor(FEATURE_NAME);
+        // license plugin should not be active when cluster is still recovering
-        assertEagerConsumerPluginDisableNotification(1);
+        assertLicenseManagerFeatureDisabled();
+        assertConsumerPluginDisabledNotification(1);
 
         logger.info("--> start second master out of two [recovered state]");
         internalCluster().startNode(nodeSettingsBuilder(1).put("discovery.zen.minimum_master_nodes", 2).put("node.master", true));
-        assertLicenseManagerEnabledFeatureFor(FEATURE_NAME);
+        assertLicenseManagerFeatureEnabled();
-        assertEagerConsumerPluginEnableNotification(1);
+        assertConsumerPluginEnabledNotification(1);
     }
 
-    private ESLicense generateAndPutLicense() throws Exception {
+    @Test
+    public void testAtMostOnceTrialLicenseGeneration() throws Exception {
+        wipeAllLicenses();
+        logger.info("--> start one node [trial license should be generated & enabled]");
+        internalCluster().startNode(nodeSettingsBuilder(0));
+        assertLicenseManagerFeatureEnabled();
+        assertConsumerPluginEnabledNotification(1);
+
+        logger.info("--> start another node [trial license should be propagated from the old master not generated]");
+        internalCluster().startNode(nodeSettings(1));
+        assertLicenseManagerFeatureEnabled();
+        assertConsumerPluginEnabledNotification(1);
+
+        logger.info("--> check if multiple trial licenses are found for a feature");
+        LicensesMetaData licensesMetaData = clusterService().state().metaData().custom(LicensesMetaData.TYPE);
+        assertThat(licensesMetaData.getEncodedTrialLicenses().size(), equalTo(FEATURES.length));
+
+        wipeAllLicenses();
+    }
+
+    private List<ESLicense> generateAndPutLicenses() throws Exception {
         ClusterAdminClient cluster = internalCluster().client().admin().cluster();
-        ESLicense license = generateSignedLicense(FEATURE_NAME, TimeValue.timeValueMinutes(1));
+        List<ESLicense> putLicenses = new ArrayList<>(FEATURES.length);
+        for (String feature : FEATURES) {
+             putLicenses.add(generateSignedLicense(feature, TimeValue.timeValueMinutes(1)));
+        }
         PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(cluster);
-        final List<ESLicense> putLicenses = Arrays.asList(license);
         putLicenseRequestBuilder.setLicense(putLicenses);
         ensureGreen();
 
-        final PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.execute().get();
+        final PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get();
 
         assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
 
-        return license;
+        return putLicenses;
     }
 
-    private void getAndCheckLicense(ESLicense license) {
+    private void getAndCheckLicense(List<ESLicense> licenses) {
         ClusterAdminClient cluster = internalCluster().client().admin().cluster();
         final GetLicenseResponse response = new GetLicenseRequestBuilder(cluster).get();
-        assertThat(response.licenses().size(), equalTo(1));
+        assertThat(response.licenses().size(), equalTo(licenses.size()));
-        TestUtils.isSame(license, response.licenses().iterator().next());
+        TestUtils.isSame(licenses, response.licenses());
     }
+
+    private void assertLicenseManagerFeatureEnabled() throws Exception {
+       for (String feature : FEATURES) {
+           assertLicenseManagerEnabledFeatureFor(feature);
+       }
+    }
+
+    private void assertLicenseManagerFeatureDisabled() throws Exception {
+        for (String feature : FEATURES) {
+            assertLicenseManagerDisabledFeatureFor(feature);
+        }
+    }
+
+    private void assertConsumerPluginEnabledNotification(int timeoutInSec) throws InterruptedException {
+        assertEagerConsumerPluginEnableNotification(timeoutInSec);
+        assertLazyConsumerPluginEnableNotification(timeoutInSec);
+    }
+
+    private void assertConsumerPluginDisabledNotification(int timeoutInSec) throws InterruptedException {
+        assertEagerConsumerPluginDisableNotification(timeoutInSec);
+        assertLazyConsumerPluginDisableNotification(timeoutInSec);
+    }
+
 }

src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationConsumerPlugin.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.license.plugin.consumer;
 
-import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 
@@ -25,7 +24,7 @@ public class EagerLicenseRegistrationConsumerPlugin extends TestConsumerPluginBa
     }
 
     @Override
-    protected Class<? extends LifecycleComponent> service() {
+    public Class<? extends TestPluginServiceBase> service() {
         return EagerLicenseRegistrationPluginService.class;
     }
 
@@ -33,4 +32,9 @@ public class EagerLicenseRegistrationConsumerPlugin extends TestConsumerPluginBa
     protected String pluginName() {
         return NAME;
     }
+
+    @Override
+    public String featureName() {
+        return EagerLicenseRegistrationPluginService.FEATURE_NAME;
+    }
 }

src/test/java/org/elasticsearch/license/plugin/consumer/LazyLicenseRegistrationConsumerPlugin.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.license.plugin.consumer;
 
-import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 
@@ -25,7 +24,7 @@ public class LazyLicenseRegistrationConsumerPlugin extends TestConsumerPluginBas
     }
 
     @Override
-    protected Class<? extends LifecycleComponent> service() {
+    public Class<? extends TestPluginServiceBase> service() {
         return LazyLicenseRegistrationPluginService.class;
     }
 
@@ -33,4 +32,9 @@ public class LazyLicenseRegistrationConsumerPlugin extends TestConsumerPluginBas
     protected String pluginName() {
         return NAME;
     }
+
+    @Override
+    public String featureName() {
+        return LazyLicenseRegistrationPluginService.FEATURE_NAME;
+    }
 }

src/test/java/org/elasticsearch/license/plugin/consumer/TestConsumerPluginBase.java

@@ -47,7 +47,9 @@ public abstract class TestConsumerPluginBase extends AbstractPlugin {
         return services;
     }
 
-    protected abstract Class<? extends LifecycleComponent> service();
+    public abstract Class<? extends TestPluginServiceBase> service();
 
     protected abstract String pluginName();
+
+    public abstract String featureName();
 }