honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Marvel: Stop collecting data X days after license expiration 

Closes elastic/elasticsearch#370, elastic/elasticsearch#470

Original commit: elastic/x-pack-elasticsearch@7ed95605ff
This commit is contained in:
Tanguy Leroux 2015-08-24 19:57:20 +02:00
parent ef7d4e2579
commit 4d65f396c8
17 changed files with 531 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java
View File

@ -13,6 +13,7 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.LicenseService;
import java.util.Collection;
@ -22,13 +23,15 @@ public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T>
protected final ClusterService clusterService;
protected final MarvelSettings marvelSettings;
protected final LicenseService licenseService;
@Inject
public AbstractCollector(Settings settings, String name, ClusterService clusterService, MarvelSettings marvelSettings) {
public AbstractCollector(Settings settings, String name, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService) {
super(settings);
this.name = name;
this.clusterService = clusterService;
this.marvelSettings = marvelSettings;
this.licenseService = licenseService;
}
@Override
@ -47,26 +50,23 @@ public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T>
}
/**
* Indicates if the current collector should
* be executed on master node only.
* Indicates if the current collector is allowed to collect data
*/
protected boolean masterOnly() {
return false;
protected boolean canCollect() {
return licenseService.enabled() || licenseService.inExpirationGracePeriod();
}
protected boolean localNodeMaster() {
protected boolean isLocalNodeMaster() {
return clusterService.state().nodes().localNodeMaster();
}
@Override
public Collection<MarvelDoc> collect() {
if (masterOnly() && !localNodeMaster()) {
logger.trace("collector [{}] runs on master only", name());
return null;
}
try {
return doCollect();
if (canCollect()) {
return doCollect();
}
logger.trace("collector [{}] can not collect data", name());
} catch (ElasticsearchTimeoutException e) {
logger.error("collector [{}] timed out when collecting data");
} catch (Exception e) {

10
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java
View File

@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.AbstractCollector;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.LicenseService;
import java.util.Collection;
@ -32,14 +33,15 @@ public class ClusterStateCollector extends AbstractCollector<ClusterStateCollect
private final Client client;
@Inject
public ClusterStateCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, Client client) {
super(settings, NAME, clusterService, marvelSettings);
public ClusterStateCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService,
Client client) {
super(settings, NAME, clusterService, marvelSettings, licenseService);
this.client = client;
}
@Override
protected boolean masterOnly() {
return true;
protected boolean canCollect() {
return super.canCollect() && isLocalNodeMaster();
}
@Override

10
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollector.java
View File

@ -14,6 +14,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.AbstractCollector;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.LicenseService;
import java.util.Collection;
@ -31,14 +32,15 @@ public class ClusterStatsCollector extends AbstractCollector<ClusterStatsCollect
private final Client client;
@Inject
public ClusterStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, Client client) {
super(settings, NAME, clusterService, marvelSettings);
public ClusterStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService,
Client client) {
super(settings, NAME, clusterService, marvelSettings, licenseService);
this.client = client;
}
@Override
protected boolean masterOnly() {
return true;
protected boolean canCollect() {
return super.canCollect() && isLocalNodeMaster();
}
@Override

10
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollector.java
View File

@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.AbstractCollector;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.LicenseService;
import java.util.Collection;
@ -32,14 +33,15 @@ public class IndexRecoveryCollector extends AbstractCollector<IndexRecoveryColle
private final Client client;
@Inject
public IndexRecoveryCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, Client client) {
super(settings, NAME, clusterService, marvelSettings);
public IndexRecoveryCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService,
Client client) {
super(settings, NAME, clusterService, marvelSettings, licenseService);
this.client = client;
}
@Override
protected boolean masterOnly() {
return true;
protected boolean canCollect() {
return super.canCollect() && isLocalNodeMaster();
}
@Override

10
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java
View File

@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.AbstractCollector;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.LicenseService;
import java.util.Collection;
@ -33,14 +34,15 @@ public class IndexStatsCollector extends AbstractCollector<IndexStatsCollector>
private final Client client;
@Inject
public IndexStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, Client client) {
super(settings, NAME, clusterService, marvelSettings);
public IndexStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService,
Client client) {
super(settings, NAME, clusterService, marvelSettings, licenseService);
this.client = client;
}
@Override
protected boolean masterOnly() {
return true;
protected boolean canCollect() {
return super.canCollect() && isLocalNodeMaster();
}
@Override

12
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/licenses/LicensesCollector.java
View File

@ -36,13 +36,19 @@ public class LicensesCollector extends AbstractCollector<LicensesMarvelDoc> {
private final LicenseService licenseService;
@Inject
public LicensesCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,
ClusterName clusterName, LicenseService licenseService) {
super(settings, NAME, clusterService, marvelSettings);
public LicensesCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService,
ClusterName clusterName) {
super(settings, NAME, clusterService, marvelSettings, licenseService);
this.clusterName = clusterName;
this.licenseService = licenseService;
}
@Override
protected boolean canCollect() {
// This collector can always collect data on the master node
return isLocalNodeMaster();
}
@Override
protected Collection<MarvelDoc> doCollect() throws Exception {
ImmutableList.Builder<MarvelDoc> results = ImmutableList.builder();

7
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsCollector.java
View File

@ -20,6 +20,7 @@ import org.elasticsearch.discovery.DiscoveryService;
import org.elasticsearch.marvel.agent.collector.AbstractCollector;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.LicenseService;
import org.elasticsearch.node.service.NodeService;
import java.util.Collection;
@ -43,10 +44,10 @@ public class NodeStatsCollector extends AbstractCollector<NodeStatsCollector> {
private final Provider<DiskThresholdDecider> diskThresholdDeciderProvider;
@Inject
public NodeStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,
public NodeStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService,
NodeService nodeService, DiscoveryService discoveryService,
Provider<DiskThresholdDecider> diskThresholdDeciderProvider) {
super(settings, NAME, clusterService, marvelSettings);
super(settings, NAME, clusterService, marvelSettings, licenseService);
this.nodeService = nodeService;
this.discoveryService = discoveryService;
this.diskThresholdDeciderProvider = diskThresholdDeciderProvider;
@ -71,7 +72,7 @@ public class NodeStatsCollector extends AbstractCollector<NodeStatsCollector> {
boolean diskThresholdDeciderEnabled = (diskThresholdDecider != null) && diskThresholdDecider.isEnabled();
results.add(new NodeStatsMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(),
discoveryService.localNode().id(), localNodeMaster(), nodeStats,
discoveryService.localNode().id(), isLocalNodeMaster(), nodeStats,
BootstrapInfo.isMemoryLocked(), diskThresholdWatermarkHigh, diskThresholdDeciderEnabled));
return results.build();

12
marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java
View File

@ -22,6 +22,7 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
private static final String PREFIX = MarvelPlugin.NAME + ".agent.";
public static final String MARVEL_DATA_INDEX_NAME = ".marvel-data";
public static final TimeValue MAX_LICENSE_GRACE_PERIOD = TimeValue.timeValueHours(7 * 24);
public static final String INTERVAL = PREFIX + "interval";
public static final String STARTUP_DELAY = PREFIX + "startup.delay";
@ -32,6 +33,7 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
public static final String INDEX_RECOVERY_TIMEOUT = PREFIX + "index.recovery.timeout";
public static final String INDEX_RECOVERY_ACTIVE_ONLY = PREFIX + "index.recovery.active_only";
public static final String COLLECTORS = PREFIX + "collectors";
public static final String LICENSE_GRACE_PERIOD = PREFIX + "license.grace.period";
private static Map<String, ? extends MarvelSetting> MARVEL_SETTINGS = Collections.EMPTY_MAP;
@ -55,6 +57,8 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
"INDEX_RECOVERY_ACTIVE_ONLY to indicate if only active recoveries should be collected (default to false: all recoveries are collected)", true));
map.put(COLLECTORS, arraySetting(COLLECTORS, Strings.EMPTY_ARRAY,
"List of collectors allowed to collect data (default to all)", false));
map.put(LICENSE_GRACE_PERIOD, timeSetting(LICENSE_GRACE_PERIOD, MAX_LICENSE_GRACE_PERIOD,
"Period during which the agent continues to collect data even if the license is expired (default to 7 days, cannot be greater than 7 days)", false));
MARVEL_SETTINGS = Collections.unmodifiableMap(map);
}
@ -164,4 +168,12 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
public String[] collectors() {
return getSettingValue(COLLECTORS);
}
public TimeValue licenseExpirationGracePeriod() {
TimeValue delay = getSettingValue(LICENSE_GRACE_PERIOD);
if ((delay.millis() >= 0) && (delay.millis() < MAX_LICENSE_GRACE_PERIOD.millis())) {
return delay;
}
return MAX_LICENSE_GRACE_PERIOD;
}
}

33
marvel/src/main/java/org/elasticsearch/marvel/license/LicenseService.java
View File

@ -17,6 +17,7 @@ import org.elasticsearch.license.plugin.core.LicensesClientService;
import org.elasticsearch.license.plugin.core.LicensesManagerService;
import org.elasticsearch.license.plugin.core.LicensesService;
import org.elasticsearch.marvel.MarvelPlugin;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.mode.Mode;
import java.util.*;
@ -32,16 +33,20 @@ public class LicenseService extends AbstractLifecycleComponent<LicenseService> {
private final LicensesManagerService managerService;
private final LicensesClientService clientService;
private final MarvelSettings marvelSettings;
private final Collection<LicensesService.ExpirationCallback> expirationLoggers;
private final LicensesClientService.AcknowledgementCallback acknowledgementCallback;
private volatile Mode mode;
private volatile boolean enabled;
private volatile long expiryDate;
@Inject
public LicenseService(Settings settings, LicensesClientService clientService, LicensesManagerService managerService) {
public LicenseService(Settings settings, LicensesClientService clientService, LicensesManagerService managerService, MarvelSettings marvelSettings) {
super(settings);
this.managerService = managerService;
this.clientService = clientService;
this.marvelSettings = marvelSettings;
this.mode = Mode.LITE;
this.expirationLoggers = Arrays.asList(
new LicensesService.ExpirationCallback.Pre(days(7), days(30), days(1)) {
@ -121,6 +126,28 @@ public class LicenseService extends AbstractLifecycleComponent<LicenseService> {
return managerService.getLicenses();
}
/**
* @return true if the marvel license is enabled
*/
public boolean enabled() {
return enabled;
}
/**
* @return true if marvel is running within the "grace period", ie when the license
* is expired but a given extra delay is not yet elapsed
*/
public boolean inExpirationGracePeriod() {
return System.currentTimeMillis() <= (expiryDate() + marvelSettings.licenseExpirationGracePeriod().millis());
}
/**
* @return the license's expiration date (as a long)
*/
public long expiryDate() {
return expiryDate;
}
class InternalListener implements LicensesClientService.Listener {
private final LicenseService service;
@ -132,6 +159,8 @@ public class LicenseService extends AbstractLifecycleComponent<LicenseService> {
@Override
public void onEnabled(License license) {
try {
service.enabled = true;
service.expiryDate = license.expiryDate();
service.mode = Mode.fromName(license.type());
} catch (IllegalArgumentException e) {
service.mode = Mode.LITE;
@ -140,6 +169,8 @@ public class LicenseService extends AbstractLifecycleComponent<LicenseService> {
@Override
public void onDisabled(License license) {
service.enabled = false;
service.expiryDate = license.expiryDate();
service.mode = Mode.LITE;
}
}

204
marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java Normal file
View File

@ -0,0 +1,204 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.collector;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import com.carrotsearch.randomizedtesting.SysGlobals;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.license.core.License;
import org.elasticsearch.license.plugin.core.LicensesClientService;
import org.elasticsearch.marvel.MarvelPlugin;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.LicenseIntegrationTests;
import org.elasticsearch.marvel.license.LicenseService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.junit.Before;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
@ClusterScope(scope = ESIntegTestCase.Scope.SUITE, randomDynamicTemplates = false, transportClientRatio = 0.0)
public class AbstractCollectorTestCase extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.settingsBuilder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugin.types", MarvelPlugin.class.getName() + "," + LicensePluginForCollectors.class.getName())
.build();
}
@Before
public void ensureLicenseIsEnabled() {
enableLicense();
}
protected void assertCanCollect(AbstractCollector collector) {
assertNotNull(collector);
assertTrue("collector [" + collector.name() + "] should be able to collect data", collector.canCollect());
Collection results = collector.collect();
assertTrue(results != null && !results.isEmpty());
}
protected void assertCannotCollect(AbstractCollector collector) {
assertNotNull(collector);
assertFalse("collector [" + collector.name() + "] should not be able to collect data", collector.canCollect());
Collection results = collector.collect();
assertTrue(results == null || results.isEmpty());
}
private static License createTestingLicense(long issueDate, long expiryDate) {
return License.builder()
.feature(LicenseService.FEATURE_NAME)
.expiryDate(expiryDate)
.issueDate(issueDate)
.issuedTo("AbstractCollectorTestCase")
.issuer("test")
.maxNodes(Integer.MAX_VALUE)
.signature("_signature")
.type("standard")
.subscriptionType("all_is_good")
.uid(String.valueOf(RandomizedTest.systemPropertyAsInt(SysGlobals.CHILDVM_SYSPROP_JVM_ID, 0)) + System.identityHashCode(LicenseIntegrationTests.class))
.build();
}
protected static void enableLicense() {
long issueDate = System.currentTimeMillis();
long expiryDate = issueDate + randomDaysInMillis();
final License license = createTestingLicense(issueDate, expiryDate);
for (LicenseServiceForCollectors service : internalCluster().getInstances(LicenseServiceForCollectors.class)) {
service.enable(license);
}
}
protected static void beginGracefulPeriod() {
long expiryDate = System.currentTimeMillis() + TimeValue.timeValueMinutes(10).millis();
long issueDate = expiryDate - randomDaysInMillis();
final License license = createTestingLicense(issueDate, expiryDate);
for (LicenseServiceForCollectors service : internalCluster().getInstances(LicenseServiceForCollectors.class)) {
service.disable(license);
}
}
protected static void endGracefulPeriod() {
long expiryDate = System.currentTimeMillis() - MarvelSettings.MAX_LICENSE_GRACE_PERIOD.millis() - TimeValue.timeValueMinutes(10).millis();
long issueDate = expiryDate - randomDaysInMillis();
final License license = createTestingLicense(issueDate, expiryDate);
for (LicenseServiceForCollectors service : internalCluster().getInstances(LicenseServiceForCollectors.class)) {
service.disable(license);
}
}
protected static void disableLicense() {
long expiryDate = System.currentTimeMillis() - MarvelSettings.MAX_LICENSE_GRACE_PERIOD.millis() - randomDaysInMillis();
long issueDate = expiryDate - randomDaysInMillis();
final License license = createTestingLicense(issueDate, expiryDate);
for (LicenseServiceForCollectors service : internalCluster().getInstances(LicenseServiceForCollectors.class)) {
service.disable(license);
}
}
private static long randomDaysInMillis() {
return TimeValue.timeValueHours(randomIntBetween(1, 30) * 24).millis();
}
public void waitForNoBlocksOnNodes() throws Exception {
assertBusy(new Runnable() {
@Override
public void run() {
for (String nodeId : internalCluster().getNodeNames()) {
try {
assertTrue(waitForNoBlocksOnNode(nodeId));
} catch (Exception e) {
fail("failed to wait for no blocks on node [" + nodeId + "]: " + e.getMessage());
}
}
}
});
}
public boolean waitForNoBlocksOnNode(final String nodeId) throws Exception {
return assertBusy(new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
ClusterBlocks clusterBlocks = client(nodeId).admin().cluster().prepareState().setLocal(true).execute().actionGet().getState().blocks();
assertTrue(clusterBlocks.global().isEmpty());
assertTrue(clusterBlocks.indices().values().isEmpty());
return true;
}
}, 30L, TimeUnit.SECONDS);
}
public static class LicensePluginForCollectors extends Plugin {
public static final String NAME = "internal-test-licensing";
@Override
public String name() {
return NAME;
}
@Override
public String description() {
return name();
}
@Override
public Collection<Module> nodeModules() {
return Collections.<Module>singletonList(new AbstractModule(){
@Override
protected void configure() {
bind(LicenseServiceForCollectors.class).asEagerSingleton();
bind(LicensesClientService.class).to(LicenseServiceForCollectors.class);
}
});
}
}
public static class LicenseServiceForCollectors extends AbstractComponent implements LicensesClientService {
private final List<Listener> listeners = new ArrayList<>();
@Inject
public LicenseServiceForCollectors(Settings settings) {
super(settings);
}
@Override
public void register(String feature, TrialLicenseOptions trialLicenseOptions, Collection<ExpirationCallback> expirationCallbacks, AcknowledgementCallback acknowledgementCallback, Listener listener) {
listeners.add(listener);
}
public void enable(License license) {
for (Listener listener : listeners) {
listener.onEnabled(license);
}
}
public void disable(License license) {
for (Listener listener : listeners) {
listener.onDisabled(license);
}
}
}
}

67
marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java
View File

@ -8,18 +8,21 @@ package org.elasticsearch.marvel.agent.collector.cluster;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.marvel.license.LicenseService;
import org.junit.Test;
import java.util.Collection;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.*;
public class ClusterStateCollectorTests extends ESSingleNodeTestCase {
public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
@Test
public void testClusterStateCollectorNoIndices() throws Exception {
@ -43,7 +46,10 @@ public class ClusterStateCollectorTests extends ESSingleNodeTestCase {
@Test
public void testClusterStateCollectorOneIndex() throws Exception {
int nbShards = randomIntBetween(1, 5);
createIndex("test", Settings.settingsBuilder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, nbShards).build());
assertAcked(prepareCreate("test").setSettings(Settings.settingsBuilder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, nbShards)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.build()));
int nbDocs = randomIntBetween(1, 20);
for (int i = 0; i < nbDocs; i++) {
@ -78,7 +84,10 @@ public class ClusterStateCollectorTests extends ESSingleNodeTestCase {
for (int i = 0; i < nbIndices; i++) {
shardsPerIndex[i] = randomIntBetween(1, 5);
createIndex("test-" + i, Settings.settingsBuilder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shardsPerIndex[i]).build());
assertAcked(prepareCreate("test-" + i).setSettings(Settings.settingsBuilder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shardsPerIndex[i])
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.build()));
docsPerIndex[i] = randomIntBetween(1, 20);
for (int j = 0; j < docsPerIndex[i]; j++) {
@ -108,10 +117,52 @@ public class ClusterStateCollectorTests extends ESSingleNodeTestCase {
}
}
@Test
public void tesClusterStateCollectorWithLicensing() {
String[] nodes = internalCluster().getNodeNames();
for (String node : nodes) {
logger.debug("--> creating a new instance of the collector");
ClusterStateCollector collector = newClusterStateCollector(node);
assertNotNull(collector);
logger.debug("--> enabling license and checks that the collector can collect data if node is master");
enableLicense();
if (node.equals(internalCluster().getMasterName())) {
assertCanCollect(collector);
} else {
assertCannotCollect(collector);
}
logger.debug("--> starting graceful period and checks that the collector can still collect data if node is master");
beginGracefulPeriod();
if (node.equals(internalCluster().getMasterName())) {
assertCanCollect(collector);
} else {
assertCannotCollect(collector);
}
logger.debug("--> ending graceful period and checks that the collector cannot collect data");
endGracefulPeriod();
assertCannotCollect(collector);
logger.debug("--> disabling license and checks that the collector cannot collect data");
disableLicense();
assertCannotCollect(collector);
}
}
private ClusterStateCollector newClusterStateCollector() {
return new ClusterStateCollector(getInstanceFromNode(Settings.class),
getInstanceFromNode(ClusterService.class),
getInstanceFromNode(MarvelSettings.class),
client());
return newClusterStateCollector(null);
}
private ClusterStateCollector newClusterStateCollector(String nodeId) {
if (!Strings.hasText(nodeId)) {
nodeId = randomFrom(internalCluster().getNodeNames());
}
return new ClusterStateCollector(internalCluster().getInstance(Settings.class, nodeId),
internalCluster().getInstance(ClusterService.class, nodeId),
internalCluster().getInstance(MarvelSettings.class, nodeId),
internalCluster().getInstance(LicenseService.class, nodeId),
client(nodeId));
}
}

56
marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollectorTests.java
View File

@ -6,17 +6,19 @@
package org.elasticsearch.marvel.agent.collector.cluster;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.marvel.license.LicenseService;
import org.junit.Test;
import java.util.Collection;
import static org.hamcrest.Matchers.*;
public class ClusterStatsCollectorTests extends ESIntegTestCase {
public class ClusterStatsCollectorTests extends AbstractCollectorTestCase {
@Test
public void testClusterStatsCollector() throws Exception {
@ -36,10 +38,52 @@ public class ClusterStatsCollectorTests extends ESIntegTestCase {
assertThat(clusterStatsMarvelDoc.getClusterStats().getNodesStats().getCounts().getTotal(), equalTo(internalCluster().getNodeNames().length));
}
@Test
public void tesClusterStatsCollectorWithLicensing() {
String[] nodes = internalCluster().getNodeNames();
for (String node : nodes) {
logger.debug("--> creating a new instance of the collector");
ClusterStatsCollector collector = newClusterStatsCollector(node);
assertNotNull(collector);
logger.debug("--> enabling license and checks that the collector can collect data if node is master");
enableLicense();
if (node.equals(internalCluster().getMasterName())) {
assertCanCollect(collector);
} else {
assertCannotCollect(collector);
}
logger.debug("--> starting graceful period and checks that the collector can still collect data if node is master");
beginGracefulPeriod();
if (node.equals(internalCluster().getMasterName())) {
assertCanCollect(collector);
} else {
assertCannotCollect(collector);
}
logger.debug("--> ending graceful period and checks that the collector cannot collect data");
endGracefulPeriod();
assertCannotCollect(collector);
logger.debug("--> disabling license and checks that the collector cannot collect data");
disableLicense();
assertCannotCollect(collector);
}
}
private ClusterStatsCollector newClusterStatsCollector() {
return new ClusterStatsCollector(internalCluster().getInstance(Settings.class),
internalCluster().getInstance(ClusterService.class),
internalCluster().getInstance(MarvelSettings.class),
client());
return newClusterStatsCollector(null);
}
private ClusterStatsCollector newClusterStatsCollector(String nodeId) {
if (!Strings.hasText(nodeId)) {
nodeId = randomFrom(internalCluster().getNodeNames());
}
return new ClusterStatsCollector(internalCluster().getInstance(Settings.class, nodeId),
internalCluster().getInstance(ClusterService.class, nodeId),
internalCluster().getInstance(MarvelSettings.class, nodeId),
internalCluster().getInstance(LicenseService.class, nodeId),
client(nodeId));
}
}

68
marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollectorTests.java
View File

@ -8,19 +8,19 @@ package org.elasticsearch.marvel.agent.collector.indices;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.recovery.ShardRecoveryResponse;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.LicenseService;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
@ -30,7 +30,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC
import static org.hamcrest.Matchers.*;
@ESIntegTestCase.ClusterScope(numDataNodes = 0)
public class IndexRecoveryCollectorTests extends ESIntegTestCase {
public class IndexRecoveryCollectorTests extends AbstractCollectorTestCase {
private final boolean activeOnly = false;
private final String indexName = "test";
@ -52,7 +52,7 @@ public class IndexRecoveryCollectorTests extends ESIntegTestCase {
waitForNoBlocksOnNode(node1);
logger.info("--> collect index recovery data");
Collection<MarvelDoc> results = newIndexRecoveryCollector().doCollect();
Collection<MarvelDoc> results = newIndexRecoveryCollector(node1).doCollect();
logger.info("--> no indices created, expecting 0 marvel documents");
assertNotNull(results);
@ -84,7 +84,7 @@ public class IndexRecoveryCollectorTests extends ESIntegTestCase {
}
logger.info("--> collect index recovery data");
results = newIndexRecoveryCollector().doCollect();
results = newIndexRecoveryCollector(null).doCollect();
logger.info("--> we should have at least 1 shard in relocation state");
assertNotNull(results);
@ -117,22 +117,48 @@ public class IndexRecoveryCollectorTests extends ESIntegTestCase {
}
}
private IndexRecoveryCollector newIndexRecoveryCollector() {
return new IndexRecoveryCollector(internalCluster().getInstance(Settings.class),
internalCluster().getInstance(ClusterService.class),
internalCluster().getInstance(MarvelSettings.class),
client());
@Test
public void tesIndexRecoveryCollectorWithLicensing() {
String[] nodes = internalCluster().getNodeNames();
for (String node : nodes) {
logger.debug("--> creating a new instance of the collector");
IndexRecoveryCollector collector = newIndexRecoveryCollector(node);
assertNotNull(collector);
logger.debug("--> enabling license and checks that the collector can collect data if node is master");
enableLicense();
if (node.equals(internalCluster().getMasterName())) {
assertCanCollect(collector);
} else {
assertCannotCollect(collector);
}
logger.debug("--> starting graceful period and checks that the collector can still collect data if node is master");
beginGracefulPeriod();
if (node.equals(internalCluster().getMasterName())) {
assertCanCollect(collector);
} else {
assertCannotCollect(collector);
}
logger.debug("--> ending graceful period and checks that the collector cannot collect data");
endGracefulPeriod();
assertCannotCollect(collector);
logger.debug("--> disabling license and checks that the collector cannot collect data");
disableLicense();
assertCannotCollect(collector);
}
}
public void waitForNoBlocksOnNode(final String nodeId) throws Exception {
assertBusy(new Callable<Void>() {
@Override
public Void call() throws Exception {
ClusterBlocks clusterBlocks = client(nodeId).admin().cluster().prepareState().setLocal(true).execute().actionGet().getState().blocks();
assertTrue(clusterBlocks.global().isEmpty());
assertTrue(clusterBlocks.indices().values().isEmpty());
return null;
}
}, 30L, TimeUnit.SECONDS);
private IndexRecoveryCollector newIndexRecoveryCollector(String nodeId) {
if (!Strings.hasText(nodeId)) {
nodeId = randomFrom(internalCluster().getNodeNames());
}
return new IndexRecoveryCollector(internalCluster().getInstance(Settings.class, nodeId),
internalCluster().getInstance(ClusterService.class, nodeId),
internalCluster().getInstance(MarvelSettings.class, nodeId),
internalCluster().getInstance(LicenseService.class, nodeId),
client(nodeId));
}
}

60
marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java
View File

@ -5,16 +5,13 @@
*/
package org.elasticsearch.marvel.agent.collector.indices;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.block.ClusterBlock;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.marvel.license.LicenseService;
import org.junit.Test;
import java.util.Collection;
@ -23,12 +20,11 @@ import java.util.Iterator;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.*;
@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/470")
public class IndexStatsCollectorTests extends ESSingleNodeTestCase {
public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
@Test
public void testIndexStatsCollectorNoIndices() throws Exception {
waitForNoBlocksOnNode();
waitForNoBlocksOnNodes();
Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
assertThat(results, is(empty()));
@ -36,11 +32,13 @@ public class IndexStatsCollectorTests extends ESSingleNodeTestCase {
@Test
public void testIndexStatsCollectorOneIndex() throws Exception {
waitForNoBlocksOnNode();
waitForNoBlocksOnNodes();
int nbDocs = randomIntBetween(1, 20);
final String indexName = "test_" + randomInt();
final int nbDocs = randomIntBetween(1, 20);
for (int i = 0; i < nbDocs; i++) {
client().prepareIndex("test", "test").setSource("num", i).get();
client().prepareIndex(indexName, "test").setSource("num", i).get();
}
client().admin().indices().prepareRefresh().get();
assertHitCount(client().prepareCount().get(), nbDocs);
@ -60,8 +58,8 @@ public class IndexStatsCollectorTests extends ESSingleNodeTestCase {
IndexStats indexStats = indexStatsMarvelDoc.getIndexStats();
assertNotNull(indexStats);
assertThat(indexStats.getIndex(), equalTo("test"));
assertThat(indexStats.getTotal().getDocs().getCount(), equalTo((long) nbDocs));
assertThat(indexStats.getIndex(), equalTo(indexName));
assertThat(indexStats.getPrimaries().getDocs().getCount(), equalTo((long) nbDocs));
assertNotNull(indexStats.getTotal().getStore());
assertThat(indexStats.getTotal().getStore().getSizeInBytes(), greaterThan(0L));
assertThat(indexStats.getTotal().getStore().getThrottleTime().millis(), equalTo(0L));
@ -71,28 +69,30 @@ public class IndexStatsCollectorTests extends ESSingleNodeTestCase {
@Test
public void testIndexStatsCollectorMultipleIndices() throws Exception {
waitForNoBlocksOnNode();
waitForNoBlocksOnNodes();
final String indexPrefix = "test_" + randomInt() + "_";
int nbIndices = randomIntBetween(1, 5);
int[] docsPerIndex = new int[nbIndices];
for (int i = 0; i < nbIndices; i++) {
docsPerIndex[i] = randomIntBetween(1, 20);
for (int j = 0; j < docsPerIndex[i]; j++) {
client().prepareIndex("test-" + i, "test").setSource("num", i).get();
client().prepareIndex(indexPrefix + i, "test").setSource("num", i).get();
}
}
String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
client().admin().indices().prepareRefresh().get();
for (int i = 0; i < nbIndices; i++) {
assertHitCount(client().prepareCount("test-" + i).get(), docsPerIndex[i]);
assertHitCount(client().prepareCount(indexPrefix + i).get(), docsPerIndex[i]);
}
Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
assertThat(results, hasSize(nbIndices));
for (int i = 0; i < nbIndices; i++) {
String indexName = indexPrefix + i;
boolean found = false;
Iterator<MarvelDoc> it = results.iterator();
@ -104,13 +104,14 @@ public class IndexStatsCollectorTests extends ESSingleNodeTestCase {
IndexStats indexStats = indexStatsMarvelDoc.getIndexStats();
assertNotNull(indexStats);
if (indexStats.getIndex().equals("test-" + i)) {
if (indexStats.getIndex().equals(indexPrefix + i)) {
assertThat(indexStatsMarvelDoc.clusterUUID(), equalTo(clusterUUID));
assertThat(indexStatsMarvelDoc.timestamp(), greaterThan(0L));
assertThat(indexStatsMarvelDoc.type(), equalTo(IndexStatsCollector.TYPE));
assertThat(indexStats.getIndex(), equalTo(indexName));
assertNotNull(indexStats.getTotal().getDocs());
assertThat(indexStats.getTotal().getDocs().getCount(), equalTo((long) docsPerIndex[i]));
assertThat(indexStats.getPrimaries().getDocs().getCount(), equalTo((long) docsPerIndex[i]));
assertNotNull(indexStats.getTotal().getStore());
assertThat(indexStats.getTotal().getStore().getSizeInBytes(), greaterThan(0L));
assertThat(indexStats.getTotal().getStore().getThrottleTime().millis(), equalTo(0L));
@ -119,25 +120,16 @@ public class IndexStatsCollectorTests extends ESSingleNodeTestCase {
found = true;
}
}
assertThat("could not find collected stats for index [test-" + i + "]", found, is(true));
assertThat("could not find collected stats for index [" + indexPrefix + i + "]", found, is(true));
}
}
private IndexStatsCollector newIndexStatsCollector() {
return new IndexStatsCollector(getInstanceFromNode(Settings.class),
getInstanceFromNode(ClusterService.class),
getInstanceFromNode(MarvelSettings.class),
client());
}
public void waitForNoBlocksOnNode() throws InterruptedException {
final long start = System.currentTimeMillis();
final TimeValue timeout = TimeValue.timeValueSeconds(30);
ImmutableSet<ClusterBlock> blocks;
do {
blocks = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState().blocks().global();
}
while (!blocks.isEmpty() && (System.currentTimeMillis() - start) < timeout.millis());
assertTrue(blocks.isEmpty());
String nodeId = randomFrom(internalCluster().getNodeNames());
return new IndexStatsCollector(internalCluster().getInstance(Settings.class, nodeId),
internalCluster().getInstance(ClusterService.class, nodeId),
internalCluster().getInstance(MarvelSettings.class, nodeId),
internalCluster().getInstance(LicenseService.class, nodeId),
client(nodeId));
}
}

32
marvel/src/test/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsCollectorTests.java
View File

@ -11,17 +11,18 @@ import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider
import org.elasticsearch.common.inject.Provider;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.discovery.DiscoveryService;
import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.LicenseService;
import org.elasticsearch.node.service.NodeService;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import java.util.Collection;
import static org.hamcrest.Matchers.*;
public class NodeStatsCollectorTests extends ESIntegTestCase {
public class NodeStatsCollectorTests extends AbstractCollectorTestCase {
@Test
public void testNodeStatsCollector() throws Exception {
@ -50,10 +51,37 @@ public class NodeStatsCollectorTests extends ESIntegTestCase {
}
}
@Test
public void testNodeStatsCollectorWithLicensing() {
String[] nodes = internalCluster().getNodeNames();
for (String node : nodes) {
logger.debug("--> creating a new instance of the collector");
NodeStatsCollector collector = newNodeStatsCollector(node);
assertNotNull(collector);
logger.debug("--> enabling license and checks that the collector can collect data");
enableLicense();
assertCanCollect(collector);
logger.debug("--> starting graceful period and checks that the collector can still collect data");
beginGracefulPeriod();
assertCanCollect(collector);
logger.debug("--> ending graceful period and checks that the collector cannot collect data");
endGracefulPeriod();
assertCannotCollect(collector);
logger.debug("--> disabling license and checks that the collector cannot collect data");
disableLicense();
assertCannotCollect(collector);
}
}
private NodeStatsCollector newNodeStatsCollector(final String nodeId) {
return new NodeStatsCollector(internalCluster().getInstance(Settings.class, nodeId),
internalCluster().getInstance(ClusterService.class, nodeId),
internalCluster().getInstance(MarvelSettings.class, nodeId),
internalCluster().getInstance(LicenseService.class, nodeId),
internalCluster().getInstance(NodeService.class, nodeId),
internalCluster().getInstance(DiscoveryService.class, nodeId),
new Provider<DiskThresholdDecider>() {

10
marvel/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingsTests.java
View File

@ -15,7 +15,7 @@ import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.*;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
public class MarvelSettingsTests extends ESIntegTestCase {
@ -29,6 +29,7 @@ public class MarvelSettingsTests extends ESIntegTestCase {
private final TimeValue recoveryTimeout = randomTimeValue();
private final Boolean recoveryActiveOnly = randomBoolean();
private final String[] collectors = randomStringArray();
private final TimeValue licenseGracePeriod = randomExpirationDelay();
@Override
protected Settings nodeSettings(int nodeOrdinal) {
@ -51,6 +52,7 @@ public class MarvelSettingsTests extends ESIntegTestCase {
.put(MarvelSettings.INDEX_RECOVERY_TIMEOUT, recoveryTimeout)
.put(MarvelSettings.INDEX_RECOVERY_ACTIVE_ONLY, recoveryActiveOnly)
.putArray(MarvelSettings.COLLECTORS, collectors)
.put(MarvelSettings.LICENSE_GRACE_PERIOD, licenseGracePeriod)
.build();
}
@ -67,6 +69,8 @@ public class MarvelSettingsTests extends ESIntegTestCase {
assertThat(marvelSettings.recoveryTimeout().millis(), equalTo(recoveryTimeout.millis()));
assertThat(marvelSettings.recoveryActiveOnly(), equalTo(recoveryActiveOnly));
assertArrayEquals(marvelSettings.collectors(), collectors);
assertThat(marvelSettings.licenseExpirationGracePeriod().millis(), equalTo(licenseGracePeriod.millis()));
assertThat(marvelSettings.licenseExpirationGracePeriod().millis(), allOf(greaterThanOrEqualTo(0L), lessThanOrEqualTo(MarvelSettings.MAX_LICENSE_GRACE_PERIOD.millis())));
for (final MarvelSetting setting : MarvelSettings.dynamicSettings()) {
assertThat(marvelSettings.getSettingValue(setting.getName()), equalTo(setting.getValue()));
@ -148,4 +152,8 @@ public class MarvelSettingsTests extends ESIntegTestCase {
}
return items;
}
private TimeValue randomExpirationDelay() {
return randomBoolean() ? randomTimeValue() : TimeValue.timeValueHours(randomIntBetween(-10, 10) * 24);
}
}

19
marvel/src/test/java/org/elasticsearch/marvel/license/LicenseIntegrationTests.java
View File

@ -30,6 +30,8 @@ import java.util.List;
import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.is;
@ClusterScope(scope = SUITE, transportClientRatio = 0, numClientNodes = 0)
public class LicenseIntegrationTests extends ESIntegTestCase {
@ -44,22 +46,27 @@ public class LicenseIntegrationTests extends ESIntegTestCase {
@Test
public void testEnableDisableLicense() {
assertMarvelMode(Mode.STANDARD);
assertThat(getLicenseService().mode(), equalTo(Mode.STANDARD));
assertThat(getLicenseService().enabled(), is(true));
assertThat(getLicenseService().expiryDate(), greaterThan(0L));
disableLicensing();
assertMarvelMode(Mode.LITE);
assertThat(getLicenseService().mode(), equalTo(Mode.LITE));
assertThat(getLicenseService().enabled(), is(false));
assertThat(getLicenseService().expiryDate(), greaterThan(0L));
enableLicensing();
assertMarvelMode(Mode.STANDARD);
assertThat(getLicenseService().mode(), equalTo(Mode.STANDARD));
assertThat(getLicenseService().enabled(), is(true));
assertThat(getLicenseService().expiryDate(), greaterThan(0L));
}
private void assertMarvelMode(Mode expected) {
private LicenseService getLicenseService() {
LicenseService licenseService = internalCluster().getInstance(LicenseService.class);
assertNotNull(licenseService);
assertThat(licenseService.mode(), equalTo(expected));
return licenseService;
}
public static void disableLicensing() {
for (MockLicenseService service : internalCluster().getInstances(MockLicenseService.class)) {
service.disable();