honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Marvel: Fix integration tests in org.elasticsearch.marvel.agent.renderer package 

Closes elastic/elasticsearch#470

Original commit: elastic/x-pack-elasticsearch@db07ac416d
This commit is contained in:
Tanguy Leroux 2015-10-06 11:28:08 +02:00
parent 1fc674abfd
commit a80cc90240
13 changed files with 267 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
marvel/src/main/java/org/elasticsearch/marvel/agent/AgentService.java
View File

@ -82,7 +82,6 @@ public class AgentService extends AbstractLifecycleComponent<AgentService> imple
workerThread = new Thread(exportingWorker, EsExecutors.threadName(settings, "marvel.exporters"));
workerThread.setDaemon(true);
workerThread.start();
}
}
@ -154,13 +153,10 @@ public class AgentService extends AbstractLifecycleComponent<AgentService> imple
@Override
public void run() {
boolean firstRun = true;
while (!closed) {
// sleep first to allow node to complete initialization before collecting the first start
try {
long interval = (firstRun && (marvelSettings.startUpDelay() != null)) ? marvelSettings.startUpDelay().millis() : samplingInterval;
Thread.sleep(interval);
Thread.sleep(samplingInterval);
if (closed) {
continue;
@ -198,8 +194,6 @@ public class AgentService extends AbstractLifecycleComponent<AgentService> imple
Thread.currentThread().interrupt();
} catch (Throwable t) {
logger.error("background thread had an uncaught exception", t);
} finally {
firstRun = false;
}
}
logger.debug("worker shutdown");

7
marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java
View File

@ -30,7 +30,6 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
public static final TimeValue MAX_LICENSE_GRACE_PERIOD = TimeValue.timeValueHours(7 * 24);
public static final String INTERVAL = PREFIX + "interval";
public static final String STARTUP_DELAY = PREFIX + "startup.delay";
public static final String INDEX_STATS_TIMEOUT = PREFIX + "index.stats.timeout";
public static final String INDICES_STATS_TIMEOUT = PREFIX + "indices.stats.timeout";
public static final String INDICES = PREFIX + "indices";
@ -60,8 +59,6 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
Map<String, MarvelSetting> map = new HashMap<>();
map.put(INTERVAL, timeSetting(INTERVAL, TimeValue.timeValueSeconds(10),
"Sampling interval between two collections (default to 10s)"));
map.put(STARTUP_DELAY, timeSetting(STARTUP_DELAY, null,
"Waiting time before the agent start to collect data (default to sampling interval)"));
map.put(INDEX_STATS_TIMEOUT, timeoutSetting(INDEX_STATS_TIMEOUT, TimeValue.timeValueMinutes(10),
"Timeout value when collecting index statistics (default to 10m)"));
map.put(INDICES_STATS_TIMEOUT, timeoutSetting(INDICES_STATS_TIMEOUT, TimeValue.timeValueMinutes(10),
@ -146,10 +143,6 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
return getSettingValue(INTERVAL);
}
public TimeValue startUpDelay() {
return getSettingValue(STARTUP_DELAY);
}
public TimeValue indexStatsTimeout() {
return getSettingValue(INDEX_STATS_TIMEOUT);
}

82
marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java
View File

@ -5,42 +5,38 @@
*/
package org.elasticsearch.marvel.agent.collector.indices;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.MarvelLicensee;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.junit.Before;
import org.junit.Test;
import java.util.Collection;
import java.util.Iterator;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.*;
@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/470")
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, randomDynamicTemplates = false, transportClientRatio = 0.0, numDataNodes = 1, numClientNodes = 0)
@ClusterScope(numClientNodes = 0)
public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
@Test
public void testIndexStatsCollectorNoIndices() throws Exception {
waitForNoBlocksOnNodes();
@Override
protected int numberOfReplicas() {
return 0;
}
Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
assertThat(results, is(empty()));
@Before
public void beforeIndexStatsCollectorTests() throws Exception {
waitForNoBlocksOnNodes();
}
@Test
public void testIndexStatsCollectorOneIndex() throws Exception {
waitForNoBlocksOnNodes();
final String indexName = "one-index";
final int nbDocs = randomIntBetween(1, 20);
@ -48,30 +44,14 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
client().prepareIndex(indexName, "test").setSource("num", i).get();
}
waitForRelocation();
ensureGreen(indexName);
refresh();
securedFlush();
securedRefresh();
securedEnsureGreen(indexName);
assertHitCount(client().prepareCount().get(), nbDocs);
logger.debug("--> wait for index stats to report data about indices");
assertBusy(new Runnable() {
@Override
public void run() {
IndicesStatsResponse response = client(internalCluster().getMasterName()).admin().indices().prepareStats().setRefresh(true).get();
assertNotNull(response.getIndices());
assertThat(response.getIndices().size(), greaterThan(0));
}
}, 30L, TimeUnit.SECONDS);
Collection<MarvelDoc> results = assertBusy(new Callable<Collection<MarvelDoc>>() {
@Override
public Collection<MarvelDoc> call() throws Exception {
Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
assertThat(results, hasSize(1));
return results;
}
}, 30L, TimeUnit.SECONDS);
Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
assertThat(results, hasSize(1));
MarvelDoc marvelDoc = results.iterator().next();
assertNotNull(marvelDoc);
@ -96,8 +76,6 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
@Test
public void testIndexStatsCollectorMultipleIndices() throws Exception {
waitForNoBlocksOnNodes();
final String indexPrefix = "multi-indices-";
final int nbIndices = randomIntBetween(1, 5);
int[] docsPerIndex = new int[nbIndices];
@ -109,34 +87,18 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
}
}
String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
client().admin().indices().prepareRefresh().get();
securedFlush();
securedRefresh();
securedEnsureGreen(indexPrefix + "*");
for (int i = 0; i < nbIndices; i++) {
assertHitCount(client().prepareCount(indexPrefix + i).get(), docsPerIndex[i]);
}
waitForRelocation();
ensureGreen();
refresh();
String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
logger.debug("--> wait for index stats to report data about indices");
assertBusy(new Runnable() {
@Override
public void run() {
IndicesStatsResponse response = client().admin().indices().prepareStats().setRefresh(true).get();
assertNotNull(response.getIndices());
assertThat(response.getIndices().size(), greaterThan(0));
}
}, 30L, TimeUnit.SECONDS);
Collection<MarvelDoc> results = assertBusy(new Callable<Collection<MarvelDoc>>() {
@Override
public Collection<MarvelDoc> call() throws Exception {
Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
assertThat(results, hasSize(nbIndices));
return results;
}
}, 30L, TimeUnit.SECONDS);
Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
assertThat(results, hasSize(nbIndices));
for (int i = 0; i < nbIndices; i++) {
String indexName = indexPrefix + i;

53
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoIT.java → marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoTests.java
View File

@ -14,47 +14,59 @@ import org.elasticsearch.license.core.License;
import org.elasticsearch.marvel.agent.collector.cluster.ClusterInfoCollector;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.*;
public class ClusterInfoIT extends MarvelIntegTestCase {
@ClusterScope(scope = TEST)
public class ClusterInfoTests extends MarvelIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MarvelSettings.INTERVAL, "3s")
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, ClusterInfoCollector.NAME)
.build();
}
@Before
public void init() throws Exception {
updateMarvelInterval(3L, TimeUnit.SECONDS);
}
@After
public void cleanup() throws Exception {
updateMarvelInterval(-1, TimeUnit.SECONDS);
wipeMarvelIndices();
}
@Test
public void testClusterInfo() throws Exception {
ensureGreen();
securedEnsureGreen();
final String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
assertTrue(Strings.hasText(clusterUUID));
logger.debug("--> waiting for cluster info collector to collect data (ie, the trial marvel license)");
GetResponse response = assertBusy(new Callable<GetResponse>() {
@Override
public GetResponse call() throws Exception {
// Checks if the marvel data index exists (it should have been created by the LicenseCollector)
assertTrue(MarvelSettings.MARVEL_DATA_INDEX_NAME + " index does not exist", client().admin().indices().prepareExists(MarvelSettings.MARVEL_DATA_INDEX_NAME).get().isExists());
ensureYellow(MarvelSettings.MARVEL_DATA_INDEX_NAME);
logger.debug("--> waiting for the marvel data index to be created (it should have been created by the LicenseCollector)");
awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_NAME);
client().admin().indices().prepareRefresh(MarvelSettings.MARVEL_DATA_INDEX_NAME).get();
GetResponse response = client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterInfoCollector.TYPE, clusterUUID).get();
assertTrue(MarvelSettings.MARVEL_DATA_INDEX_NAME + " document does not exist", response.isExists());
return response;
}
}, 30L, TimeUnit.SECONDS);
logger.debug("--> waiting for cluster info collector to collect data");
awaitMarvelDocsCount(equalTo(1L), ClusterInfoCollector.TYPE);
logger.debug("--> retrieving cluster info document");
GetResponse response = client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterInfoCollector.TYPE, clusterUUID).get();
assertTrue(MarvelSettings.MARVEL_DATA_INDEX_NAME + " document does not exist", response.isExists());
logger.debug("--> checking that the document contains all required information");
logger.debug("--> checking that the document contains license information");
assertThat(response.getIndex(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_NAME));
@ -98,8 +110,12 @@ public class ClusterInfoIT extends MarvelIntegTestCase {
assertThat(clusterStats, instanceOf(Map.class));
assertThat(((Map) clusterStats).size(), greaterThan(0));
assertMarvelTemplateInstalled();
logger.debug("--> check that the cluster_info is not indexed");
refresh();
securedFlush();
securedRefresh();
assertHitCount(client().prepareCount()
.setIndices(MarvelSettings.MARVEL_DATA_INDEX_NAME)
.setTypes(ClusterInfoCollector.TYPE)
@ -109,6 +125,5 @@ public class ClusterInfoIT extends MarvelIntegTestCase {
.should(QueryBuilders.matchQuery(License.XFields.STATUS.underscore().toString(), License.Status.EXPIRED.label()))
.minimumNumberShouldMatch(1)
).get(), 0L);
}
}

46
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateIT.java → marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java
View File

@ -10,41 +10,54 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.greaterThan;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/730")
public class ClusterStateIT extends MarvelIntegTestCase {
@ClusterScope(scope = Scope.TEST)
public class ClusterStateTests extends MarvelIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MarvelSettings.INTERVAL, "3s")
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, ClusterStateCollector.NAME)
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}
@Before
public void init() throws Exception {
updateMarvelInterval(3L, TimeUnit.SECONDS);
waitForMarvelIndices();
}
@After
public void cleanup() throws Exception {
updateMarvelInterval(-1, TimeUnit.SECONDS);
wipeMarvelIndices();
}
@Test
public void testClusterState() throws Exception {
ensureGreen();
logger.debug("--> waiting for documents to be collected");
awaitMarvelDocsCount(greaterThan(0L), ClusterStateCollector.TYPE);
logger.debug("--> searching for marvel documents of type [{}]", ClusterStateCollector.TYPE);
SearchResponse response = client().prepareSearch().setTypes(ClusterStateCollector.TYPE).get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> checking that every document contains the expected fields");
@ -63,22 +76,11 @@ public class ClusterStateIT extends MarvelIntegTestCase {
/**
* This test should fail if the mapping for the 'nodes' attribute
* in the 'cluster_state' document is NOT set to 'enable: false'
*
* See
*/
@Test
public void testNoNodesIndexing() throws Exception {
ensureGreen();
logger.debug("--> forcing marvel's index template update");
assertAcked(client().admin().indices().preparePutTemplate("marvel").setSource(MarvelTemplateUtils.loadDefaultTemplate()).execute().actionGet());
logger.debug("--> deleting all marvel indices");
deleteMarvelIndices();
logger.debug("--> checking for template existence");
logger.debug("--> waiting for documents to be collected");
awaitMarvelDocsCount(greaterThan(0L), ClusterStateCollector.TYPE);
assertMarvelTemplateInstalled();
logger.debug("--> searching for marvel documents of type [{}]", ClusterStateCollector.TYPE);
SearchResponse response = client().prepareSearch().setTypes(ClusterStateCollector.TYPE).get();

24
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsTests.java
View File

@ -13,34 +13,40 @@ import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
import static org.hamcrest.Matchers.greaterThan;
@ClusterScope(scope = SUITE, numClientNodes = 0)
@ClusterScope(scope = Scope.TEST, numClientNodes = 0)
public class ClusterStatsTests extends MarvelIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MarvelSettings.INTERVAL, "3s")
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, ClusterStatsCollector.NAME)
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}
@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/729")
@After
public void cleanup() throws Exception {
updateMarvelInterval(-1, TimeUnit.SECONDS);
wipeMarvelIndices();
}
@Test
public void testClusterStats() throws Exception {
// lets wait with the collection until all the shards started
stopCollection();
logger.debug("--> creating some indices so that every data nodes will at least a shard");
ClusterStatsNodes.Counts counts = client().admin().cluster().prepareClusterStats().get().getNodesStats().getCounts();
assertThat(counts.getTotal(), greaterThan(0));
@ -58,7 +64,7 @@ public class ClusterStatsTests extends MarvelIntegTestCase {
securedEnsureGreen();
// ok.. we'll start collecting now...
startCollection();
updateMarvelInterval(3L, TimeUnit.SECONDS);
awaitMarvelTemplateInstalled();

44
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexRecoveryIT.java → marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexRecoveryTests.java
View File

@ -5,43 +5,73 @@
*/
package org.elasticsearch.marvel.agent.renderer.indices;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryCollector;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.junit.After;
import org.junit.Test;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.is;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
@ClusterScope(scope = TEST)
public class IndexRecoveryTests extends MarvelIntegTestCase {
@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/730")
public class IndexRecoveryIT extends MarvelIntegTestCase {
private static final String INDEX_PREFIX = "test-index-recovery-";
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MarvelSettings.INTERVAL, "3s")
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.INDICES, INDEX_PREFIX + "*")
.put(MarvelSettings.COLLECTORS, IndexRecoveryCollector.NAME)
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}
@After
public void cleanup() throws Exception {
updateMarvelInterval(-1, TimeUnit.SECONDS);
wipeMarvelIndices();
}
@Test
public void testIndexRecovery() throws Exception {
logger.debug("--> creating some indices so that index recovery collector reports data");
for (int i = 0; i < randomIntBetween(1, 5); i++) {
client().prepareIndex("test-" + i, "foo").setRefresh(true).setSource("field1", "value1").get();
for (int i = 0; i < randomIntBetween(1, 10); i++) {
client().prepareIndex(INDEX_PREFIX + i, "foo").setSource("field1", "value1").get();
}
logger.debug("--> wait for index recovery collector to collect data");
assertBusy(new Runnable() {
@Override
public void run() {
securedFlush();
securedRefresh();
RecoveryResponse recoveries = client().admin().indices().prepareRecoveries().get();
assertThat(recoveries.hasRecoveries(), is(true));
}
});
updateMarvelInterval(3L, TimeUnit.SECONDS);
waitForMarvelIndices();
awaitMarvelDocsCount(greaterThan(0L), IndexRecoveryCollector.TYPE);
logger.debug("--> searching for marvel documents of type [{}]", IndexRecoveryCollector.TYPE);
SearchResponse response = client().prepareSearch().setTypes(IndexRecoveryCollector.TYPE).get();
SearchResponse response = client().prepareSearch(MarvelSettings.MARVEL_INDICES_PREFIX + "*").setTypes(IndexRecoveryCollector.TYPE).get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> checking that every document contains the expected fields");

25
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsIT.java → marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsTests.java
View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.marvel.agent.renderer.indices;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
@ -14,24 +13,36 @@ import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.junit.After;
import org.junit.Test;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.hamcrest.Matchers.greaterThan;
@LuceneTestCase.AwaitsFix(bugUrl = "https://internal-build.elastic.co/job/es_xplugins_master_medium/3319/testReport/junit/org.elasticsearch.marvel.agent.renderer.indices/IndexStatsIT/testIndexStats/")
public class IndexStatsIT extends MarvelIntegTestCase {
@ClusterScope(scope = Scope.TEST, numClientNodes = 0)
public class IndexStatsTests extends MarvelIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MarvelSettings.INTERVAL, "3s")
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, IndexStatsCollector.NAME)
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}
@After
public void cleanup() throws Exception {
updateMarvelInterval(-1, TimeUnit.SECONDS);
wipeMarvelIndices();
}
@Test
public void testIndexStats() throws Exception {
logger.debug("--> creating some indices for future index stats");
@ -50,7 +61,11 @@ public class IndexStatsIT extends MarvelIntegTestCase {
}
}
ensureGreen();
securedFlush();
securedRefresh();
updateMarvelInterval(3L, TimeUnit.SECONDS);
waitForMarvelIndices();
awaitMarvelDocsCount(greaterThan(0L), IndexStatsCollector.TYPE);

44
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsIT.java → marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsTests.java
View File

@ -5,63 +5,79 @@
*/
package org.elasticsearch.marvel.agent.renderer.indices;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsCollector;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.junit.After;
import org.junit.Test;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.hamcrest.Matchers.greaterThan;
@AwaitsFix(bugUrl="https://github.com/elastic/x-plugins/issues/729")
public class IndicesStatsIT extends MarvelIntegTestCase {
@ClusterScope(scope = Scope.TEST, numClientNodes = 0)
public class IndicesStatsTests extends MarvelIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MarvelSettings.INTERVAL, "3s")
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, IndicesStatsCollector.NAME)
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}
@After
public void cleanup() throws Exception {
updateMarvelInterval(-1, TimeUnit.SECONDS);
wipeMarvelIndices();
}
@Test
public void testIndicesStats() throws Exception {
logger.debug("--> creating some indices for future indices stats");
final int nbIndices = randomIntBetween(1, 5);
for (int i = 0; i < nbIndices; i++) {
createIndex("stat" + i);
createIndex("stat-" + i);
}
final long[] nbDocsPerIndex = new long[nbIndices];
for (int i = 0; i < nbIndices; i++) {
nbDocsPerIndex[i] = randomIntBetween(1, 50);
for (int j = 0; j < nbDocsPerIndex[i]; j++) {
client().prepareIndex("stat" + i, "type1").setSource("num", i).get();
client().prepareIndex("stat-" + i, "type1").setSource("num", i).get();
}
}
awaitMarvelDocsCount(greaterThan(0L), IndicesStatsCollector.TYPE);
logger.debug("--> wait for indicesx stats collector to collect global stat");
logger.debug("--> wait for indices stats collector to collect stats for all primaries shards");
assertBusy(new Runnable() {
@Override
public void run() {
securedFlush();
securedRefresh();
for (int i = 0; i < nbIndices; i++) {
CountResponse count = client().prepareCount()
.setTypes(IndicesStatsCollector.TYPE)
.get();
assertThat(count.getCount(), greaterThan(0L));
IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().get();
assertThat(indicesStats.getPrimaries().getDocs().getCount(), greaterThan(0L));
}
}
});
updateMarvelInterval(3L, TimeUnit.SECONDS);
waitForMarvelIndices();
logger.debug("--> wait for indices stats collector to collect global stat");
awaitMarvelDocsCount(greaterThan(0L), IndicesStatsCollector.TYPE);
logger.debug("--> searching for marvel documents of type [{}]", IndicesStatsCollector.TYPE);
SearchResponse response = client().prepareSearch().setTypes(IndicesStatsCollector.TYPE).get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));

25
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsIT.java → marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java
View File

@ -8,26 +8,41 @@ package org.elasticsearch.marvel.agent.renderer.node;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
import org.elasticsearch.marvel.agent.exporter.local.LocalExporter;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.junit.After;
import org.junit.Test;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.hamcrest.Matchers.greaterThan;
public class NodeStatsIT extends MarvelIntegTestCase {
@ClusterScope(scope = Scope.TEST)
public class NodeStatsTests extends MarvelIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MarvelSettings.INTERVAL, "3s")
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, NodeStatsCollector.NAME)
.put("marvel.agent.exporters.default_local.type", LocalExporter.TYPE)
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}
@After
public void cleanup() throws Exception {
updateMarvelInterval(-1, TimeUnit.SECONDS);
wipeMarvelIndices();
}
@Test
public void testNodeStats() throws Exception {
logger.debug("--> creating some indices for future node stats");
@ -36,6 +51,12 @@ public class NodeStatsIT extends MarvelIntegTestCase {
client().prepareIndex("test", "foo").setSource("value", randomInt()).get();
}
securedFlush();
securedRefresh();
updateMarvelInterval(3L, TimeUnit.SECONDS);
waitForMarvelIndices();
awaitMarvelDocsCount(greaterThan(0L), NodeStatsCollector.TYPE);
logger.debug("--> searching for marvel documents of type [{}]", NodeStatsCollector.TYPE);

28
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsIT.java → marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsTests.java
View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.marvel.agent.renderer.shards;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -18,15 +17,19 @@ import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.junit.After;
import org.junit.Test;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.*;
@AwaitsFix(bugUrl="https://github.com/elastic/x-plugins/issues/729")
public class ShardsIT extends MarvelIntegTestCase {
@ClusterScope(scope = Scope.TEST)
public class ShardsTests extends MarvelIntegTestCase {
private static final String INDEX_PREFIX = "test-shards-";
@ -34,12 +37,20 @@ public class ShardsIT extends MarvelIntegTestCase {
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MarvelSettings.INTERVAL, "3s")
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, ShardsCollector.NAME)
.put(MarvelSettings.INDICES, INDEX_PREFIX + "*")
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}
@After
public void cleanup() throws Exception {
updateMarvelInterval(-1, TimeUnit.SECONDS);
wipeMarvelIndices();
}
@Test
public void testShards() throws Exception {
logger.debug("--> creating some indices so that shards collector reports data");
@ -47,6 +58,12 @@ public class ShardsIT extends MarvelIntegTestCase {
client().prepareIndex(INDEX_PREFIX + i, "foo").setRefresh(true).setSource("field1", "value1").get();
}
securedFlush();
securedRefresh();
updateMarvelInterval(3L, TimeUnit.SECONDS);
waitForMarvelIndices();
awaitMarvelDocsCount(greaterThan(0L), ShardsCollector.TYPE);
logger.debug("--> searching for marvel documents of type [{}]", ShardsCollector.TYPE);
@ -75,6 +92,9 @@ public class ShardsIT extends MarvelIntegTestCase {
final String indexName = INDEX_PREFIX + randomInt();
assertAcked(prepareCreate(indexName).setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
updateMarvelInterval(3L, TimeUnit.SECONDS);
waitForMarvelIndices();
awaitMarvelDocsCount(greaterThan(0L), ShardsCollector.TYPE);
SearchRequestBuilder searchRequestBuilder = client()

3
marvel/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingsTests.java
View File

@ -19,7 +19,6 @@ import static org.hamcrest.Matchers.equalTo;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
public class MarvelSettingsTests extends MarvelIntegTestCase {
private final TimeValue startUp = newRandomTimeValue();
private final TimeValue interval = newRandomTimeValue();
private final TimeValue indexStatsTimeout = newRandomTimeValue();
private final String[] indices = randomStringArray();
@ -41,7 +40,6 @@ public class MarvelSettingsTests extends MarvelIntegTestCase {
private Settings marvelSettings() {
return Settings.builder()
.put(MarvelSettings.STARTUP_DELAY, startUp)
.put(MarvelSettings.INTERVAL, interval)
.put(MarvelSettings.INDEX_STATS_TIMEOUT, indexStatsTimeout)
.putArray(MarvelSettings.INDICES, indices)
@ -57,7 +55,6 @@ public class MarvelSettingsTests extends MarvelIntegTestCase {
public void testMarvelSettings() throws Exception {
logger.info("--> testing marvel settings service initialization");
for (final MarvelSettings marvelSettings : internalCluster().getInstances(MarvelSettings.class)) {
assertThat(marvelSettings.startUpDelay().millis(), equalTo(startUp.millis()));
assertThat(marvelSettings.interval().millis(), equalTo(interval.millis()));
assertThat(marvelSettings.indexStatsTimeout().millis(), equalTo(indexStatsTimeout.millis()));
assertArrayEquals(marvelSettings.indices(), indices);

38
marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java
View File

@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.CountDown;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.license.plugin.LicensePlugin;
@ -119,6 +120,26 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
}
}
protected void wipeMarvelIndices() throws Exception {
CountDown retries = new CountDown(3);
assertBusy(new Runnable() {
@Override
public void run() {
try {
boolean exist = client().admin().indices().prepareExists(".marvel-es-*").get().isExists();
if (exist) {
deleteMarvelIndices();
} else {
retries.countDown();
}
} catch (IndexNotFoundException e) {
retries.countDown();
}
assertThat(retries.isCountedDown(), is(true));
}
});
}
protected void deleteMarvelIndices() {
if (shieldEnabled) {
try {
@ -223,6 +244,17 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
fail("marvel template could not be found");
}
protected void waitForMarvelIndices() throws Exception {
awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_NAME);
awaitIndexExists(MarvelSettings.MARVEL_INDICES_PREFIX + "*");
assertBusy(new Runnable() {
@Override
public void run() {
ensureMarvelIndicesGreen();
}
});
}
protected void awaitIndexExists(final String... indices) throws Exception {
assertBusy(new Runnable() {
@Override
@ -313,6 +345,10 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
}
}
protected void updateMarvelInterval(long value, TimeUnit timeUnit) {
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(MarvelSettings.INTERVAL, value, timeUnit)));
}
/** Shield related settings */
public static class ShieldSettings {
@ -340,7 +376,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
public static final String ROLES =
"test:\n" + // a user for the test infra.
" cluster: cluster:monitor/nodes/info, cluster:monitor/nodes/stats, cluster:monitor/state, cluster:monitor/health, cluster:monitor/stats, cluster:admin/settings/update, cluster:admin/repository/delete, cluster:monitor/nodes/liveness, indices:admin/template/get, indices:admin/template/put, indices:admin/template/delete\n" +
" cluster: cluster:monitor/nodes/info, cluster:monitor/nodes/stats, cluster:monitor/state, cluster:monitor/health, cluster:monitor/stats, cluster:monitor/task, cluster:admin/settings/update, cluster:admin/repository/delete, cluster:monitor/nodes/liveness, indices:admin/template/get, indices:admin/template/put, indices:admin/template/delete\n" +
" indices:\n" +
" '*': all\n" +
"\n" +