honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[TEST] Changed internal distributed ml tests to extend from ESIntegTest and only run with ml to avoid noise. 

Changed DatafeedJobsIT to extend directly from SecurityIntegTest as it uses external cluster.

Original commit: elastic/x-pack-elasticsearch@d646c59491
This commit is contained in:
Martijn van Groningen 2017-03-04 08:10:46 +01:00
parent 49826ae134
commit 6365dec42e
4 changed files with 49 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
plugin/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
View File

@ -8,14 +8,12 @@ package org.elasticsearch.license;
import org.elasticsearch.ElasticsearchSecurityException;
import org.elasticsearch.action.support.PlainListenableActionFuture;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.license.License.OperationMode;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.xpack.TestXPackTransportClient;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.ml.action.CloseJobAction;
import org.elasticsearch.xpack.ml.action.DeleteDatafeedAction;
import org.elasticsearch.xpack.ml.action.DeleteJobAction;
@ -37,20 +35,6 @@ import static org.hamcrest.Matchers.is;
public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal));
settings.put(XPackSettings.SECURITY_ENABLED.getKey(), true);
return settings.build();
}
@Override
protected Settings transportClientSettings() {
Settings.Builder settings = Settings.builder().put(super.transportClientSettings());
settings.put(XPackSettings.SECURITY_ENABLED.getKey(), true);
return settings.build();
}
@Before
public void resetLicensing() {
enableLicensing();

33
plugin/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java
View File

@ -16,7 +16,7 @@ import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.SecurityIntegTestCase;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
@ -24,12 +24,12 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
import org.elasticsearch.xpack.persistent.PersistentActionCoordinator;
import org.elasticsearch.xpack.persistent.PersistentActionRequest;
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
import org.elasticsearch.xpack.persistent.PersistentTasks;
import org.elasticsearch.xpack.security.Security;
import org.junit.After;
import java.io.IOException;
import java.util.ArrayList;
@ -40,16 +40,15 @@ import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.XContentTestUtils.convertToMap;
import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder;
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDatafeed;
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob;
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.deleteAllDatafeeds;
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.deleteAllJobs;
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.getDataCounts;
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.indexDocs;
import static org.hamcrest.Matchers.equalTo;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0, numClientNodes = 0,
transportClientRatio = 0, supportsDedicatedMasters = false)
public class DatafeedJobsIT extends BaseMlIntegTestCase {
@Override
protected boolean ignoreExternalCluster() {
return false;
}
public class DatafeedJobsIT extends SecurityIntegTestCase {
@Override
protected Settings externalClusterClientSettings() {
@ -60,6 +59,12 @@ public class DatafeedJobsIT extends BaseMlIntegTestCase {
return settings.build();
}
@After
public void cleanupWorkaround() throws Exception {
deleteAllDatafeeds(client());
deleteAllJobs(client());
}
public void ensureClusterStateConsistency() throws IOException {
if (cluster() != null && cluster().size() > 0) {
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedWriteables());
@ -118,13 +123,13 @@ public class DatafeedJobsIT extends BaseMlIntegTestCase {
long now = System.currentTimeMillis();
long oneWeekAgo = now - 604800000;
long twoWeeksAgo = oneWeekAgo - 604800000;
indexDocs("data-1", numDocs, twoWeeksAgo, oneWeekAgo);
indexDocs(logger, "data-1", numDocs, twoWeeksAgo, oneWeekAgo);
client().admin().indices().prepareCreate("data-2")
.addMapping("type", "time", "type=date")
.get();
long numDocs2 = randomIntBetween(32, 2048);
indexDocs("data-2", numDocs2, oneWeekAgo, now);
indexDocs(logger, "data-2", numDocs2, oneWeekAgo, now);
Job.Builder job = createScheduledJob("lookback-job");
PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build());
@ -163,7 +168,7 @@ public class DatafeedJobsIT extends BaseMlIntegTestCase {
long numDocs1 = randomIntBetween(32, 2048);
long now = System.currentTimeMillis();
long lastWeek = now - 604800000;
indexDocs("data", numDocs1, lastWeek, now);
indexDocs(logger, "data", numDocs1, lastWeek, now);
Job.Builder job = createScheduledJob("realtime-job");
PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build());
@ -192,7 +197,7 @@ public class DatafeedJobsIT extends BaseMlIntegTestCase {
long numDocs2 = randomIntBetween(2, 64);
now = System.currentTimeMillis();
indexDocs("data", numDocs2, now + 5000, now + 6000);
indexDocs(logger, "data", numDocs2, now + 5000, now + 6000);
assertBusy(() -> {
DataCounts dataCounts = getDataCounts(job.getId());
assertThat(dataCounts.getProcessedRecordCount(), equalTo(numDocs1 + numDocs2));

4
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
View File

@ -94,7 +94,7 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
long now = System.currentTimeMillis();
long weekAgo = now - 604800000;
long twoWeeksAgo = weekAgo - 604800000;
indexDocs("data", numDocs1, twoWeeksAgo, weekAgo);
indexDocs(logger, "data", numDocs1, twoWeeksAgo, weekAgo);
Job.Builder job = createScheduledJob("job_id");
PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build());
@ -147,7 +147,7 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
long numDocs2 = randomIntBetween(2, 64);
long now2 = System.currentTimeMillis();
indexDocs("data", numDocs2, now2 + 5000, now2 + 6000);
indexDocs(logger, "data", numDocs2, now2 + 5000, now2 + 6000);
assertBusy(() -> {
DataCounts dataCounts = getDataCounts(job.getId());
assertEquals(numDocs1 + numDocs2, dataCounts.getProcessedRecordCount());

51
plugin/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.ml.support;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@ -16,11 +17,11 @@ import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.node.NodeMocksPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.SecurityIntegTestCase;
import org.elasticsearch.test.discovery.TestZenDiscovery;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry;
@ -42,15 +43,13 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.junit.After;
import org.junit.Before;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
@ -63,7 +62,7 @@ import static org.hamcrest.Matchers.is;
*/
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0,
transportClientRatio = 0, supportsDedicatedMasters = false)
public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
public abstract class BaseMlIntegTestCase extends ESIntegTestCase {
@Override
protected boolean ignoreExternalCluster() {
@ -93,11 +92,19 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
return settings.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singleton(XPackPlugin.class);
}
@Override
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return nodePlugins();
}
@Override
protected Collection<Class<? extends Plugin>> getMockPlugins() {
Set<Class<? extends Plugin>> mocks = new HashSet<>(super.getMockPlugins());
mocks.remove(NodeMocksPlugin.class);
return mocks;
return Arrays.asList(TestZenDiscovery.TestPlugin.class, TestSeedPlugin.class);
}
@Before
@ -180,19 +187,17 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
public void cleanupWorkaround() throws Exception {
deleteAllDatafeeds(client());
deleteAllJobs(client());
if (ignoreExternalCluster()) {
assertBusy(() -> {
RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries()
.setActiveOnly(true)
.get();
for (List<RecoveryState> recoveryStates : recoveryResponse.shardRecoveryStates().values()) {
assertThat(recoveryStates.size(), equalTo(0));
}
});
}
assertBusy(() -> {
RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries()
.setActiveOnly(true)
.get();
for (List<RecoveryState> recoveryStates : recoveryResponse.shardRecoveryStates().values()) {
assertThat(recoveryStates.size(), equalTo(0));
}
});
}
protected void indexDocs(String index, long numDocs, long start, long end) {
public static void indexDocs(Logger logger, String index, long numDocs, long start, long end) {
int maxDelta = (int) (end - start - 1);
BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
for (int i = 0; i < numDocs; i++) {
@ -209,7 +214,7 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
logger.info("Indexed [{}] documents", numDocs);
}
protected DataCounts getDataCounts(String jobId) {
public static DataCounts getDataCounts(String jobId) {
GetJobsStatsAction.Request request = new GetJobsStatsAction.Request(jobId);
GetJobsStatsAction.Response response = client().execute(GetJobsStatsAction.INSTANCE, request).actionGet();
if (response.getResponse().results().isEmpty()) {
@ -219,7 +224,7 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
}
}
private void deleteAllDatafeeds(Client client) throws Exception {
public static void deleteAllDatafeeds(Client client) throws Exception {
MetaData metaData = client.admin().cluster().prepareState().get().getState().getMetaData();
MlMetadata mlMetadata = metaData.custom(MlMetadata.TYPE);
for (DatafeedConfig datafeed : mlMetadata.getDatafeeds().values()) {
@ -249,7 +254,7 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
}
}
private void deleteAllJobs(Client client) throws Exception {
public static void deleteAllJobs(Client client) throws Exception {
MetaData metaData = client.admin().cluster().prepareState().get().getState().getMetaData();
MlMetadata mlMetadata = metaData.custom(MlMetadata.TYPE);
for (Map.Entry<String, Job> entry : mlMetadata.getJobs().entrySet()) {