[TEST] Changed internal distributed ml tests to extend from ESIntegTest and only run with ml to avoid noise.
Changed DatafeedJobsIT to extend directly from SecurityIntegTest as it uses external cluster. Original commit: elastic/x-pack-elasticsearch@d646c59491
This commit is contained in:
Martijn van Groningen
parent
49826ae134
commit
6365dec42e
|
|
@ -8,14 +8,12 @@ package org.elasticsearch.license;
|
|||
import org.elasticsearch.ElasticsearchSecurityException;
|
||||
import org.elasticsearch.action.support.PlainListenableActionFuture;
|
||||
import org.elasticsearch.client.transport.TransportClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.license.License.OperationMode;
|
||||
import org.elasticsearch.rest.RestStatus;
|
||||
import org.elasticsearch.transport.Transport;
|
||||
import org.elasticsearch.xpack.TestXPackTransportClient;
|
||||
import org.elasticsearch.xpack.XPackPlugin;
|
||||
import org.elasticsearch.xpack.XPackSettings;
|
||||
import org.elasticsearch.xpack.ml.action.CloseJobAction;
|
||||
import org.elasticsearch.xpack.ml.action.DeleteDatafeedAction;
|
||||
import org.elasticsearch.xpack.ml.action.DeleteJobAction;
|
||||
|
|
@ -37,20 +35,6 @@ import static org.hamcrest.Matchers.is;
|
|||
|
||||
public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
|
||||
|
||||
@Override
|
||||
protected Settings nodeSettings(int nodeOrdinal) {
|
||||
Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal));
|
||||
settings.put(XPackSettings.SECURITY_ENABLED.getKey(), true);
|
||||
return settings.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Settings transportClientSettings() {
|
||||
Settings.Builder settings = Settings.builder().put(super.transportClientSettings());
|
||||
settings.put(XPackSettings.SECURITY_ENABLED.getKey(), true);
|
||||
return settings.build();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void resetLicensing() {
|
||||
enableLicensing();
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import org.elasticsearch.common.network.NetworkModule;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.search.SearchModule;
|
||||
import org.elasticsearch.tasks.Task;
|
||||
import org.elasticsearch.test.ESIntegTestCase;
|
||||
import org.elasticsearch.test.SecurityIntegTestCase;
|
||||
import org.elasticsearch.xpack.XPackSettings;
|
||||
import org.elasticsearch.xpack.ml.MlMetadata;
|
||||
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
|
||||
|
|
@ -24,12 +24,12 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
|
|||
import org.elasticsearch.xpack.ml.job.config.Job;
|
||||
import org.elasticsearch.xpack.ml.job.config.JobState;
|
||||
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
|
||||
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
|
||||
import org.elasticsearch.xpack.persistent.PersistentActionCoordinator;
|
||||
import org.elasticsearch.xpack.persistent.PersistentActionRequest;
|
||||
import org.elasticsearch.xpack.persistent.PersistentActionResponse;
|
||||
import org.elasticsearch.xpack.persistent.PersistentTasks;
|
||||
import org.elasticsearch.xpack.security.Security;
|
||||
import org.junit.After;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -40,16 +40,15 @@ import java.util.concurrent.TimeUnit;
|
|||
|
||||
import static org.elasticsearch.test.XContentTestUtils.convertToMap;
|
||||
import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder;
|
||||
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDatafeed;
|
||||
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob;
|
||||
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.deleteAllDatafeeds;
|
||||
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.deleteAllJobs;
|
||||
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.getDataCounts;
|
||||
import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.indexDocs;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
||||
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0, numClientNodes = 0,
|
||||
transportClientRatio = 0, supportsDedicatedMasters = false)
|
||||
public class DatafeedJobsIT extends BaseMlIntegTestCase {
|
||||
|
||||
@Override
|
||||
protected boolean ignoreExternalCluster() {
|
||||
return false;
|
||||
}
|
||||
public class DatafeedJobsIT extends SecurityIntegTestCase {
|
||||
|
||||
@Override
|
||||
protected Settings externalClusterClientSettings() {
|
||||
|
|
@ -60,6 +59,12 @@ public class DatafeedJobsIT extends BaseMlIntegTestCase {
|
|||
return settings.build();
|
||||
}
|
||||
|
||||
@After
|
||||
public void cleanupWorkaround() throws Exception {
|
||||
deleteAllDatafeeds(client());
|
||||
deleteAllJobs(client());
|
||||
}
|
||||
|
||||
public void ensureClusterStateConsistency() throws IOException {
|
||||
if (cluster() != null && cluster().size() > 0) {
|
||||
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedWriteables());
|
||||
|
|
@ -118,13 +123,13 @@ public class DatafeedJobsIT extends BaseMlIntegTestCase {
|
|||
long now = System.currentTimeMillis();
|
||||
long oneWeekAgo = now - 604800000;
|
||||
long twoWeeksAgo = oneWeekAgo - 604800000;
|
||||
indexDocs("data-1", numDocs, twoWeeksAgo, oneWeekAgo);
|
||||
indexDocs(logger, "data-1", numDocs, twoWeeksAgo, oneWeekAgo);
|
||||
|
||||
client().admin().indices().prepareCreate("data-2")
|
||||
.addMapping("type", "time", "type=date")
|
||||
.get();
|
||||
long numDocs2 = randomIntBetween(32, 2048);
|
||||
indexDocs("data-2", numDocs2, oneWeekAgo, now);
|
||||
indexDocs(logger, "data-2", numDocs2, oneWeekAgo, now);
|
||||
|
||||
Job.Builder job = createScheduledJob("lookback-job");
|
||||
PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build());
|
||||
|
|
@ -163,7 +168,7 @@ public class DatafeedJobsIT extends BaseMlIntegTestCase {
|
|||
long numDocs1 = randomIntBetween(32, 2048);
|
||||
long now = System.currentTimeMillis();
|
||||
long lastWeek = now - 604800000;
|
||||
indexDocs("data", numDocs1, lastWeek, now);
|
||||
indexDocs(logger, "data", numDocs1, lastWeek, now);
|
||||
|
||||
Job.Builder job = createScheduledJob("realtime-job");
|
||||
PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build());
|
||||
|
|
@ -192,7 +197,7 @@ public class DatafeedJobsIT extends BaseMlIntegTestCase {
|
|||
|
||||
long numDocs2 = randomIntBetween(2, 64);
|
||||
now = System.currentTimeMillis();
|
||||
indexDocs("data", numDocs2, now + 5000, now + 6000);
|
||||
indexDocs(logger, "data", numDocs2, now + 5000, now + 6000);
|
||||
assertBusy(() -> {
|
||||
DataCounts dataCounts = getDataCounts(job.getId());
|
||||
assertThat(dataCounts.getProcessedRecordCount(), equalTo(numDocs1 + numDocs2));
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
|
|||
long now = System.currentTimeMillis();
|
||||
long weekAgo = now - 604800000;
|
||||
long twoWeeksAgo = weekAgo - 604800000;
|
||||
indexDocs("data", numDocs1, twoWeeksAgo, weekAgo);
|
||||
indexDocs(logger, "data", numDocs1, twoWeeksAgo, weekAgo);
|
||||
|
||||
Job.Builder job = createScheduledJob("job_id");
|
||||
PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build());
|
||||
|
|
@ -147,7 +147,7 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
|
|||
|
||||
long numDocs2 = randomIntBetween(2, 64);
|
||||
long now2 = System.currentTimeMillis();
|
||||
indexDocs("data", numDocs2, now2 + 5000, now2 + 6000);
|
||||
indexDocs(logger, "data", numDocs2, now2 + 5000, now2 + 6000);
|
||||
assertBusy(() -> {
|
||||
DataCounts dataCounts = getDataCounts(job.getId());
|
||||
assertEquals(numDocs1 + numDocs2, dataCounts.getProcessedRecordCount());
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
package org.elasticsearch.xpack.ml.support;
|
||||
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.ExceptionsHelper;
|
||||
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
|
||||
import org.elasticsearch.action.bulk.BulkRequestBuilder;
|
||||
|
|
@ -16,11 +17,11 @@ import org.elasticsearch.cluster.metadata.MetaData;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.indices.recovery.RecoveryState;
|
||||
import org.elasticsearch.node.NodeMocksPlugin;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.rest.RestStatus;
|
||||
import org.elasticsearch.test.ESIntegTestCase;
|
||||
import org.elasticsearch.test.SecurityIntegTestCase;
|
||||
import org.elasticsearch.test.discovery.TestZenDiscovery;
|
||||
import org.elasticsearch.xpack.XPackPlugin;
|
||||
import org.elasticsearch.xpack.XPackSettings;
|
||||
import org.elasticsearch.xpack.ml.MachineLearning;
|
||||
import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry;
|
||||
|
|
@ -42,15 +43,13 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
|
|||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
|
|
@ -63,7 +62,7 @@ import static org.hamcrest.Matchers.is;
|
|||
*/
|
||||
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0,
|
||||
transportClientRatio = 0, supportsDedicatedMasters = false)
|
||||
public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
|
||||
public abstract class BaseMlIntegTestCase extends ESIntegTestCase {
|
||||
|
||||
@Override
|
||||
protected boolean ignoreExternalCluster() {
|
||||
|
|
@ -93,11 +92,19 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
|
|||
return settings.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> nodePlugins() {
|
||||
return Collections.singleton(XPackPlugin.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
|
||||
return nodePlugins();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> getMockPlugins() {
|
||||
Set<Class<? extends Plugin>> mocks = new HashSet<>(super.getMockPlugins());
|
||||
mocks.remove(NodeMocksPlugin.class);
|
||||
return mocks;
|
||||
return Arrays.asList(TestZenDiscovery.TestPlugin.class, TestSeedPlugin.class);
|
||||
}
|
||||
|
||||
@Before
|
||||
|
|
@ -180,7 +187,6 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
|
|||
public void cleanupWorkaround() throws Exception {
|
||||
deleteAllDatafeeds(client());
|
||||
deleteAllJobs(client());
|
||||
if (ignoreExternalCluster()) {
|
||||
assertBusy(() -> {
|
||||
RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries()
|
||||
.setActiveOnly(true)
|
||||
|
|
@ -190,9 +196,8 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
|
|||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
protected void indexDocs(String index, long numDocs, long start, long end) {
|
||||
public static void indexDocs(Logger logger, String index, long numDocs, long start, long end) {
|
||||
int maxDelta = (int) (end - start - 1);
|
||||
BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
|
||||
for (int i = 0; i < numDocs; i++) {
|
||||
|
|
@ -209,7 +214,7 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
|
|||
logger.info("Indexed [{}] documents", numDocs);
|
||||
}
|
||||
|
||||
protected DataCounts getDataCounts(String jobId) {
|
||||
public static DataCounts getDataCounts(String jobId) {
|
||||
GetJobsStatsAction.Request request = new GetJobsStatsAction.Request(jobId);
|
||||
GetJobsStatsAction.Response response = client().execute(GetJobsStatsAction.INSTANCE, request).actionGet();
|
||||
if (response.getResponse().results().isEmpty()) {
|
||||
|
|
@ -219,7 +224,7 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
private void deleteAllDatafeeds(Client client) throws Exception {
|
||||
public static void deleteAllDatafeeds(Client client) throws Exception {
|
||||
MetaData metaData = client.admin().cluster().prepareState().get().getState().getMetaData();
|
||||
MlMetadata mlMetadata = metaData.custom(MlMetadata.TYPE);
|
||||
for (DatafeedConfig datafeed : mlMetadata.getDatafeeds().values()) {
|
||||
|
|
@ -249,7 +254,7 @@ public abstract class BaseMlIntegTestCase extends SecurityIntegTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
private void deleteAllJobs(Client client) throws Exception {
|
||||
public static void deleteAllJobs(Client client) throws Exception {
|
||||
MetaData metaData = client.admin().cluster().prepareState().get().getState().getMetaData();
|
||||
MlMetadata mlMetadata = metaData.custom(MlMetadata.TYPE);
|
||||
for (Map.Entry<String, Job> entry : mlMetadata.getJobs().entrySet()) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue