honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] [ML] Fix log spam and disable ILM/SLM history for native ML tests (#55475) 

* [ML] fix native ML test log spam (#55459)

This adds a dependency to ingest common. This removes the log spam resulting from basic plugins being enabled that require the common ingest processors.

* removing unnecessary changes

* removing unused imports

* removing unnecessary java setting
This commit is contained in:
Benjamin Trent 2020-04-20 15:41:30 -04:00 committed by GitHub
parent 8e618fdf10
commit fa0373a19f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 78 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
x-pack/plugin/ml/qa/basic-multi-node/build.gradle
View File

@ -15,4 +15,6 @@ testClusters.integTest {
setting 'xpack.watcher.enabled', 'false'
setting 'xpack.ml.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
setting 'indices.lifecycle.history_index_enabled', 'false'
setting 'slm.history_index_enabled', 'false'
}

3
x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
View File

@ -7,6 +7,7 @@ dependencies {
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
testCompile project(path: xpackModule('ml'), configuration: 'runtime')
testCompile project(path: xpackModule('ml'), configuration: 'testArtifacts')
testCompile project(path: ':modules:ingest-common')
}
// location for keys and certificates
@ -51,6 +52,8 @@ testClusters.integTest {
setting 'xpack.security.audit.enabled', 'false'
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.ml.min_disk_space_off_heap', '200mb'
setting 'indices.lifecycle.history_index_enabled', 'false'
setting 'slm.history_index_enabled', 'false'
keystore 'bootstrap.password', 'x-pack-test-password'
keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'

78
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
View File

@ -18,9 +18,16 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.index.reindex.ReindexPlugin;
import org.elasticsearch.ingest.common.IngestCommonPlugin;
import org.elasticsearch.persistent.PersistentTaskParams;
import org.elasticsearch.persistent.PersistentTaskState;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.IngestScript;
import org.elasticsearch.script.MockScriptEngine;
import org.elasticsearch.script.MockScriptPlugin;
import org.elasticsearch.script.ScoreScript;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptEngine;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.SecuritySettingsSourceField;
@ -35,10 +42,10 @@ import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
import org.elasticsearch.xpack.core.ilm.LifecycleType;
import org.elasticsearch.xpack.core.ilm.RolloverAction;
import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
import org.elasticsearch.xpack.core.ml.MlMetaIndex;
import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.MlTasks;
import org.elasticsearch.xpack.core.ml.action.DeleteExpiredDataAction;
import org.elasticsearch.xpack.core.ml.action.GetFiltersAction;
import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
import org.elasticsearch.xpack.core.ml.action.PutFilterAction;
import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction;
@ -46,10 +53,15 @@ import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
import org.elasticsearch.xpack.core.ml.job.config.MlFilter;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.security.SecurityField;
import org.elasticsearch.xpack.core.security.authc.TokenMetadata;
import org.elasticsearch.xpack.core.slm.history.SnapshotLifecycleTemplateRegistry;
import org.elasticsearch.xpack.ilm.IndexLifecycle;
import java.io.IOException;
@ -59,8 +71,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import static org.elasticsearch.test.XContentTestUtils.convertToMap;
import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder;
@ -82,6 +97,11 @@ abstract class MlNativeIntegTestCase extends ESIntegTestCase {
return Arrays.asList(
LocalStateCompositeXPackPlugin.class,
Netty4Plugin.class,
// The monitoring plugin requires script and gsub processors to be loaded
IngestCommonPlugin.class,
// The monitoring plugin script processor references painless. Include this for script compilation.
// This is to reduce log spam
MockPainlessScriptEngine.TestPlugin.class,
// ILM is required for .ml-state template index settings
IndexLifecycle.class);
}
@ -110,7 +130,9 @@ abstract class MlNativeIntegTestCase extends ESIntegTestCase {
builder.put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NAME4);
builder.put(SecurityField.USER_SETTING.getKey(), "x_pack_rest_user:" + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true);
builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
builder.put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
builder.put(LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
builder.put("xpack.security.transport.ssl.enabled", true);
builder.put("xpack.security.transport.ssl.key", key.toAbsolutePath().toString());
builder.put("xpack.security.transport.ssl.certificate", certificate.toAbsolutePath().toString());
@ -125,6 +147,18 @@ abstract class MlNativeIntegTestCase extends ESIntegTestCase {
waitForPendingTasks();
}
@Override
protected Set<String> excludeTemplates() {
return new HashSet<>(Arrays.asList(
NotificationsIndex.NOTIFICATIONS_INDEX,
MlMetaIndex.INDEX_NAME,
AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
AnomalyDetectorsIndex.jobResultsIndexPrefix(),
InferenceIndexConstants.LATEST_INDEX_NAME,
SnapshotLifecycleTemplateRegistry.SLM_TEMPLATE_NAME
));
}
protected abstract void cleanUpResources();
private void waitForPendingTasks() {
@ -166,10 +200,6 @@ abstract class MlNativeIntegTestCase extends ESIntegTestCase {
return client().execute(PutFilterAction.INSTANCE, new PutFilterAction.Request(filter)).actionGet();
}
protected GetFiltersAction.Response getMlFilters() {
return client().execute(GetFiltersAction.INSTANCE, new GetFiltersAction.Request()).actionGet();
}
@Override
protected void ensureClusterStateConsistency() throws IOException {
if (cluster() != null && cluster().size() > 0) {
@ -228,4 +258,42 @@ abstract class MlNativeIntegTestCase extends ESIntegTestCase {
}
}
}
public static class MockPainlessScriptEngine extends MockScriptEngine {
public static final String NAME = "painless";
public static class TestPlugin extends MockScriptPlugin {
@Override
public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
return new MockPainlessScriptEngine();
}
@Override
protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
return Collections.emptyMap();
}
}
@Override
public String getType() {
return NAME;
}
@Override
public <T> T compile(String name, String script, ScriptContext<T> context, Map<String, String> options) {
if (context.instanceClazz.equals(ScoreScript.class)) {
return context.factoryClazz.cast(new MockScoreScript(p -> 0.0));
}
if (context.name.equals("ingest")) {
IngestScript.Factory factory = vars -> new IngestScript(vars) {
@Override
public void execute(Map<String, Object> ctx) {
}
};
return context.factoryClazz.cast(factory);
}
throw new IllegalArgumentException("mock painless does not know how to handle context [" + context.name + "]");
}
}
}