honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] Fix projects that failed to build within Intellij (#62258) (#62408) 

This commit address some build failures from the perspective of Intellij.
These changes include:
* changing an order of a dependency definition that seems to can cause Intellij build to fail.
* introduction of an abstract class out of the test source set (seems to be an issue sharing 
  classes cross projects with non-standard source sets. 
* a couple of missing dependency definitions (not sure how the command line worked prior to this)
This commit is contained in:
Jake Landis 2020-09-17 17:45:12 -05:00 committed by GitHub
parent b764f8977e
commit 5b7246157f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 331 additions and 288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java
View File

@ -155,7 +155,7 @@ public abstract class GradleUtils {
project.getPluginManager().withPlugin("idea", p -> {
IdeaModel idea = project.getExtensions().getByType(IdeaModel.class);
idea.getModule().setTestSourceDirs(testSourceSet.getJava().getSrcDirs());
idea.getModule().getScopes().put("TEST", Map.of("plus", List.of(runtimeClasspathConfiguration)));
idea.getModule().getScopes().put(testSourceSet.getName(), Map.of("plus", List.of(runtimeClasspathConfiguration)));
});
project.getPluginManager().withPlugin("eclipse", p -> {
EclipseModel eclipse = project.getExtensions().getByType(EclipseModel.class);
@ -196,6 +196,19 @@ public abstract class GradleUtils {
child.setRuntimeClasspath(project.getObjects().fileCollection().from(child.getRuntimeClasspath(), parent.getOutput()));
}
/**
* Extends one configuration from another and refreshes the classpath of a provided Test.
* The Test parameter is only needed for eagerly defined test tasks.
*/
public static void extendSourceSet(Project project, String parentSourceSetName, String childSourceSetName, Test test) {
extendSourceSet(project, parentSourceSetName, childSourceSetName);
if (test != null) {
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
SourceSet child = sourceSets.getByName(childSourceSetName);
test.setClasspath(child.getRuntimeClasspath());
}
}
public static Dependency projectDependency(Project project, String projectPath, String projectConfig) {
if (project.findProject(projectPath) == null) {
throw new GradleException("no project [" + projectPath + "], project names: " + project.getRootProject().getAllprojects());

5
qa/die-with-dignity/build.gradle
View File

@ -1,4 +1,5 @@
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.util.GradleUtils
apply plugin: 'elasticsearch.java-rest-test'
apply plugin: 'elasticsearch.esplugin'
@ -8,10 +9,8 @@ esplugin {
classname 'org.elasticsearch.DieWithDignityPlugin'
}
dependencies {
// let the javaRestTest see the classpath of main
javaRestTestImplementation project.sourceSets.main.runtimeClasspath
}
GradleUtils.extendSourceSet(project, "main", "javaRestTest", javaRestTest)
javaRestTest {
systemProperty 'tests.security.manager', 'false'

4
x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java
View File

@ -17,7 +17,7 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ClientYamlTestClient;
import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
import org.elasticsearch.xpack.test.rest.XPackRestIT;
import org.elasticsearch.xpack.test.rest.AbstractXPackRestTest;
import org.junit.After;
import java.util.List;
@ -29,7 +29,7 @@ import static java.util.Collections.singletonMap;
import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
import static org.hamcrest.Matchers.is;
public class XDocsClientYamlTestSuiteIT extends XPackRestIT {
public class XDocsClientYamlTestSuiteIT extends AbstractXPackRestTest {
private static final String USER_TOKEN = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
public XDocsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

15
x-pack/plugin/build.gradle
View File

@ -12,9 +12,11 @@ apply plugin: 'elasticsearch.validate-rest-spec'
archivesBaseName = 'x-pack'
dependencies {
yamlRestTestImplementation project(xpackModule('core')) // this redundant dependency is here to make IntelliJ happy
yamlRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
testImplementation project(xpackModule('core'))
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
// let the yamlRestTest see the classpath of test
yamlRestTestImplementation project.sourceSets.test.runtimeClasspath
}
subprojects {
@ -61,8 +63,6 @@ subprojects {
configurations {
testArtifacts.extendsFrom testRuntime
testArtifacts.extendsFrom testImplementation
testArtifacts.extendsFrom yamlRestTestImplementation
testArtifacts.extendsFrom javaRestTestImplementation
}
restResources {
@ -70,9 +70,6 @@ restResources {
includeCore '*'
includeXpack '*'
}
restTests {
includeXpack '*'
}
}
//The api and tests need to stay at src/test/... since some external tooling depends on that exact file path.
@ -83,10 +80,8 @@ artifacts {
def testJar = tasks.register("testJar", Jar) {
appendix 'test'
duplicatesStrategy = DuplicatesStrategy.INCLUDE
from sourceSets.test.output
from sourceSets.yamlRestTest.output
from sourceSets.javaRestTest.output
/*
* Stick the license and notice file in the jar. This isn't strictly
* needed because we don't publish it but it makes our super-paranoid

8
x-pack/plugin/deprecation/qa/rest/build.gradle
View File

@ -1,3 +1,5 @@
import org.elasticsearch.gradle.util.GradleUtils
apply plugin: 'elasticsearch.esplugin'
apply plugin: 'elasticsearch.java-rest-test'
@ -9,10 +11,12 @@ esplugin {
dependencies {
javaRestTestImplementation("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
javaRestTestImplementation("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
// let the javaRestTest see the classpath of main
javaRestTestImplementation project.sourceSets.main.runtimeClasspath
}
// let the javaRestTest see the classpath of main
GradleUtils.extendSourceSet(project, "main", "javaRestTest", javaRestTest)
restResources {
restApi {
includeCore '_common', 'indices', 'index'

6
x-pack/plugin/ilm/qa/multi-node/build.gradle
View File

@ -1,9 +1,15 @@
import org.elasticsearch.gradle.util.GradleUtils
apply plugin: 'elasticsearch.java-rest-test'
dependencies {
javaRestTestImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
}
// let the javaRestTest see the classpath of main
GradleUtils.extendSourceSet(project, "main", "javaRestTest", javaRestTest)
File repoDir = file("$buildDir/testclusters/repo")
javaRestTest {

1
x-pack/plugin/ilm/qa/with-security/build.gradle
View File

@ -1,6 +1,7 @@
apply plugin: 'elasticsearch.java-rest-test'
dependencies {
javaRestTestImplementation project(path: xpackModule('core'))
javaRestTestImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
testImplementation project(":client:rest-high-level")
}

2
x-pack/plugin/ml/qa/ml-with-security/build.gradle
View File

@ -1,8 +1,8 @@
apply plugin: 'elasticsearch.yaml-rest-test'
dependencies {
yamlRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
yamlRestTestImplementation project(path: xpackModule('core'))
yamlRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
yamlRestTestImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
}

4
x-pack/plugin/ml/qa/ml-with-security/src/yamlRestTest/java/org/elasticsearch/smoketest/MlWithSecurityIT.java
View File

@ -11,7 +11,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.test.SecuritySettingsSourceField;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.xpack.test.rest.XPackRestIT;
import org.elasticsearch.xpack.test.rest.AbstractXPackRestTest;
import java.util.Collections;
import java.util.Map;
@ -19,7 +19,7 @@ import java.util.Map;
import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
public class MlWithSecurityIT extends XPackRestIT {
public class MlWithSecurityIT extends AbstractXPackRestTest {
private static final String TEST_ADMIN_USERNAME = "x_pack_rest_user";

1
x-pack/plugin/runtime-fields/qa/with-security/build.gradle
View File

@ -1,6 +1,7 @@
apply plugin: 'elasticsearch.java-rest-test'
dependencies {
javaRestTestImplementation project(path: xpackModule('core'))
javaRestTestImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
}

285
x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/AbstractXPackRestTest.java Normal file
View File

@ -0,0 +1,285 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.test.rest;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import org.apache.http.HttpStatus;
import org.apache.lucene.util.TimeUnits;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.plugins.MetadataUpgrader;
import org.elasticsearch.test.SecuritySettingsSourceField;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import org.elasticsearch.xpack.core.ml.MlConfigIndex;
import org.elasticsearch.xpack.core.ml.MlMetaIndex;
import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.rollup.job.RollupJob;
import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
/** Runs rest tests against external cluster */
// TODO: Remove this timeout increase once this test suite is broken up
@TimeoutSuite(millis = 60 * TimeUnits.MINUTE)
public class AbstractXPackRestTest extends ESClientYamlSuiteTestCase {
private static final String BASIC_AUTH_VALUE =
basicAuthHeaderValue("x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
public AbstractXPackRestTest(ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return createParameters();
}
@Override
protected Settings restClientSettings() {
return Settings.builder()
.put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE)
.build();
}
@Before
public void setupForTests() throws Exception {
waitForTemplates();
enableMonitoring();
}
/**
* Waits for Machine Learning and Transform templates to be created by the {@link MetadataUpgrader}
*/
private void waitForTemplates() throws Exception {
if (installTemplates()) {
List<String> templates = new ArrayList<>();
templates.addAll(
Arrays.asList(
NotificationsIndex.NOTIFICATIONS_INDEX,
MlMetaIndex.indexName(),
AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
AnomalyDetectorsIndex.jobResultsIndexPrefix(),
MlConfigIndex.indexName(),
TransformInternalIndexConstants.AUDIT_INDEX,
TransformInternalIndexConstants.LATEST_INDEX_NAME
));
for (String template : templates) {
awaitCallApi("indices.exists_template", singletonMap("name", template), emptyList(),
response -> true,
() -> "Exception when waiting for [" + template + "] template to be created");
}
}
}
/**
* Enable monitoring and waits for monitoring documents to be collected and indexed in
* monitoring indices.This is the signal that the local exporter is started and ready
* for the tests.
*/
private void enableMonitoring() throws Exception {
if (isMonitoringTest()) {
final ClientYamlTestResponse xpackUsage =
callApi("xpack.usage", singletonMap("filter_path", "monitoring.enabled_exporters"), emptyList(), getApiCallHeaders());
@SuppressWarnings("unchecked")
final Map<String, Object> exporters = (Map<String, Object>) xpackUsage.evaluate("monitoring.enabled_exporters");
assertNotNull("List of monitoring exporters must not be null", exporters);
assertThat("List of enabled exporters must be empty before enabling monitoring",
XContentMapValues.extractRawValues("monitoring.enabled_exporters", exporters), hasSize(0));
final Map<String, Object> settings = new HashMap<>();
settings.put("xpack.monitoring.collection.enabled", true);
settings.put("xpack.monitoring.collection.interval", "1s");
settings.put("xpack.monitoring.exporters._local.type", "local");
settings.put("xpack.monitoring.exporters._local.enabled", true);
awaitCallApi("cluster.put_settings", emptyMap(),
singletonList(singletonMap("transient", settings)),
response -> {
Object acknowledged = response.evaluate("acknowledged");
return acknowledged != null && (Boolean) acknowledged;
},
() -> "Exception when enabling monitoring");
Map<String, String> searchParams = new HashMap<>();
searchParams.put("index", ".monitoring-*");
searchParams.put(TOTAL_HITS_AS_INT_PARAM, "true");
awaitCallApi("search", searchParams, emptyList(),
response -> ((Number) response.evaluate("hits.total")).intValue() > 0,
() -> "Exception when waiting for monitoring documents to be indexed");
}
}
/**
* Disable monitoring
*/
private void disableMonitoring() throws Exception {
if (isMonitoringTest()) {
final Map<String, Object> settings = new HashMap<>();
settings.put("xpack.monitoring.collection.enabled", null);
settings.put("xpack.monitoring.collection.interval", null);
settings.put("xpack.monitoring.exporters._local.enabled", null);
awaitCallApi("cluster.put_settings", emptyMap(),
singletonList(singletonMap("transient", settings)),
response -> {
Object acknowledged = response.evaluate("acknowledged");
return acknowledged != null && (Boolean) acknowledged;
},
() -> "Exception when disabling monitoring");
assertBusy(() -> {
try {
ClientYamlTestResponse response =
callApi("xpack.usage", singletonMap("filter_path", "monitoring.enabled_exporters"), emptyList(),
getApiCallHeaders());
@SuppressWarnings("unchecked")
final Map<String, ?> exporters = (Map<String, ?>) response.evaluate("monitoring.enabled_exporters");
if (exporters.isEmpty() == false) {
fail("Exporters were not found");
}
final Map<String, String> params = new HashMap<>();
params.put("node_id", "_local");
params.put("metric", "thread_pool");
params.put("filter_path", "nodes.*.thread_pool.write.active");
response = callApi("nodes.stats", params, emptyList(), getApiCallHeaders());
@SuppressWarnings("unchecked")
final Map<String, Object> nodes = (Map<String, Object>) response.evaluate("nodes");
@SuppressWarnings("unchecked")
final Map<String, Object> node = (Map<String, Object>) nodes.values().iterator().next();
final Number activeWrites = (Number) extractValue("thread_pool.write.active", node);
assertNotNull(activeWrites);
assertThat(activeWrites, equalTo(0));
} catch (Exception e) {
throw new ElasticsearchException("Failed to wait for monitoring exporters to stop:", e);
}
});
}
}
/**
* Cleanup after tests.
*
* Feature-specific cleanup methods should be called from here rather than using
* separate @After annotated methods to ensure there is a well-defined cleanup order.
*/
@After
public void cleanup() throws Exception {
disableMonitoring();
clearMlState();
if (isWaitForPendingTasks()) {
// This waits for pending tasks to complete, so must go last (otherwise
// it could be waiting for pending tasks while monitoring is still running).
waitForPendingTasks(adminClient(), task -> {
// Don't check rollup jobs because we clear them in the superclass.
return task.contains(RollupJob.NAME);
});
}
}
/**
* Delete any left over machine learning datafeeds and jobs.
*/
private void clearMlState() throws Exception {
if (isMachineLearningTest()) {
new MlRestTestStateCleaner(logger, adminClient()).clearMlMetadata();
}
}
/**
* Executes an API call using the admin context, waiting for it to succeed.
*/
private void awaitCallApi(String apiName,
Map<String, String> params,
List<Map<String, Object>> bodies,
CheckedFunction<ClientYamlTestResponse, Boolean, IOException> success,
Supplier<String> error) {
try {
final AtomicReference<ClientYamlTestResponse> response = new AtomicReference<>();
assertBusy(() -> {
// The actual method call that sends the API requests returns a Future, but we immediately
// call .get() on it so there's no need for this method to do any other awaiting.
response.set(callApi(apiName, params, bodies, getApiCallHeaders()));
assertEquals(HttpStatus.SC_OK, response.get().getStatusCode());
});
success.apply(response.get());
} catch (Exception e) {
throw new IllegalStateException(error.get(), e);
}
}
private ClientYamlTestResponse callApi(String apiName,
Map<String, String> params,
List<Map<String, Object>> bodies,
Map<String, String> headers) throws IOException {
return getAdminExecutionContext().callApi(apiName, params, bodies, headers);
}
protected Map<String, String> getApiCallHeaders() {
return Collections.emptyMap();
}
protected boolean installTemplates() {
return true;
}
protected boolean isMonitoringTest() {
String testName = getTestName();
return testName != null && (testName.contains("=monitoring/") || testName.contains("=monitoring\\"));
}
protected boolean isMachineLearningTest() {
String testName = getTestName();
return testName != null && (testName.contains("=ml/") || testName.contains("=ml\\"));
}
/**
* Should each test wait for pending tasks to finish after execution?
* @return Wait for pending tasks
*/
protected boolean isWaitForPendingTasks() {
return true;
}
}

266
x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
View File

@ -3,61 +3,13 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.test.rest;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import org.apache.http.HttpStatus;
import org.apache.lucene.util.TimeUnits;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.plugins.MetadataUpgrader;
import org.elasticsearch.test.SecuritySettingsSourceField;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import org.elasticsearch.xpack.core.ml.MlConfigIndex;
import org.elasticsearch.xpack.core.ml.MlMetaIndex;
import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.rollup.job.RollupJob;
import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
/** Runs rest tests against external cluster */
// TODO: Remove this timeout increase once this test suite is broken up
@TimeoutSuite(millis = 60 * TimeUnits.MINUTE)
public class XPackRestIT extends ESClientYamlSuiteTestCase {
static final String BASIC_AUTH_VALUE =
basicAuthHeaderValue("x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
public class XPackRestIT extends AbstractXPackRestTest {
public XPackRestIT(ClientYamlTestCandidate testCandidate) {
super(testCandidate);
@ -67,219 +19,5 @@ public class XPackRestIT extends ESClientYamlSuiteTestCase {
public static Iterable<Object[]> parameters() throws Exception {
return createParameters();
}
@Override
protected Settings restClientSettings() {
return Settings.builder()
.put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE)
.build();
}
@Before
public void setupForTests() throws Exception {
waitForTemplates();
enableMonitoring();
}
/**
* Waits for Machine Learning and Transform templates to be created by the {@link MetadataUpgrader}
*/
private void waitForTemplates() throws Exception {
if (installTemplates()) {
List<String> templates = new ArrayList<>();
templates.addAll(
Arrays.asList(
NotificationsIndex.NOTIFICATIONS_INDEX,
MlMetaIndex.indexName(),
AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
AnomalyDetectorsIndex.jobResultsIndexPrefix(),
MlConfigIndex.indexName(),
TransformInternalIndexConstants.AUDIT_INDEX,
TransformInternalIndexConstants.LATEST_INDEX_NAME
));
for (String template : templates) {
awaitCallApi("indices.exists_template", singletonMap("name", template), emptyList(),
response -> true,
() -> "Exception when waiting for [" + template + "] template to be created");
}
}
}
/**
* Enable monitoring and waits for monitoring documents to be collected and indexed in
* monitoring indices.This is the signal that the local exporter is started and ready
* for the tests.
*/
private void enableMonitoring() throws Exception {
if (isMonitoringTest()) {
final ClientYamlTestResponse xpackUsage =
callApi("xpack.usage", singletonMap("filter_path", "monitoring.enabled_exporters"), emptyList(), getApiCallHeaders());
@SuppressWarnings("unchecked")
final Map<String, Object> exporters = (Map<String, Object>) xpackUsage.evaluate("monitoring.enabled_exporters");
assertNotNull("List of monitoring exporters must not be null", exporters);
assertThat("List of enabled exporters must be empty before enabling monitoring",
XContentMapValues.extractRawValues("monitoring.enabled_exporters", exporters), hasSize(0));
final Map<String, Object> settings = new HashMap<>();
settings.put("xpack.monitoring.collection.enabled", true);
settings.put("xpack.monitoring.collection.interval", "1s");
settings.put("xpack.monitoring.exporters._local.type", "local");
settings.put("xpack.monitoring.exporters._local.enabled", true);
awaitCallApi("cluster.put_settings", emptyMap(),
singletonList(singletonMap("transient", settings)),
response -> {
Object acknowledged = response.evaluate("acknowledged");
return acknowledged != null && (Boolean) acknowledged;
},
() -> "Exception when enabling monitoring");
Map<String, String> searchParams = new HashMap<>();
searchParams.put("index", ".monitoring-*");
searchParams.put(TOTAL_HITS_AS_INT_PARAM, "true");
awaitCallApi("search", searchParams, emptyList(),
response -> ((Number) response.evaluate("hits.total")).intValue() > 0,
() -> "Exception when waiting for monitoring documents to be indexed");
}
}
/**
* Disable monitoring
*/
private void disableMonitoring() throws Exception {
if (isMonitoringTest()) {
final Map<String, Object> settings = new HashMap<>();
settings.put("xpack.monitoring.collection.enabled", null);
settings.put("xpack.monitoring.collection.interval", null);
settings.put("xpack.monitoring.exporters._local.enabled", null);
awaitCallApi("cluster.put_settings", emptyMap(),
singletonList(singletonMap("transient", settings)),
response -> {
Object acknowledged = response.evaluate("acknowledged");
return acknowledged != null && (Boolean) acknowledged;
},
() -> "Exception when disabling monitoring");
assertBusy(() -> {
try {
ClientYamlTestResponse response =
callApi("xpack.usage", singletonMap("filter_path", "monitoring.enabled_exporters"), emptyList(),
getApiCallHeaders());
@SuppressWarnings("unchecked")
final Map<String, ?> exporters = (Map<String, ?>) response.evaluate("monitoring.enabled_exporters");
if (exporters.isEmpty() == false) {
fail("Exporters were not found");
}
final Map<String, String> params = new HashMap<>();
params.put("node_id", "_local");
params.put("metric", "thread_pool");
params.put("filter_path", "nodes.*.thread_pool.write.active");
response = callApi("nodes.stats", params, emptyList(), getApiCallHeaders());
@SuppressWarnings("unchecked")
final Map<String, Object> nodes = (Map<String, Object>) response.evaluate("nodes");
@SuppressWarnings("unchecked")
final Map<String, Object> node = (Map<String, Object>) nodes.values().iterator().next();
final Number activeWrites = (Number) extractValue("thread_pool.write.active", node);
assertNotNull(activeWrites);
assertThat(activeWrites, equalTo(0));
} catch (Exception e) {
throw new ElasticsearchException("Failed to wait for monitoring exporters to stop:", e);
}
});
}
}
/**
* Cleanup after tests.
*
* Feature-specific cleanup methods should be called from here rather than using
* separate @After annotated methods to ensure there is a well-defined cleanup order.
*/
@After
public void cleanup() throws Exception {
disableMonitoring();
clearMlState();
if (isWaitForPendingTasks()) {
// This waits for pending tasks to complete, so must go last (otherwise
// it could be waiting for pending tasks while monitoring is still running).
waitForPendingTasks(adminClient(), task -> {
// Don't check rollup jobs because we clear them in the superclass.
return task.contains(RollupJob.NAME);
});
}
}
/**
* Delete any left over machine learning datafeeds and jobs.
*/
private void clearMlState() throws Exception {
if (isMachineLearningTest()) {
new MlRestTestStateCleaner(logger, adminClient()).clearMlMetadata();
}
}
/**
* Executes an API call using the admin context, waiting for it to succeed.
*/
private void awaitCallApi(String apiName,
Map<String, String> params,
List<Map<String, Object>> bodies,
CheckedFunction<ClientYamlTestResponse, Boolean, IOException> success,
Supplier<String> error) {
try {
final AtomicReference<ClientYamlTestResponse> response = new AtomicReference<>();
assertBusy(() -> {
// The actual method call that sends the API requests returns a Future, but we immediately
// call .get() on it so there's no need for this method to do any other awaiting.
response.set(callApi(apiName, params, bodies, getApiCallHeaders()));
assertEquals(HttpStatus.SC_OK, response.get().getStatusCode());
});
success.apply(response.get());
} catch (Exception e) {
throw new IllegalStateException(error.get(), e);
}
}
private ClientYamlTestResponse callApi(String apiName,
Map<String, String> params,
List<Map<String, Object>> bodies,
Map<String, String> headers) throws IOException {
return getAdminExecutionContext().callApi(apiName, params, bodies, headers);
}
protected Map<String, String> getApiCallHeaders() {
return Collections.emptyMap();
}
protected boolean installTemplates() {
return true;
}
protected boolean isMonitoringTest() {
String testName = getTestName();
return testName != null && (testName.contains("=monitoring/") || testName.contains("=monitoring\\"));
}
protected boolean isMachineLearningTest() {
String testName = getTestName();
return testName != null && (testName.contains("=ml/") || testName.contains("=ml\\"));
}
/**
* Should each test wait for pending tasks to finish after execution?
* @return Wait for pending tasks
*/
protected boolean isWaitForPendingTasks() {
return true;
}
}

3
x-pack/qa/security-example-spi-extension/build.gradle
View File

@ -1,3 +1,5 @@
import org.elasticsearch.gradle.util.GradleUtils
apply plugin: 'elasticsearch.java-rest-test'
apply plugin: 'elasticsearch.esplugin'
@ -18,7 +20,6 @@ dependencies {
}
javaRestTest {
systemProperty 'tests.security.manager', 'false'
}