OpenSearch/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java
Alexander Reelsen c62f6f8177 Watcher: Distributed watch execution (elastic/x-pack-elasticsearch#544)
The distribution of watches now happens on the node which holds the
watches index, instead of on the master node. This requires several
changes to the current implementation.

1. Running on shards and replicas
   In order to run watches on the nodes with the watches index on its
   primaries and replicas. To ensure that watches do not run twice, there is
   a logic which checks the local shards, runs a murmurhash on the id and
   runs modulo against the number of shards and replicas, this is the way to
   find out, if a watch should run local. Reloading happens
2. Several master node actions moved to a HandledTransportAction, as they
   are basically just aliases for indexing actions, among them the
   put/delete/get watch actions, the acknowledgement action, the de/activate
   actions
3. Stats action moved to a broadcast node action, because we potentially
   have to query every node to get watcher statistics
4. Starting/Stopping watcher now is a master node action, which updates
   the cluster state and then listeners acts on those. Because of this watches
   can be running on two systems, if you those have different cluster state
   versions, until the new watcher state is propagated
5. Watcher is started on all nodes now. With the exception of the ticker
   schedule engine most classes do not need a lot of resources while running.
   However they have to run, because of the execute watch API, which can hit
   any node - it does not make sense to find the right shard for this watch
   and only then execute (as this also has to work with a watch, that has not
   been stored before)
6. By using a indexing operation listener, each storing of a watch now
   parses the watch first and only stores on successful parsing
7. Execute watch API now uses the watcher threadpool for execution
8. Getting the number of watches for the stats now simply queries the
   different execution engines, how many watches are scheduled, so this is
   not doing a search anymore

There will be follow up commits on this one, mainly to ensure BWC compatibility.

Original commit: elastic/x-pack-elasticsearch@0adb46e658
2017-05-02 10:12:46 +02:00

92 lines
3.7 KiB
Java

/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.smoketest;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import org.junit.After;
import java.util.List;
import java.util.Map;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
public class XDocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
private static final String USER_TOKEN = basicAuthHeaderValue("test_admin", new SecureString("changeme".toCharArray()));
public XDocsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return ESClientYamlSuiteTestCase.createParameters();
}
@Override
protected void afterIfFailed(List<Throwable> errors) {
super.afterIfFailed(errors);
String name = getTestName().split("=")[1];
name = name.substring(0, name.length() - 1);
name = name.replaceAll("/([^/]+)$", ".asciidoc:$1");
logger.error("This failing test was generated by documentation starting at {}. It may include many snippets. "
+ "See Elasticsearch's docs/README.asciidoc for an explanation of test generation.", name);
}
@Override
protected boolean preserveTemplatesUponCompletion() {
return true;
}
/**
* All tests run as a an administrative user but use <code>es-shield-runas-user</code> to become a less privileged user.
*/
@Override
protected Settings restClientSettings() {
return Settings.builder()
.put(ThreadContext.PREFIX + ".Authorization", USER_TOKEN)
.build();
}
/**
* Re-enables watcher after every test just in case any test disables it. One does.
*/
@After
public void reenableWatcher() throws Exception {
getAdminExecutionContext().callApi("xpack.watcher.start", emptyMap(), emptyList(), emptyMap());
}
/**
* Deletes users after every test just in case any test adds any.
*/
@After
public void deleteUsers() throws Exception {
ClientYamlTestResponse response = getAdminExecutionContext().callApi("xpack.security.get_user", emptyMap(), emptyList(),
emptyMap());
@SuppressWarnings("unchecked")
Map<String, Object> users = (Map<String, Object>) response.getBody();
for (String user: users.keySet()) {
Map<?, ?> metaDataMap = (Map<?, ?>) ((Map<?, ?>) users.get(user)).get("metadata");
Boolean reserved = metaDataMap == null ? null : (Boolean) metaDataMap.get("_reserved");
if (reserved == null || reserved == false) {
logger.warn("Deleting leftover user {}", user);
getAdminExecutionContext().callApi("xpack.security.delete_user", singletonMap("username", user), emptyList(), emptyMap());
}
}
}
}