Merge branch 'master' into fix/remove-license

Original commit: elastic/x-pack-elasticsearch@1e84c8431d
This commit is contained in:
Areek Zillur 2016-05-18 16:16:53 -04:00
commit 703dfda921
21 changed files with 295 additions and 98 deletions

View File

@ -1,9 +0,0 @@
As a quick helper, below are the equivalent commands from maven to gradle. You can also run `gradle tasks` to see all tasks that are available to run.
| Maven | Gradle | Description |
| ----------------------------| ------------|---------------------|
| `clean` | `clean` | |
| `verify` | `check` | |
| `verify -Dskip.unit.tests` | `integTest` | |
| `package -DskipTests` | `assemble` | |
| `install -DskipTests` | `install` | |

View File

@ -0,0 +1,61 @@
To compile `x-plugins`, you must clone the Elasticsearch repository into the same parent directory. For example:
[source,bash]
----
$ mkdir elastic
$ cd elastic
$ git clone git@github.com:elastic/elasticsearch.git
$ git clone git@github.com:elastic/x-plugins.git
$ git clone git@github.com:elastic/kibana.git <1>
----
<1> For anyone doing UI development, it's also useful to have Kibana at the same level.
Once cloned, any command should be executed from the **elasticsearch** directory. This ensures that the full dependency tree is available.
[source,bash]
----
$ cd elasticsearch
$ gradle clean test check <1>
----
<1> This will run the `clean` task, `test` task, and then the `check` task on _every_ project that has it. However, `check` requires that `test` be run, so it won't _rerun_ `test`. `clean` is unnecessary here, but people often use it anyway.
If this command were run in a different order, then it would still follow the same rules, but the behavior would change:
[source,bash]
----
$ gradle check test clean <1>
----
<1> It would run every task that `check` requires (e.g., `test` and `integTest`), skip `test` because it has already been run (indirectly by `check`), and then finally it would _wastefully_ delete every project output.
As a quick helper, below are the equivalent commands from `maven` to `gradle`. You can also run `gradle tasks` to see all tasks that are available to run.
[cols="3*", options="header"]
|====
| Maven | Gradle | Description
| `clean` | `clean` | Delete anything that exists already. You do _not_ generally need to run `clean` with Gradle for any task that _Gradle_ manages the inputs/outputs (in other words, it knows when it needs to rebuild versus reuse).
| `test` | `test` | Run all unit tests.
| `verify` | `check` | Run all tests, plus extra checks (e.g., `checkStyle`, `forbiddenApis`, etc.).
| `verify -Dskip.unit.tests` | `integTest` | Run only integration tests.
| `package -DskipTests` | `assemble` | Output is in `${project.projectDir}/build/distributions`
| `install -DskipTests` | `install` | Build jars and place them into the local _Maven_ repository (yes, even with Gradle).
This should be unnecessary with the unified build!
|====
The full task list, with a minor breakout as a graph of dependencies can be seen with:
[source,bash]
----
$ gradle tasks --all
----
Given that we currently have 80 projects, this can be extremely verbose.
With Gradle, you can easily target specific `projects` to run commands against, and it will build all necessary dependencies to make it happen. For example, if you make a change to a specific test in the `x-pack` subproject, then you can specifically invoke its `test` task.
[source,bash]
----
$ gradle :x-plugins:elasticsearch:x-pack:test -Dtests.class=*YourTests
----
This applies to any command that follows the Directed Acyclic Graph (DAG) for its dependencies. The above example would trigger Elasticsearch `core` to be built, as well as the test framework and any other dependencies that it may have.

View File

@ -42,7 +42,7 @@ public final class MessyTestUtils {
engineServiceSet.add(groovyScriptEngineService);
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(
Arrays.asList(
new ScriptEngineRegistry.ScriptEngineRegistration(GroovyScriptEngineService.class, GroovyScriptEngineService.TYPES)
new ScriptEngineRegistry.ScriptEngineRegistration(GroovyScriptEngineService.class, GroovyScriptEngineService.NAME)
)
);
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Arrays.asList(ScriptServiceProxy.INSTANCE));

View File

@ -56,7 +56,7 @@ public class WatcherTemplateTests extends ESTestCase {
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(
Arrays.asList(
new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class,
MustacheScriptEngineService.TYPES)
MustacheScriptEngineService.NAME)
)
);
ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, registry);

View File

@ -524,7 +524,7 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
logger.debug("notifying [{}] listeners", registeredLicensees.size());
long now = System.currentTimeMillis();
if (license.issueDate() > now) {
logger.info("license [{}] - invalid", license.uid());
logger.warn("license [{}] - invalid", license.uid());
return;
}
long expiryDuration = license.expiryDate() - now;
@ -532,7 +532,7 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
for (InternalLicensee licensee : registeredLicensees) {
licensee.onChange(license, LicenseState.ENABLED);
}
logger.info("license [{}] - valid", license.uid());
logger.debug("license [{}] - valid", license.uid());
final TimeValue delay = TimeValue.timeValueMillis(expiryDuration);
// cancel any previous notifications
cancelNotifications(expiryNotifications);
@ -546,7 +546,7 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
for (InternalLicensee licensee : registeredLicensees) {
licensee.onChange(license, LicenseState.GRACE_PERIOD);
}
logger.info("license [{}] - grace", license.uid());
logger.warn("license [{}] - grace", license.uid());
final TimeValue delay = TimeValue.timeValueMillis(expiryDuration + gracePeriodDuration.getMillis());
// cancel any previous notifications
cancelNotifications(expiryNotifications);
@ -560,7 +560,7 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
for (InternalLicensee licensee : registeredLicensees) {
licensee.onChange(license, LicenseState.DISABLED);
}
logger.info("license [{}] - expired", license.uid());
logger.warn("license [{}] - expired", license.uid());
}
if (!license.equals(currentLicense.get())) {
currentLicense.set(license);

View File

@ -85,10 +85,10 @@ public class MonitoringBulkRequest extends ActionRequest<MonitoringBulkRequest>
* Parses a monitoring bulk request and builds the list of documents to be indexed.
*/
public MonitoringBulkRequest add(BytesReference content, String defaultMonitoringId, String defaultMonitoringVersion,
String defaultIndex, String defaultType) throws Exception {
String defaultType) throws Exception {
// MonitoringBulkRequest accepts a body request that has the same format as the BulkRequest:
// instead of duplicating the parsing logic here we use a new BulkRequest instance to parse the content.
BulkRequest bulkRequest = Requests.bulkRequest().add(content, defaultIndex, defaultType);
BulkRequest bulkRequest = Requests.bulkRequest().add(content, null, defaultType);
for (ActionRequest request : bulkRequest.requests()) {
if (request instanceof IndexRequest) {
@ -114,18 +114,12 @@ public class MonitoringBulkRequest extends ActionRequest<MonitoringBulkRequest>
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
int size = in.readVInt();
for (int i = 0; i < size; i++) {
add(new MonitoringBulkDoc(in));
}
docs.addAll(in.readList(MonitoringBulkDoc::new));
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeVInt(docs.size());
for (MonitoringBulkDoc doc : docs) {
doc.writeTo(out);
}
out.writeList(docs);
}
}

View File

@ -21,9 +21,9 @@ public class MonitoringBulkRequestBuilder
return this;
}
public MonitoringBulkRequestBuilder add(BytesReference content, String defaultId, String defaultVersion, String defaultIndex,
String defaultType) throws Exception {
request.add(content, defaultId, defaultVersion, defaultIndex, defaultType);
public MonitoringBulkRequestBuilder add(BytesReference content, String defaultId, String defaultVersion, String defaultType)
throws Exception {
request.add(content, defaultId, defaultVersion, defaultType);
return this;
}
}

View File

@ -7,26 +7,13 @@ package org.elasticsearch.marvel.rest;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.client.MonitoringClient;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.RestChannel;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.rest.XPackRestHandler;
import java.util.Locale;
public abstract class MonitoringRestHandler extends XPackRestHandler {
public abstract class MonitoringRestHandler extends BaseRestHandler {
protected static String URI_BASE = String.format(Locale.ROOT, "/_%s/monitoring", XPackPlugin.NAME);
protected static String URI_BASE = XPackRestHandler.URI_BASE + "/monitoring";
public MonitoringRestHandler(Settings settings, Client client) {
super(settings, client);
}
@Override
protected final void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
handleRequest(request, channel, new MonitoringClient(client));
}
protected abstract void handleRequest(RestRequest request, RestChannel channel, MonitoringClient client) throws Exception;
}

View File

@ -13,7 +13,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.marvel.action.MonitoringBulkRequestBuilder;
import org.elasticsearch.marvel.action.MonitoringBulkResponse;
import org.elasticsearch.marvel.client.MonitoringClient;
import org.elasticsearch.marvel.rest.MonitoringRestHandler;
import org.elasticsearch.rest.BytesRestResponse;
import org.elasticsearch.rest.RestChannel;
@ -22,6 +21,7 @@ import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.action.support.RestActions;
import org.elasticsearch.rest.action.support.RestBuilderListener;
import org.elasticsearch.xpack.XPackClient;
import static org.elasticsearch.rest.RestRequest.Method.POST;
import static org.elasticsearch.rest.RestRequest.Method.PUT;
@ -38,13 +38,10 @@ public class RestMonitoringBulkAction extends MonitoringRestHandler {
controller.registerHandler(PUT, URI_BASE + "/_bulk", this);
controller.registerHandler(POST, URI_BASE + "/{type}/_bulk", this);
controller.registerHandler(PUT, URI_BASE + "/{type}/_bulk", this);
controller.registerHandler(POST, URI_BASE + "/{index}/{type}/_bulk", this);
controller.registerHandler(PUT, URI_BASE + "/{index}/{type}/_bulk", this);
}
@Override
protected void handleRequest(RestRequest request, RestChannel channel, MonitoringClient client) throws Exception {
String defaultIndex = request.param("index");
protected void handleRequest(RestRequest request, RestChannel channel, XPackClient client) throws Exception {
String defaultType = request.param("type");
String id = request.param(MONITORING_ID);
@ -60,8 +57,8 @@ public class RestMonitoringBulkAction extends MonitoringRestHandler {
throw new ElasticsearchParseException("no body content for monitoring bulk request");
}
MonitoringBulkRequestBuilder requestBuilder = client.prepareMonitoringBulk();
requestBuilder.add(request.content(), id, version, defaultIndex, defaultType);
MonitoringBulkRequestBuilder requestBuilder = client.monitoring().prepareMonitoringBulk();
requestBuilder.add(request.content(), id, version, defaultType);
requestBuilder.execute(new RestBuilderListener<MonitoringBulkResponse>(channel) {
@Override
public RestResponse buildResponse(MonitoringBulkResponse response, XContentBuilder builder) throws Exception {

View File

@ -13,6 +13,9 @@
"xpack.version": "${project.version}"
}
},
"kibana": {
"enabled": false
},
"node": {
"enabled": false
}

View File

@ -52,6 +52,156 @@
}
}
}
},
"kibana_stats": {
"properties": {
"kibana_stats": {
"properties": {
"kibana": {
"properties": {
"uuid": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"transport_address": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"snapshot": {
"type": "boolean"
},
"status": {
"type": "keyword"
},
"statuses": {
"properties": {
"name": {
"type": "keyword"
},
"state": {
"type": "keyword"
}
}
}
}
},
"os": {
"properties": {
"load": {
"properties": {
"1m": {
"type": "float"
},
"5m": {
"type": "float"
},
"15m": {
"type": "float"
}
}
},
"memory": {
"properties": {
"total_in_bytes": {
"type": "float"
},
"free_in_bytes": {
"type": "float"
},
"used_in_bytes": {
"type": "float"
}
}
},
"uptime_in_millis": {
"type": "long"
}
}
},
"process": {
"properties": {
"memory": {
"properties": {
"heap": {
"properties": {
"total_in_bytes": {
"type": "float"
},
"used_in_bytes": {
"type": "float"
}
}
},
"resident_set_size_in_bytes": {
"type": "float"
}
}
},
"event_loop_delay": {
"type": "float"
},
"uptime_in_millis": {
"type": "long"
}
}
},
"sockets": {
"properties": {
"http": {
"properties": {
"total": {
"type": "long"
}
}
},
"https": {
"properties": {
"total": {
"type": "long"
}
}
}
}
},
"timestamp": {
"type": "date"
},
"requests": {
"properties": {
"disconnects": {
"type": "long"
},
"total": {
"type": "long"
},
"status_codes": {
"type": "object"
}
}
},
"response_times": {
"properties": {
"average": {
"type": "float"
},
"max": {
"type": "float"
}
}
},
"concurrent_connections": {
"type": "long"
}
}
}
}
}
}
}

View File

@ -13,7 +13,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.CoreMatchers;
import org.hamcrest.Matcher;
import java.io.IOException;
@ -109,12 +108,26 @@ public class MonitoringBulkRequestTests extends ESTestCase {
public void testAddMultipleDocs() throws Exception {
final int nbDocs = randomIntBetween(3, 20);
final MonitoringIndex[] indices = new MonitoringIndex[nbDocs];
final String[] types = new String[nbDocs];
final XContentType xContentType = XContentType.JSON;
int i;
try (BytesStreamOutput content = new BytesStreamOutput()) {
try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, content)) {
for (int i = 0; i < nbDocs; i++) {
builder.startObject().startObject("index").endObject().endObject().flush();
for (i = 0; i < nbDocs; i++) {
builder.startObject().startObject("index");
if (rarely()) {
indices[i] = MonitoringIndex.DATA;
builder.field("_index", "_data");
} else {
indices[i] = MonitoringIndex.TIMESTAMPED;
}
if (randomBoolean()) {
types[i] = randomAsciiOfLength(5);
builder.field("_type", types[i]);
}
builder.endObject().endObject().flush();
content.write(xContentType.xContent().streamSeparator());
builder.startObject().field("foo").value(i).endObject().flush();
content.write(xContentType.xContent().streamSeparator());
@ -123,20 +136,23 @@ public class MonitoringBulkRequestTests extends ESTestCase {
String defaultMonitoringId = randomBoolean() ? randomAsciiOfLength(2) : null;
String defaultMonitoringVersion = randomBoolean() ? randomAsciiOfLength(3) : null;
String defaultIndex = randomFrom("_data", null);
String defaultType = randomBoolean() ? randomAsciiOfLength(4) : null;
MonitoringIndex index = MonitoringIndex.from(defaultIndex);
String defaultType = rarely() ? randomAsciiOfLength(4) : null;
MonitoringBulkRequest request = new MonitoringBulkRequest();
request.add(content.bytes(), defaultMonitoringId, defaultMonitoringVersion, defaultIndex, defaultType);
request.add(content.bytes(), defaultMonitoringId, defaultMonitoringVersion, defaultType);
assertThat(request.getDocs(), hasSize(nbDocs));
i = 0;
for (MonitoringBulkDoc doc : request.getDocs()) {
String expectedType = types[i] != null ? types[i] : defaultType;
assertThat(doc.getMonitoringId(), equalTo(defaultMonitoringId));
assertThat(doc.getMonitoringVersion(), equalTo(defaultMonitoringVersion));
assertThat(doc.getIndex(), sameInstance(index));
assertThat(doc.getType(), equalTo(defaultType));
assertThat(doc.getIndex(), sameInstance(indices[i]));
assertThat(doc.getType(), equalTo(expectedType));
++i;
}
}
}
@ -170,7 +186,9 @@ public class MonitoringBulkRequestTests extends ESTestCase {
MonitoringBulkRequest request2 = new MonitoringBulkRequest();
request2.readFrom(in);
assertThat(request2.docs.size(), CoreMatchers.equalTo(request.docs.size()));
assertThat(in.available(), equalTo(0));
assertThat(request2.docs.size(), equalTo(request.docs.size()));
for (int i = 0; i < request2.docs.size(); i++) {
MonitoringBulkDoc doc = request.docs.get(i);
MonitoringBulkDoc doc2 = request2.docs.get(i);

View File

@ -128,7 +128,7 @@ public class ClusterStatsResolverTests extends MonitoringIndexNameResolverTestCa
statsByShard.put(index, Collections.singletonList(new IndexShardStats(new ShardId(index, 0), randomShardStats())));
return new NodeStats(new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
new NodeIndicesStats(new CommonStats(), statsByShard), null, null, null, null,
new FsInfo(0, pathInfo), null, null, null, null, null, null);
new FsInfo(0, null, pathInfo), null, null, null, null, null, null);
}
/**

View File

@ -137,6 +137,6 @@ public class NodeStatsResolverTests extends MonitoringIndexNameResolverTestCase<
new NodeIndicesStats(new CommonStats(), statsByShard), OsProbe.getInstance().osStats(),
ProcessProbe.getInstance().processStats(), JvmStats.jvmStats(),
new ThreadPoolStats(threadPoolStats),
new FsInfo(0, pathInfo), null, null, null, null, null, null);
new FsInfo(0, null, pathInfo), null, null, null, null, null, null);
}
}

View File

@ -1,15 +1,11 @@
{
"monitoring.bulk": {
"xpack.monitoring.bulk": {
"documentation": "http://www.elastic.co/guide/en/marvel/current/appendix-api-bulk.html",
"methods": ["POST", "PUT"],
"url": {
"path": "/_x-pack/monitoring/_bulk",
"paths": ["/_x-pack/monitoring/_bulk", "/_x-pack/monitoring/{index}/_bulk", "/_x-pack/monitoring/{index}/{type}/_bulk"],
"path": "/_xpack/monitoring/_bulk",
"paths": ["/_xpack/monitoring/_bulk", "/_xpack/monitoring/{type}/_bulk"],
"parts": {
"index": {
"type" : "string",
"description" : "Default index for items which don't provide one"
},
"type": {
"type" : "string",
"description" : "Default document type for items which don't provide one"

View File

@ -1,5 +1,9 @@
---
setup:
- do:
cluster.health:
wait_for_status: yellow
- do:
# Waits for the monitoring data index to be available:
# it indicates that the local exporter is ready
@ -14,7 +18,7 @@ setup:
- set: {version.number: version}
- do:
monitoring.bulk:
xpack.monitoring.bulk:
system_id: "kibana"
system_version: $version
body:
@ -48,7 +52,7 @@ setup:
- match: { hits.total: 2 }
- do:
monitoring.bulk:
xpack.monitoring.bulk:
system_id: "kibana"
system_version: $version
type: "default_type"

View File

@ -107,11 +107,11 @@ public class ShieldServerTransportService extends TransportService {
@Override
public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> request, String executor,
boolean forceExecution,
boolean forceExecution, boolean canTripCircuitBreaker,
TransportRequestHandler<Request> handler) {
TransportRequestHandler<Request> wrappedHandler = new ProfileSecuredRequestHandler<>(action, handler, profileFilters,
licenseState, threadPool.getThreadContext());
super.registerRequestHandler(action, request, executor, forceExecution, wrappedHandler);
super.registerRequestHandler(action, request, executor, forceExecution, canTripCircuitBreaker, wrappedHandler);
}
protected Map<String, ServerTransportFilter> initializeProfileFilters() {

View File

@ -17,7 +17,7 @@ import org.elasticsearch.xpack.XPackClient;
*/
public abstract class XPackRestHandler extends BaseRestHandler {
protected static String URI_BASE = "_xpack";
protected static String URI_BASE = "/_xpack";
public XPackRestHandler(Settings settings, Client client) {
super(settings, client);

View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.script;
import org.elasticsearch.script.ScriptMode;
import org.elasticsearch.xpack.common.text.DefaultTextTemplateEngine;
import java.util.Collections;
@ -29,28 +30,27 @@ public class MockMustacheScriptEngine extends MockScriptEngine {
public void onModule(ScriptModule module) {
module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MockMustacheScriptEngine.class,
Collections.singletonList(NAME)));
NAME, ScriptMode.ON));
}
}
@Override
public List<String> getTypes() {
return Collections.singletonList(NAME);
public String getType() {
return NAME;
}
@Override
public List<String> getExtensions() {
return getTypes();
public String getExtension() {
return NAME;
}
@Override
public Object compile(String script, Map<String, String> params) {
public Object compile(String name, String script, Map<String, String> params) {
if (script.contains("{{") && script.contains("}}")) {
throw new IllegalArgumentException("Fix your test to not rely on mustache");
}
return script;
return super.compile(name, script, params);
}
}

View File

@ -7,6 +7,7 @@ package org.elasticsearch.script;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.ScriptMode;
import org.elasticsearch.search.lookup.SearchLookup;
import java.io.IOException;
@ -21,8 +22,6 @@ public class SleepScriptEngine implements ScriptEngineService {
public static final String NAME = "sleep";
public static final List<String> TYPES = Collections.singletonList(NAME);
public static class TestPlugin extends Plugin {
public TestPlugin() {
@ -39,29 +38,25 @@ public class SleepScriptEngine implements ScriptEngineService {
}
public void onModule(ScriptModule module) {
module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(SleepScriptEngine.class, SleepScriptEngine.TYPES));
module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(SleepScriptEngine.class,
SleepScriptEngine.NAME, ScriptMode.ON));
}
}
@Override
public List<String> getTypes() {
return TYPES;
public String getType() {
return NAME;
}
@Override
public List<String> getExtensions() {
return TYPES;
public String getExtension() {
return NAME;
}
@Override
public boolean isSandboxed() {
return true;
}
@Override
public Object compile(String script, Map<String, String> params) {
return script;
public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
return scriptSource;
}
@Override

View File

@ -136,6 +136,7 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase
.put("xpack.watcher.watch.scroll.size", randomIntBetween(1, 100))
.put(ShieldSettings.settings(shieldEnabled))
.put("xpack.watcher.trigger.schedule.engine", scheduleImplName)
.put("script.inline", "true")
.build();
}