honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/sql 

Original commit: elastic/x-pack-elasticsearch@bbc72c0368
This commit is contained in:
Nik Everett 2017-11-17 11:44:13 -05:00
parent 193bc5f2b5 0f97e28074
commit b8e082107f
40 changed files with 463 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
docs/en/ml/categories.asciidoc
View File

@ -11,10 +11,15 @@ example:
//NOTCONSOLE
You can use {ml} to observe the static parts of the message, cluster similar
messages together, and classify them into message categories. The {ml} model
learns what volume and pattern is normal for each category over time. You can
then detect anomalies and surface rare events or unusual types of messages by
using count or rare functions. For example:
messages together, and classify them into message categories.
NOTE: Categorization uses English tokenization rules and dictionary words in
order to identify log message categories. As such, only English language log
messages are supported.
The {ml} model learns what volume and pattern is normal for each category over
time. You can then detect anomalies and surface rare events or unusual types of
messages by using count or rare functions. For example:
//Obtained from it_ops_new_app_logs.sh
[source,js]

7
docs/en/ml/limitations.asciidoc
View File

@ -4,6 +4,13 @@
The following limitations and known problems apply to the {version} release of
{xpack}:
[float]
=== Categorization uses English tokenization rules and dictionary words
//See x-pack-elasticsearch/#3021
Categorization identifies static parts of unstructured logs and groups similar
messages together. This is currently supported only for English language log
messages.
[float]
=== Pop-ups must be enabled in browsers
//See x-pack-elasticsearch/#844

2
docs/en/ml/populations.asciidoc
View File

@ -50,7 +50,7 @@ PUT _xpack/ml/anomaly_detectors/population
}
----------------------------------
//CONSOLE
<1> This `over_field-name` property indicates that the metrics for each user (
<1> This `over_field_name` property indicates that the metrics for each user (
as identified by their `username` value) are analyzed relative to other users
in each bucket.

108
plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java
View File

@ -67,13 +67,10 @@ import org.elasticsearch.xpack.deprecation.Deprecation;
import org.elasticsearch.xpack.extensions.XPackExtension;
import org.elasticsearch.xpack.extensions.XPackExtensionsService;
import org.elasticsearch.xpack.graph.Graph;
import org.elasticsearch.xpack.graph.GraphFeatureSet;
import org.elasticsearch.xpack.logstash.Logstash;
import org.elasticsearch.xpack.logstash.LogstashFeatureSet;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.MachineLearningFeatureSet;
import org.elasticsearch.xpack.monitoring.Monitoring;
import org.elasticsearch.xpack.monitoring.MonitoringFeatureSet;
import org.elasticsearch.xpack.persistent.CompletionPersistentTaskAction;
import org.elasticsearch.xpack.persistent.PersistentTasksClusterService;
import org.elasticsearch.xpack.persistent.PersistentTasksExecutor;
@ -86,10 +83,8 @@ import org.elasticsearch.xpack.rest.action.RestXPackInfoAction;
import org.elasticsearch.xpack.rest.action.RestXPackUsageAction;
import org.elasticsearch.xpack.security.InternalClient;
import org.elasticsearch.xpack.security.Security;
import org.elasticsearch.xpack.security.SecurityFeatureSet;
import org.elasticsearch.xpack.security.authc.AuthenticationService;
import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.sql.analysis.catalog.FilteredCatalog;
import org.elasticsearch.xpack.sql.SecurityCatalogFilter;
import org.elasticsearch.xpack.sql.plugin.SqlLicenseChecker;
@ -98,41 +93,16 @@ import org.elasticsearch.xpack.ssl.SSLConfigurationReloader;
import org.elasticsearch.xpack.ssl.SSLService;
import org.elasticsearch.xpack.upgrade.Upgrade;
import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.WatcherFeatureSet;
import org.elasticsearch.xpack.watcher.common.http.HttpClient;
import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
import org.elasticsearch.xpack.watcher.common.http.HttpSettings;
import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory;
import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
import org.elasticsearch.xpack.watcher.notification.email.Account;
import org.elasticsearch.xpack.watcher.notification.email.EmailService;
import org.elasticsearch.xpack.watcher.notification.email.attachment.DataAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.EmailAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.EmailAttachmentsParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.HttpEmailAttachementParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.ReportingAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.support.BodyPartSource;
import org.elasticsearch.xpack.watcher.notification.hipchat.HipChatService;
import org.elasticsearch.xpack.watcher.notification.jira.JiraService;
import org.elasticsearch.xpack.watcher.notification.pagerduty.PagerDutyAccount;
import org.elasticsearch.xpack.watcher.notification.pagerduty.PagerDutyService;
import org.elasticsearch.xpack.watcher.notification.slack.SlackService;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.security.AccessController;
import java.security.GeneralSecurityException;
import java.security.PrivilegedAction;
import java.time.Clock;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -209,9 +179,6 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
throw bogus; // some other bug
}
}
// some classes need to have their own clinit blocks
BodyPartSource.init();
Account.init();
}
protected final Settings settings;
@ -323,29 +290,8 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
}
components.addAll(monitoring.createComponents(internalClient, threadPool, clusterService, licenseService, sslService));
final CryptoService cryptoService;
try {
cryptoService = ENCRYPT_SENSITIVE_DATA_SETTING.get(settings) ? new CryptoService(settings) : null;
} catch (IOException e) {
throw new UncheckedIOException(e);
}
// watcher http stuff
Map<String, HttpAuthFactory> httpAuthFactories = new HashMap<>();
httpAuthFactories.put(BasicAuth.TYPE, new BasicAuthFactory(cryptoService));
// TODO: add more auth types, or remove this indirection
HttpAuthRegistry httpAuthRegistry = new HttpAuthRegistry(httpAuthFactories);
HttpRequestTemplate.Parser httpTemplateParser = new HttpRequestTemplate.Parser(httpAuthRegistry);
components.add(httpTemplateParser);
final HttpClient httpClient = new HttpClient(settings, httpAuthRegistry, sslService);
components.add(httpClient);
Collection<Object> notificationComponents = createNotificationComponents(clusterService.getClusterSettings(), httpClient,
httpTemplateParser, scriptService, httpAuthRegistry, cryptoService);
components.addAll(notificationComponents);
components.addAll(watcher.createComponents(getClock(), scriptService, internalClient, licenseState,
httpClient, httpTemplateParser, threadPool, clusterService, cryptoService, xContentRegistry, components));
components.addAll(watcher.createComponents(getClock(), scriptService, internalClient, licenseState, threadPool, clusterService,
xContentRegistry, sslService));
PersistentTasksService persistentTasksService = new PersistentTasksService(settings, clusterService, threadPool, internalClient);
@ -377,28 +323,6 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
return components;
}
private Collection<Object> createNotificationComponents(ClusterSettings clusterSettings, HttpClient httpClient,
HttpRequestTemplate.Parser httpTemplateParser, ScriptService scriptService,
HttpAuthRegistry httpAuthRegistry, CryptoService cryptoService) {
List<Object> components = new ArrayList<>();
components.add(new EmailService(settings, cryptoService, clusterSettings));
components.add(new HipChatService(settings, httpClient, clusterSettings));
components.add(new JiraService(settings, httpClient, clusterSettings));
components.add(new SlackService(settings, httpClient, clusterSettings));
components.add(new PagerDutyService(settings, httpClient, clusterSettings));
TextTemplateEngine textTemplateEngine = new TextTemplateEngine(settings, scriptService);
components.add(textTemplateEngine);
Map<String, EmailAttachmentParser> parsers = new HashMap<>();
parsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, httpTemplateParser, textTemplateEngine));
parsers.put(DataAttachmentParser.TYPE, new DataAttachmentParser());
parsers.put(ReportingAttachmentParser.TYPE, new ReportingAttachmentParser(settings, httpClient, textTemplateEngine,
httpAuthRegistry));
components.add(new EmailAttachmentsParser(parsers));
return components;
}
@Override
public Settings additionalSettings() {
Settings.Builder builder = Settings.builder();
@ -425,7 +349,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
@Override
public List<ScriptContext> getContexts() {
return Arrays.asList(Watcher.SCRIPT_SEARCH_CONTEXT, Watcher.SCRIPT_EXECUTABLE_CONTEXT, Watcher.SCRIPT_TEMPLATE_CONTEXT);
return watcher.getContexts();
}
@Override
@ -443,30 +367,13 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
// we add the `xpack.version` setting to all internal indices
settings.add(Setting.simpleString("index.xpack.version", Setting.Property.IndexScope));
// notification services
settings.add(SlackService.SLACK_ACCOUNT_SETTING);
settings.add(EmailService.EMAIL_ACCOUNT_SETTING);
settings.add(HipChatService.HIPCHAT_ACCOUNT_SETTING);
settings.add(JiraService.JIRA_ACCOUNT_SETTING);
settings.add(PagerDutyService.PAGERDUTY_ACCOUNT_SETTING);
settings.add(ReportingAttachmentParser.RETRIES_SETTING);
settings.add(ReportingAttachmentParser.INTERVAL_SETTING);
// http settings
settings.addAll(HttpSettings.getSettings());
return settings;
}
@Override
public List<String> getSettingsFilter() {
List<String> filters = new ArrayList<>();
filters.add("xpack.notification.email.account.*.smtp.password");
filters.add("xpack.notification.jira.account.*.password");
filters.add("xpack.notification.slack.account.*.url");
filters.add("xpack.notification.pagerduty.account.*.url");
filters.add("xpack.notification.pagerduty." + PagerDutyAccount.SERVICE_KEY_SETTING);
filters.add("xpack.notification.pagerduty.account.*." + PagerDutyAccount.SERVICE_KEY_SETTING);
filters.add("xpack.notification.hipchat.account.*.auth_token");
filters.addAll(watcher.getSettingsFilter());
filters.addAll(security.getSettingsFilter(extensionsService));
filters.addAll(monitoring.getSettingsFilter());
if (transportClientMode == false) {
@ -555,17 +462,14 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
@Override
public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, SECURITY, SecurityFeatureSet.Usage::new));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, WATCHER, WatcherFeatureSet.Usage::new));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, MONITORING, MonitoringFeatureSet.Usage::new));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, GRAPH, GraphFeatureSet.Usage::new));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, MACHINE_LEARNING, MachineLearningFeatureSet.Usage::new));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, LOGSTASH, LogstashFeatureSet.Usage::new));
entries.addAll(watcher.getNamedWriteables());
entries.addAll(machineLearning.getNamedWriteables());
entries.addAll(licensing.getNamedWriteables());
entries.addAll(Security.getNamedWriteables());
entries.addAll(SqlPlugin.getNamedWriteables());
entries.addAll(Monitoring.getNamedWriteables());
entries.addAll(Graph.getNamedWriteables());
return entries;
}

11
plugin/src/main/java/org/elasticsearch/xpack/graph/Graph.java
View File

@ -10,20 +10,22 @@ import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.graph.action.GraphExploreAction;
import org.elasticsearch.xpack.graph.action.TransportGraphExploreAction;
import org.elasticsearch.xpack.graph.rest.action.RestGraphAction;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@ -31,8 +33,9 @@ import java.util.function.Supplier;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.XPackPlugin.GRAPH;
public class Graph extends Plugin implements ActionPlugin {
public class Graph implements ActionPlugin {
public static final String NAME = "graph";
protected final boolean enabled;
@ -65,4 +68,8 @@ public class Graph extends Plugin implements ActionPlugin {
}
return singletonList(new RestGraphAction(settings, restController));
}
public static Collection<? extends NamedWriteableRegistry.Entry> getNamedWriteables() {
return Arrays.asList(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, GRAPH, GraphFeatureSet.Usage::new));
}
}

7
plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ExecutorBuilder;
import org.elasticsearch.threadpool.FixedExecutorBuilder;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.ml.action.CloseJobAction;
@ -145,6 +146,7 @@ import java.util.List;
import java.util.function.Supplier;
import static java.util.Collections.emptyList;
import static org.elasticsearch.xpack.XPackPlugin.MACHINE_LEARNING;
public class MachineLearning implements ActionPlugin {
public static final String NAME = "ml";
@ -270,7 +272,10 @@ public class MachineLearning implements ActionPlugin {
new NamedWriteableRegistry.Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME,
PersistentTasksNodeService.Status::new),
new NamedWriteableRegistry.Entry(Task.Status.class, JobTaskStatus.NAME, JobTaskStatus::new),
new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream)
new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream),
// feature set
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, MACHINE_LEARNING, MachineLearningFeatureSet.Usage::new)
);
}

24
plugin/src/main/java/org/elasticsearch/xpack/ml/action/ForecastJobAction.java
View File

@ -19,6 +19,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -56,12 +57,14 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
public static class Request extends TransportJobTaskAction.JobTaskRequest<Request> implements ToXContentObject {
public static final ParseField END_TIME = new ParseField("end");
public static final ParseField DURATION = new ParseField("duration");
private static final ObjectParser<Request, Void> PARSER = new ObjectParser<>(NAME, Request::new);
static {
PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID);
PARSER.declareString(Request::setEndTime, END_TIME);
PARSER.declareString(Request::setDuration, DURATION);
}
public static Request parseRequest(String jobId, XContentParser parser) {
@ -73,6 +76,7 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
}
private String endTime;
private TimeValue duration;
Request() {
}
@ -89,21 +93,31 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
this.endTime = endTime;
}
public TimeValue getDuration() {
return duration;
}
public void setDuration(String duration) {
this.duration = TimeValue.parseTimeValue(duration, DURATION.getPreferredName());
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
this.endTime = in.readOptionalString();
this.duration = in.readOptionalWriteable(TimeValue::new);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeOptionalString(endTime);
out.writeOptionalWriteable(duration);
}
@Override
public int hashCode() {
return Objects.hash(jobId, endTime);
return Objects.hash(jobId, endTime, duration);
}
@Override
@ -115,7 +129,7 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
return false;
}
Request other = (Request) obj;
return Objects.equals(jobId, other.jobId) && Objects.equals(endTime, other.endTime);
return Objects.equals(jobId, other.jobId) && Objects.equals(endTime, other.endTime) && Objects.equals(duration, other.duration);
}
@Override
@ -125,6 +139,9 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
if (endTime != null) {
builder.field(END_TIME.getPreferredName(), endTime);
}
if (duration != null) {
builder.field(DURATION.getPreferredName(), duration.getStringRep());
}
builder.endObject();
return builder;
}
@ -219,6 +236,9 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
if (request.getEndTime() != null) {
paramsBuilder.endTime(request.getEndTime(), END_TIME);
}
if (request.getDuration() != null) {
paramsBuilder.duration(request.getDuration());
}
ForecastParams params = paramsBuilder.build();
processManager.forecastJob(task, params, e -> {

1
plugin/src/main/java/org/elasticsearch/xpack/ml/job/messages/Messages.java
View File

@ -164,6 +164,7 @@ public final class Messages {
"Model snapshot ''{0}'' is the active snapshot for job ''{1}'', so cannot be deleted";
public static final String REST_INVALID_DATETIME_PARAMS =
"Query param [{0}] with value [{1}] cannot be parsed as a date or converted to a number (epoch).";
public static final String REST_INVALID_DURATION_AND_ENDTIME = "Specify either duration or end time";
public static final String REST_INVALID_FLUSH_PARAMS_MISSING = "Invalid flush parameters: ''{0}'' has not been specified.";
public static final String REST_INVALID_FLUSH_PARAMS_UNEXPECTED = "Invalid flush parameters: unexpected ''{0}''.";
public static final String REST_JOB_NOT_CLOSED_REVERT = "Can only revert to a model snapshot when the job is closed.";

37
plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParams.java
View File

@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ml.job.process.autodetect.params;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.joda.DateMathParser;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.xpack.ml.job.messages.Messages;
@ -16,11 +17,13 @@ import java.util.Objects;
public class ForecastParams {
private final long endTime;
private final long duration;
private final long id;
private ForecastParams(long id, long endTime) {
private ForecastParams(long id, long endTime, long duration) {
this.id = id;
this.endTime = endTime;
this.duration = duration;
}
/**
@ -31,6 +34,14 @@ public class ForecastParams {
return endTime;
}
/**
* The forecast duration in seconds
* @return The duration in seconds
*/
public long getDuration() {
return duration;
}
/**
* The forecast id
*
@ -42,7 +53,7 @@ public class ForecastParams {
@Override
public int hashCode() {
return Objects.hash(id, endTime);
return Objects.hash(id, endTime, duration);
}
@Override
@ -54,27 +65,24 @@ public class ForecastParams {
return false;
}
ForecastParams other = (ForecastParams) obj;
return Objects.equals(id, other.id) && Objects.equals(endTime, other.endTime);
return Objects.equals(id, other.id) && Objects.equals(endTime, other.endTime) && Objects.equals(duration, other.duration);
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private long endTimeEpochSecs;
private long durationSecs;
private long startTime;
private long forecastId;
private Builder() {
startTime = System.currentTimeMillis();
endTimeEpochSecs = tomorrow(startTime);
endTimeEpochSecs = 0;
forecastId = generateId();
}
static long tomorrow(long now) {
return (now / 1000) + (60 * 60 * 24);
durationSecs = 0;
}
private long generateId() {
@ -94,8 +102,17 @@ public class ForecastParams {
return this;
}
public Builder duration(TimeValue duration) {
durationSecs = duration.seconds();
return this;
}
public ForecastParams build() {
return new ForecastParams(forecastId, endTimeEpochSecs);
if (endTimeEpochSecs != 0 && durationSecs != 0) {
throw new ElasticsearchParseException(Messages.getMessage(Messages.REST_INVALID_DURATION_AND_ENDTIME));
}
return new ForecastParams(forecastId, endTimeEpochSecs, durationSecs);
}
}
}

12
plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriter.java
View File

@ -153,9 +153,15 @@ public class ControlMsgToProcessWriter {
public void writeForecastMessage(ForecastParams params) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.field("forecast_id", params.getId())
.field("end_time", params.getEndTime())
.endObject();
.field("forecast_id", params.getId());
if (params.getEndTime() != 0) {
builder.field("end_time", params.getEndTime());
}
if (params.getDuration() != 0) {
builder.field("duration", params.getDuration());
}
builder.endObject();
writeMessage(FORECAST_MESSAGE_CODE + builder.string());
fillCommandBuffer();

17
plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ForecastRequestStats.java
View File

@ -42,6 +42,7 @@ public class ForecastRequestStats implements ToXContentObject, Writeable {
public static final ParseField PROGRESS = new ParseField("forecast_progress");
public static final ParseField PROCESSED_RECORD_COUNT = new ParseField("processed_record_count");
public static final ParseField STATUS = new ParseField("forecast_status");
public static final ParseField MEMORY_USAGE = new ParseField("forecast_memory_bytes");
public static final ConstructingObjectParser<ForecastRequestStats, Void> PARSER =
new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> new ForecastRequestStats((String) a[0], (long) a[1]));
@ -60,6 +61,7 @@ public class ForecastRequestStats implements ToXContentObject, Writeable {
PARSER.declareDouble(ForecastRequestStats::setProgress, PROGRESS);
PARSER.declareLong(ForecastRequestStats::setProcessingTime, PROCESSING_TIME_MS);
PARSER.declareField(ForecastRequestStats::setStatus, p -> ForecastRequestStatus.fromString(p.text()), STATUS, ValueType.STRING);
PARSER.declareLong(ForecastRequestStats::setMemoryUsage, MEMORY_USAGE);
}
public enum ForecastRequestStatus implements Writeable {
@ -92,6 +94,7 @@ public class ForecastRequestStats implements ToXContentObject, Writeable {
private Instant dateEnded = Instant.EPOCH;
private double progress;
private long processingTime;
private long memoryUsage;
private ForecastRequestStatus status = ForecastRequestStatus.OK;
public ForecastRequestStats(String jobId, long forecastId) {
@ -108,6 +111,7 @@ public class ForecastRequestStats implements ToXContentObject, Writeable {
dateEnded = Instant.ofEpochMilli(in.readVLong());
progress = in.readDouble();
processingTime = in.readLong();
setMemoryUsage(in.readLong());
status = ForecastRequestStatus.readFromStream(in);
}
@ -121,6 +125,7 @@ public class ForecastRequestStats implements ToXContentObject, Writeable {
out.writeVLong(dateEnded.toEpochMilli());
out.writeDouble(progress);
out.writeLong(processingTime);
out.writeLong(getMemoryUsage());
status.writeTo(out);
}
@ -142,6 +147,7 @@ public class ForecastRequestStats implements ToXContentObject, Writeable {
}
builder.field(PROGRESS.getPreferredName(), progress);
builder.field(PROCESSING_TIME_MS.getPreferredName(), processingTime);
builder.field(MEMORY_USAGE.getPreferredName(), getMemoryUsage());
builder.field(STATUS.getPreferredName(), status);
builder.endObject();
return builder;
@ -215,6 +221,14 @@ public class ForecastRequestStats implements ToXContentObject, Writeable {
this.processingTime = processingTime;
}
public long getMemoryUsage() {
return memoryUsage;
}
public void setMemoryUsage(long memoryUsage) {
this.memoryUsage = memoryUsage;
}
public ForecastRequestStatus getStatus() {
return status;
}
@ -241,12 +255,13 @@ public class ForecastRequestStats implements ToXContentObject, Writeable {
Objects.equals(this.dateEnded, that.dateEnded) &&
this.progress == that.progress &&
this.processingTime == that.processingTime &&
this.memoryUsage == that.memoryUsage &&
Objects.equals(this.status, that.status);
}
@Override
public int hashCode() {
return Objects.hash(jobId, forecastId, recordCount, message, dateStarted, dateEnded, progress,
processingTime, status);
processingTime, memoryUsage, status);
}
}

3
plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestForecastJobAction.java
View File

@ -43,6 +43,9 @@ public class RestForecastJobAction extends BaseRestHandler {
if (restRequest.hasParam(ForecastJobAction.Request.END_TIME.getPreferredName())) {
request.setEndTime(restRequest.param(ForecastJobAction.Request.END_TIME.getPreferredName()));
}
if (restRequest.hasParam(ForecastJobAction.Request.DURATION.getPreferredName())) {
request.setDuration(restRequest.param(ForecastJobAction.Request.DURATION.getPreferredName()));
}
}
return channel -> client.execute(ForecastJobAction.INSTANCE, request, new RestToXContentListener<>(channel));

7
plugin/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.util.Providers;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Setting;
@ -24,6 +25,7 @@ import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.monitoring.action.MonitoringBulkAction;
@ -59,6 +61,7 @@ import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.common.settings.Setting.boolSetting;
import static org.elasticsearch.common.settings.Setting.timeSetting;
import static org.elasticsearch.xpack.XPackPlugin.MONITORING;
/**
* This class activates/deactivates the monitoring modules depending if we're running a node client, transport client or tribe client:
@ -111,6 +114,10 @@ public class Monitoring implements ActionPlugin {
this.tribeNode = XPackPlugin.isTribeNode(settings);
}
public static Collection<? extends NamedWriteableRegistry.Entry> getNamedWriteables() {
return Arrays.asList(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, MONITORING, MonitoringFeatureSet.Usage::new));
}
boolean isEnabled() {
return enabled;
}

2
plugin/src/main/java/org/elasticsearch/xpack/security/Security.java
View File

@ -72,6 +72,7 @@ import org.elasticsearch.transport.TransportInterceptor;
import org.elasticsearch.transport.TransportRequest;
import org.elasticsearch.transport.TransportRequestHandler;
import org.elasticsearch.watcher.ResourceWatcherService;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.extensions.XPackExtension;
@ -889,6 +890,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
entries.add(new NamedWriteableRegistry.Entry(ClusterState.Custom.class, TokenMetaData.TYPE, TokenMetaData::new));
entries.add(new NamedWriteableRegistry.Entry(NamedDiff.class, TokenMetaData.TYPE, TokenMetaData::readDiffFrom));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackPlugin.SECURITY, SecurityFeatureSet.Usage::new));
entries.addAll(Arrays.asList(ExpressionParser.NAMED_WRITEABLES));
return entries;
}

27
plugin/src/main/java/org/elasticsearch/xpack/security/crypto/tool/SystemKeyTool.java
View File

@ -7,8 +7,9 @@ package org.elasticsearch.xpack.security.crypto.tool;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.elasticsearch.cli.ExitCodes;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cli.EnvironmentAwareCommand;
import org.elasticsearch.cli.ExitCodes;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cli.UserException;
import org.elasticsearch.common.SuppressForbidden;
@ -16,19 +17,24 @@ import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.env.Environment;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.PosixFileAttributeView;
import java.nio.file.attribute.PosixFilePermission;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.Locale;
import java.util.Set;
public class SystemKeyTool extends EnvironmentAwareCommand {
static final String KEY_ALGO = "HmacSHA512";
static final int KEY_SIZE = 1024;
private final OptionSpec<String> arguments;
SystemKeyTool() {
@ -67,7 +73,7 @@ public class SystemKeyTool extends EnvironmentAwareCommand {
// write the key
terminal.println(Terminal.Verbosity.VERBOSE, "generating...");
byte[] key = CryptoService.generateKey();
byte[] key = generateKey();
terminal.println(String.format(Locale.ROOT, "Storing generated key in [%s]...", keyPath.toAbsolutePath()));
Files.write(keyPath, key, StandardOpenOption.CREATE_NEW);
@ -80,6 +86,21 @@ public class SystemKeyTool extends EnvironmentAwareCommand {
}
}
static byte[] generateKey() {
return generateSecretKey(KEY_SIZE).getEncoded();
}
static SecretKey generateSecretKey(int keyLength) {
try {
KeyGenerator generator = KeyGenerator.getInstance(KEY_ALGO);
generator.init(keyLength);
return generator.generateKey();
} catch (NoSuchAlgorithmException e) {
throw new ElasticsearchException("failed to generate key", e);
}
}
@SuppressForbidden(reason = "Parsing command line path")
private static Path parsePath(String path) {
return PathUtils.get(path);

2
plugin/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4Transport.java
View File

@ -109,7 +109,7 @@ public class SecurityNetty4Transport extends Netty4Transport {
}
@Override
protected void onException(NettyTcpChannel channel, Exception e) {
protected void onException(TcpChannel channel, Exception e) {
if (!lifecycle.started()) {
// just close and ignore - we are already stopped and just need to make sure we release all resources
TcpChannel.closeChannel(channel, false);

145
plugin/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
View File

@ -47,19 +47,11 @@ import org.elasticsearch.script.TemplateScript;
import org.elasticsearch.threadpool.ExecutorBuilder;
import org.elasticsearch.threadpool.FixedExecutorBuilder;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.watcher.common.http.HttpClient;
import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
import org.elasticsearch.xpack.watcher.notification.email.EmailService;
import org.elasticsearch.xpack.watcher.notification.email.attachment.EmailAttachmentsParser;
import org.elasticsearch.xpack.watcher.notification.hipchat.HipChatService;
import org.elasticsearch.xpack.watcher.notification.jira.JiraService;
import org.elasticsearch.xpack.watcher.notification.pagerduty.PagerDutyService;
import org.elasticsearch.xpack.watcher.notification.slack.SlackService;
import org.elasticsearch.xpack.security.InternalClient;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.ssl.SSLService;
import org.elasticsearch.xpack.watcher.actions.ActionFactory;
import org.elasticsearch.xpack.watcher.actions.ActionRegistry;
import org.elasticsearch.xpack.watcher.actions.email.EmailAction;
@ -79,6 +71,14 @@ import org.elasticsearch.xpack.watcher.actions.slack.SlackActionFactory;
import org.elasticsearch.xpack.watcher.actions.webhook.WebhookAction;
import org.elasticsearch.xpack.watcher.actions.webhook.WebhookActionFactory;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.common.http.HttpClient;
import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
import org.elasticsearch.xpack.watcher.common.http.HttpSettings;
import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory;
import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
import org.elasticsearch.xpack.watcher.condition.AlwaysCondition;
import org.elasticsearch.xpack.watcher.condition.ArrayCompareCondition;
import org.elasticsearch.xpack.watcher.condition.CompareCondition;
@ -86,6 +86,7 @@ import org.elasticsearch.xpack.watcher.condition.ConditionFactory;
import org.elasticsearch.xpack.watcher.condition.ConditionRegistry;
import org.elasticsearch.xpack.watcher.condition.NeverCondition;
import org.elasticsearch.xpack.watcher.condition.ScriptCondition;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.execution.AsyncTriggerEventConsumer;
import org.elasticsearch.xpack.watcher.execution.ExecutionService;
import org.elasticsearch.xpack.watcher.execution.InternalWatchExecutor;
@ -105,6 +106,19 @@ import org.elasticsearch.xpack.watcher.input.search.SearchInput;
import org.elasticsearch.xpack.watcher.input.search.SearchInputFactory;
import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
import org.elasticsearch.xpack.watcher.input.simple.SimpleInputFactory;
import org.elasticsearch.xpack.watcher.notification.email.Account;
import org.elasticsearch.xpack.watcher.notification.email.EmailService;
import org.elasticsearch.xpack.watcher.notification.email.attachment.DataAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.EmailAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.EmailAttachmentsParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.HttpEmailAttachementParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.ReportingAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.support.BodyPartSource;
import org.elasticsearch.xpack.watcher.notification.hipchat.HipChatService;
import org.elasticsearch.xpack.watcher.notification.jira.JiraService;
import org.elasticsearch.xpack.watcher.notification.pagerduty.PagerDutyAccount;
import org.elasticsearch.xpack.watcher.notification.pagerduty.PagerDutyService;
import org.elasticsearch.xpack.watcher.notification.slack.SlackService;
import org.elasticsearch.xpack.watcher.rest.action.RestAckWatchAction;
import org.elasticsearch.xpack.watcher.rest.action.RestActivateWatchAction;
import org.elasticsearch.xpack.watcher.rest.action.RestDeleteWatchAction;
@ -155,7 +169,9 @@ import org.elasticsearch.xpack.watcher.watch.Watch;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.time.Clock;
import java.util.ArrayList;
import java.util.Arrays;
@ -170,12 +186,17 @@ import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import static java.util.Collections.emptyList;
public class Watcher implements ActionPlugin {
static {
// some classes need to have their own clinit blocks
BodyPartSource.init();
Account.init();
}
public static final Setting<String> INDEX_WATCHER_TEMPLATE_VERSION_SETTING =
new Setting<>("index.xpack.watcher.template.version", "", Function.identity(), Setting.Property.IndexScope);
public static final Setting<Boolean> ENCRYPT_SENSITIVE_DATA_SETTING =
@ -199,6 +220,7 @@ public class Watcher implements ActionPlugin {
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, WatcherMetaData.TYPE, WatcherMetaData::new));
entries.add(new NamedWriteableRegistry.Entry(NamedDiff.class, WatcherMetaData.TYPE, WatcherMetaData::readDiffFrom));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackPlugin.WATCHER, WatcherFeatureSet.Usage::new));
return entries;
}
@ -225,13 +247,44 @@ public class Watcher implements ActionPlugin {
public Collection<Object> createComponents(Clock clock, ScriptService scriptService, InternalClient internalClient,
XPackLicenseState licenseState,
HttpClient httpClient, HttpRequestTemplate.Parser httpTemplateParser,
ThreadPool threadPool, ClusterService clusterService, CryptoService cryptoService,
NamedXContentRegistry xContentRegistry, Collection<Object> components) {
ThreadPool threadPool, ClusterService clusterService,
NamedXContentRegistry xContentRegistry, SSLService sslService) {
if (enabled == false) {
return Collections.emptyList();
}
final CryptoService cryptoService;
try {
cryptoService = ENCRYPT_SENSITIVE_DATA_SETTING.get(settings) ? new CryptoService(settings) : null;
} catch (IOException e) {
throw new UncheckedIOException(e);
}
// http client
Map<String, HttpAuthFactory> httpAuthFactories = new HashMap<>();
httpAuthFactories.put(BasicAuth.TYPE, new BasicAuthFactory(cryptoService));
// TODO: add more auth types, or remove this indirection
HttpAuthRegistry httpAuthRegistry = new HttpAuthRegistry(httpAuthFactories);
HttpRequestTemplate.Parser httpTemplateParser = new HttpRequestTemplate.Parser(httpAuthRegistry);
final HttpClient httpClient = new HttpClient(settings, httpAuthRegistry, sslService);
// notification
EmailService emailService = new EmailService(settings, cryptoService, clusterService.getClusterSettings());
HipChatService hipChatService = new HipChatService(settings, httpClient, clusterService.getClusterSettings());
JiraService jiraService = new JiraService(settings, httpClient, clusterService.getClusterSettings());
SlackService slackService = new SlackService(settings, httpClient, clusterService.getClusterSettings());
PagerDutyService pagerDutyService = new PagerDutyService(settings, httpClient, clusterService.getClusterSettings());
TextTemplateEngine templateEngine = new TextTemplateEngine(settings, scriptService);
Map<String, EmailAttachmentParser> emailAttachmentParsers = new HashMap<>();
emailAttachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, httpTemplateParser,
templateEngine));
emailAttachmentParsers.put(DataAttachmentParser.TYPE, new DataAttachmentParser());
emailAttachmentParsers.put(ReportingAttachmentParser.TYPE, new ReportingAttachmentParser(settings, httpClient, templateEngine,
httpAuthRegistry));
EmailAttachmentsParser emailAttachmentsParser = new EmailAttachmentsParser(emailAttachmentParsers);
// conditions
final Map<String, ConditionFactory> parsers = new HashMap<>();
parsers.put(AlwaysCondition.TYPE, (c, id, p) -> AlwaysCondition.parse(id, p));
parsers.put(NeverCondition.TYPE, (c, id, p) -> NeverCondition.parse(id, p));
@ -245,24 +298,19 @@ public class Watcher implements ActionPlugin {
transformFactories.put(SearchTransform.TYPE, new SearchTransformFactory(settings, internalClient, xContentRegistry, scriptService));
final TransformRegistry transformRegistry = new TransformRegistry(settings, Collections.unmodifiableMap(transformFactories));
// actions
final Map<String, ActionFactory> actionFactoryMap = new HashMap<>();
TextTemplateEngine templateEngine = getService(TextTemplateEngine.class, components);
actionFactoryMap.put(EmailAction.TYPE, new EmailActionFactory(settings, getService(EmailService.class, components), templateEngine,
getService(EmailAttachmentsParser.class, components)));
actionFactoryMap.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient,
getService(HttpRequestTemplate.Parser.class, components), templateEngine));
actionFactoryMap.put(EmailAction.TYPE, new EmailActionFactory(settings, emailService, templateEngine, emailAttachmentsParser));
actionFactoryMap.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient, httpTemplateParser, templateEngine));
actionFactoryMap.put(IndexAction.TYPE, new IndexActionFactory(settings, internalClient));
actionFactoryMap.put(LoggingAction.TYPE, new LoggingActionFactory(settings, templateEngine));
actionFactoryMap.put(HipChatAction.TYPE, new HipChatActionFactory(settings, templateEngine,
getService(HipChatService.class, components)));
actionFactoryMap.put(JiraAction.TYPE, new JiraActionFactory(settings, templateEngine,
getService(JiraService.class, components)));
actionFactoryMap.put(SlackAction.TYPE, new SlackActionFactory(settings, templateEngine,
getService(SlackService.class, components)));
actionFactoryMap.put(PagerDutyAction.TYPE, new PagerDutyActionFactory(settings, templateEngine,
getService(PagerDutyService.class, components)));
actionFactoryMap.put(HipChatAction.TYPE, new HipChatActionFactory(settings, templateEngine, hipChatService));
actionFactoryMap.put(JiraAction.TYPE, new JiraActionFactory(settings, templateEngine, jiraService));
actionFactoryMap.put(SlackAction.TYPE, new SlackActionFactory(settings, templateEngine, slackService));
actionFactoryMap.put(PagerDutyAction.TYPE, new PagerDutyActionFactory(settings, templateEngine, pagerDutyService));
final ActionRegistry registry = new ActionRegistry(actionFactoryMap, conditionRegistry, transformRegistry, clock, licenseState);
// inputs
final Map<String, InputFactory> inputFactories = new HashMap<>();
inputFactories.put(SearchInput.TYPE,
new SearchInputFactory(settings, internalClient, xContentRegistry, scriptService));
@ -275,6 +323,8 @@ public class Watcher implements ActionPlugin {
final WatcherClient watcherClient = new WatcherClient(internalClient);
final HistoryStore historyStore = new HistoryStore(settings, internalClient);
// schedulers
final Set<Schedule.Parser> scheduleParsers = new HashSet<>();
scheduleParsers.add(new CronSchedule.Parser());
scheduleParsers.add(new DailySchedule.Parser());
@ -321,7 +371,8 @@ public class Watcher implements ActionPlugin {
return Arrays.asList(registry, watcherClient, inputRegistry, historyStore, triggerService, triggeredWatchParser,
watcherLifeCycleService, executionService, triggerEngineListener, watcherService, watchParser,
configuredTriggerEngine, triggeredWatchStore, watcherSearchTemplateService, watcherIndexTemplateRegistry);
configuredTriggerEngine, triggeredWatchStore, watcherSearchTemplateService, watcherIndexTemplateRegistry,
slackService, pagerDutyService);
}
protected TriggerEngine getTriggerEngine(Clock clock, ScheduleRegistry scheduleRegistry) {
@ -336,16 +387,6 @@ public class Watcher implements ActionPlugin {
return new AsyncTriggerEventConsumer(settings, executionService);
}
private <T> T getService(Class<T> serviceClass, Collection<Object> services) {
List<Object> collect = services.stream().filter(o -> o.getClass() == serviceClass).collect(Collectors.toList());
if (collect.isEmpty()) {
throw new IllegalArgumentException("no service for class " + serviceClass.getName());
} else if (collect.size() > 1) {
throw new IllegalArgumentException("more than one service for class " + serviceClass.getName());
}
return (T) collect.get(0);
}
public Collection<Module> nodeModules() {
List<Module> modules = new ArrayList<>();
modules.add(b -> {
@ -384,6 +425,18 @@ public class Watcher implements ActionPlugin {
settings.add(Setting.simpleString("xpack.watcher.execution.scroll.timeout", Setting.Property.NodeScope));
settings.add(Setting.simpleString("xpack.watcher.start_immediately", Setting.Property.NodeScope));
// notification services
settings.add(SlackService.SLACK_ACCOUNT_SETTING);
settings.add(EmailService.EMAIL_ACCOUNT_SETTING);
settings.add(HipChatService.HIPCHAT_ACCOUNT_SETTING);
settings.add(JiraService.JIRA_ACCOUNT_SETTING);
settings.add(PagerDutyService.PAGERDUTY_ACCOUNT_SETTING);
settings.add(ReportingAttachmentParser.RETRIES_SETTING);
settings.add(ReportingAttachmentParser.INTERVAL_SETTING);
// http settings
settings.addAll(HttpSettings.getSettings());
// encryption settings
CryptoService.addSettings(settings);
return settings;
@ -520,4 +573,20 @@ public class Watcher implements ActionPlugin {
public List<BootstrapCheck> getBootstrapChecks(Environment env) {
return Collections.singletonList(new EncryptSensitiveDataBootstrapCheck(env));
}
public Collection<? extends String> getSettingsFilter() {
List<String> filters = new ArrayList<>();
filters.add("xpack.notification.email.account.*.smtp.password");
filters.add("xpack.notification.jira.account.*.password");
filters.add("xpack.notification.slack.account.*.url");
filters.add("xpack.notification.pagerduty.account.*.url");
filters.add("xpack.notification.pagerduty." + PagerDutyAccount.SERVICE_KEY_SETTING);
filters.add("xpack.notification.pagerduty.account.*." + PagerDutyAccount.SERVICE_KEY_SETTING);
filters.add("xpack.notification.hipchat.account.*.auth_token");
return filters;
}
public List<ScriptContext> getContexts() {
return Arrays.asList(Watcher.SCRIPT_SEARCH_CONTEXT, Watcher.SCRIPT_EXECUTABLE_CONTEXT, Watcher.SCRIPT_TEMPLATE_CONTEXT);
}
}

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/ApplicableBasicAuth.java
View File

@ -9,7 +9,7 @@ import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.elasticsearch.xpack.watcher.common.http.auth.ApplicableHttpAuth;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import java.net.HttpURLConnection;
import java.nio.charset.StandardCharsets;

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuthFactory.java
View File

@ -8,7 +8,7 @@ package org.elasticsearch.xpack.watcher.common.http.auth.basic;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import java.io.IOException;

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/common/secret/Secret.java
View File

@ -5,7 +5,7 @@
*/
package org.elasticsearch.xpack.watcher.common.secret;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import java.io.IOException;
import java.util.Arrays;

35
plugin/src/main/java/org/elasticsearch/xpack/security/crypto/CryptoService.java → plugin/src/main/java/org/elasticsearch/xpack/watcher/crypto/CryptoService.java
View File

@ -3,12 +3,20 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.security.crypto;
package org.elasticsearch.xpack.watcher.crypto;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.security.authc.support.CharArrays;
import org.elasticsearch.xpack.watcher.Watcher;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
@ -21,15 +29,6 @@ import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.security.authc.support.CharArrays;
import org.elasticsearch.xpack.watcher.Watcher;
import static org.elasticsearch.xpack.security.Security.setting;
/**
@ -88,20 +87,6 @@ public class CryptoService extends AbstractComponent {
assert encryptionKey != null : "the encryption key should never be null";
}
public static byte[] generateKey() {
return generateSecretKey(KEY_SIZE).getEncoded();
}
static SecretKey generateSecretKey(int keyLength) {
try {
KeyGenerator generator = KeyGenerator.getInstance(KEY_ALGO);
generator.init(keyLength);
return generator.generateKey();
} catch (NoSuchAlgorithmException e) {
throw new ElasticsearchException("failed to generate key", e);
}
}
private static SecretKey readSystemKey(InputStream in) throws IOException {
final int keySizeBytes = KEY_SIZE / 8;
final byte[] keyBytes = new byte[keySizeBytes];

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Account.java
View File

@ -10,7 +10,7 @@ import org.elasticsearch.SpecialPermission;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import javax.activation.CommandMap;
import javax.activation.MailcapCommandMap;

3
plugin/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
View File

@ -9,8 +9,8 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.notification.NotificationService;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import javax.mail.MessagingException;
@ -36,7 +36,6 @@ public class EmailService extends NotificationService<Account> {
return new Account(config, cryptoService, logger);
}
public EmailSent send(Email email, Authentication auth, Profile profile, String accountName) throws MessagingException {
Account account = getAccount(accountName);
if (account == null) {

15
plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRespo
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
@ -77,13 +78,15 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
return;
}
if (event.localNodeMaster() == false) {
// Only the node that runs or will run Watcher should update the templates. Otherwise unnecessary put template
// calls would happen
return;
}
// if this node is newer than the master node, we probably need to add the history template, which might be newer than the
// history template the master node has, so we need potentially add new templates despite being not the master node
DiscoveryNode localNode = event.state().getNodes().getLocalNode();
DiscoveryNode masterNode = event.state().getNodes().getMasterNode();
boolean localNodeVersionAfterMaster = localNode.getVersion().after(masterNode.getVersion());
addTemplatesIfMissing(state);
if (event.localNodeMaster() || localNodeVersionAfterMaster) {
addTemplatesIfMissing(state);
}
}
private void addTemplatesIfMissing(ClusterState state) {

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/support/xcontent/WatcherXContentParser.java
View File

@ -13,7 +13,7 @@ import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.watcher.common.secret.Secret;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import java.io.IOException;
import java.time.Clock;

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/watch/Watch.java
View File

@ -20,7 +20,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.watcher.common.secret.Secret;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.watch.clock.HaltedClock;
import org.elasticsearch.xpack.watcher.actions.ActionRegistry;
import org.elasticsearch.xpack.watcher.actions.ActionStatus;

11
plugin/src/test/java/org/elasticsearch/xpack/ml/action/ForecastJobActionRequestTests.java
View File

@ -5,10 +5,13 @@
*/
package org.elasticsearch.xpack.ml.action;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractStreamableXContentTestCase;
import org.elasticsearch.xpack.ml.action.ForecastJobAction.Request;
import java.time.Instant;
public class ForecastJobActionRequestTests extends AbstractStreamableXContentTestCase<Request> {
@Override
@ -24,6 +27,13 @@ public class ForecastJobActionRequestTests extends AbstractStreamableXContentTes
@Override
protected Request createTestInstance() {
Request request = new Request(randomAlphaOfLengthBetween(1, 20));
if (randomBoolean()) {
request.setEndTime(Instant.ofEpochMilli(randomNonNegativeLong()).toString());
}
if (randomBoolean()) {
request.setDuration(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)).getStringRep());
}
return request;
}
@ -31,5 +41,4 @@ public class ForecastJobActionRequestTests extends AbstractStreamableXContentTes
protected Request createBlankInstance() {
return new Request();
}
}

49
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java
View File

@ -184,6 +184,55 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
assertThat(modelSizeStats.getTotalOverFieldCount(), lessThan(60000L));
}
public void testTooManyDistinctOverFields() throws Exception {
Detector.Builder detector = new Detector.Builder("sum", "value");
detector.setOverFieldName("user");
TimeValue bucketSpan = TimeValue.timeValueHours(1);
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(
Arrays.asList(detector.build()));
analysisConfig.setBucketSpan(bucketSpan);
DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setTimeFormat("epoch");
Job.Builder job = new Job.Builder("autodetect-memory-limit-test-too-many-distinct-over-fields");
job.setAnalysisConfig(analysisConfig);
job.setDataDescription(dataDescription);
// Set the memory limit to 300MB
AnalysisLimits limits = new AnalysisLimits(70L, null);
job.setAnalysisLimits(limits);
registerJob(job);
putJob(job);
openJob(job.getId());
long now = Instant.now().getEpochSecond();
long timestamp = now - 15 * bucketSpan.seconds();
int user = 0;
while (timestamp < now) {
List<String> data = new ArrayList<>();
for (int i = 0; i < 10000; i++) {
Map<String, Object> record = new HashMap<>();
record.put("time", timestamp);
record.put("user", user++);
record.put("value", 42.0);
data.add(createJsonRecord(record));
}
postData(job.getId(), data.stream().collect(Collectors.joining()));
timestamp += bucketSpan.seconds();
}
closeJob(job.getId());
// Assert we haven't violated the limit too much
GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0);
ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.OK));
assertThat(modelSizeStats.getModelBytes(), lessThan(45000000L));
assertThat(modelSizeStats.getModelBytes(), greaterThan(35000000L));
assertThat(modelSizeStats.getTotalOverFieldCount(), greaterThan(140000L));
}
private static Map<String, Object> createRecord(long timestamp, String user, String department) {
Map<String, Object> record = new HashMap<>();
record.put("time", timestamp);

24
plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParamsTests.java
View File

@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ml.job.process.autodetect.params;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.job.messages.Messages;
@ -16,15 +17,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
public class ForecastParamsTests extends ESTestCase {
private static ParseField END = new ParseField("end");
public void testDefault_GivesTomorrowTimeInSeconds() {
long nowSecs = System.currentTimeMillis() / 1000;
nowSecs += 60 * 60 * 24;
ForecastParams params = ForecastParams.builder().build();
assertThat(params.getEndTime(), greaterThanOrEqualTo(nowSecs));
assertThat(params.getEndTime(), lessThanOrEqualTo(nowSecs +1));
}
private static ParseField DURATION = new ParseField("duration");
public void test_UnparseableEndTimeThrows() {
ElasticsearchParseException e =
@ -41,4 +34,17 @@ public class ForecastParamsTests extends ESTestCase {
assertThat(end, greaterThanOrEqualTo(nowSecs + 7200));
assertThat(end, lessThanOrEqualTo(nowSecs + 7200 +1));
}
public void testDurationFormats() {
assertEquals(34678L,
ForecastParams.builder().duration(TimeValue.parseTimeValue("34678s", DURATION.getPreferredName())).build().getDuration());
assertEquals(172800L,
ForecastParams.builder().duration(TimeValue.parseTimeValue("2d", DURATION.getPreferredName())).build().getDuration());
}
public void testDurationEndTimeThrows() {
ElasticsearchParseException e = ESTestCase.expectThrows(ElasticsearchParseException.class, () -> ForecastParams.builder()
.endTime("2016-05-01T10:00:00Z", END).duration(TimeValue.parseTimeValue("33d", DURATION.getPreferredName())).build());
assertEquals(Messages.getMessage(Messages.REST_INVALID_DURATION_AND_ENDTIME), e.getMessage());
}
}

3
plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastRequestStatsTests.java
View File

@ -46,6 +46,9 @@ public class ForecastRequestStatsTests extends AbstractSerializingTestCase<Forec
if (randomBoolean()) {
forecastRequestStats.setProcessingTime(randomNonNegativeLong());
}
if (randomBoolean()) {
forecastRequestStats.setMemoryUsage(randomNonNegativeLong());
}
if (randomBoolean()) {
forecastRequestStats.setStatus(randomFrom(ForecastRequestStatus.values()));
}

7
plugin/src/test/java/org/elasticsearch/xpack/security/crypto/tool/SystemKeyToolTests.java
View File

@ -17,7 +17,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.junit.After;
import java.nio.file.FileSystem;
@ -68,7 +67,7 @@ public class SystemKeyToolTests extends CommandTestCase {
execute("-Epath.home=" + homeDir, path.toString());
byte[] bytes = Files.readAllBytes(path);
// TODO: maybe we should actually check the key is...i dunno...valid?
assertEquals(CryptoService.KEY_SIZE / 8, bytes.length);
assertEquals(SystemKeyTool.KEY_SIZE / 8, bytes.length);
Set<PosixFilePermission> perms = Files.getPosixFilePermissions(path);
assertTrue(perms.toString(), perms.contains(PosixFilePermission.OWNER_READ));
@ -83,7 +82,7 @@ public class SystemKeyToolTests extends CommandTestCase {
Files.createDirectories(xpackConf);
execute("-Epath.home=" + homeDir.toString());
byte[] bytes = Files.readAllBytes(xpackConf.resolve("system_key"));
assertEquals(CryptoService.KEY_SIZE / 8, bytes.length);
assertEquals(SystemKeyTool.KEY_SIZE / 8, bytes.length);
}
public void testGenerateDefaultPath() throws Exception {
@ -92,7 +91,7 @@ public class SystemKeyToolTests extends CommandTestCase {
Files.createDirectories(keyPath.getParent());
execute("-Epath.home=" + homeDir.toString());
byte[] bytes = Files.readAllBytes(keyPath);
assertEquals(CryptoService.KEY_SIZE / 8, bytes.length);
assertEquals(SystemKeyTool.KEY_SIZE / 8, bytes.length);
}
public void testThatSystemKeyMayOnlyBeReadByOwner() throws Exception {

4
plugin/src/test/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheckTests.java
View File

@ -11,7 +11,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoServiceTests;
public class EncryptSensitiveDataBootstrapCheckTests extends ESTestCase {
@ -35,7 +35,7 @@ public class EncryptSensitiveDataBootstrapCheckTests extends ESTestCase {
public void testKeyInKeystore() {
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), CryptoService.generateKey());
secureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), CryptoServiceTests.generateKey());
Settings settings = Settings.builder()
.put("path.home", createTempDir())
.put(Watcher.ENCRYPT_SENSITIVE_DATA_SETTING.getKey(), true)

2
plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
View File

@ -72,6 +72,6 @@ public class WatcherPluginTests extends ESTestCase {
watcher.onIndexModule(indexModule);
// also no component creation if not enabled
assertThat(watcher.createComponents(null, null, null, null, null, null, null, null, null, null, null), hasSize(0));
assertThat(watcher.createComponents(null, null, null, null, null, null, null, null), hasSize(0));
}
}

21
plugin/src/test/java/org/elasticsearch/xpack/security/crypto/CryptoServiceTests.java → plugin/src/test/java/org/elasticsearch/xpack/watcher/crypto/CryptoServiceTests.java
View File

@ -3,16 +3,19 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.security.crypto;
import java.util.Arrays;
package org.elasticsearch.xpack.watcher.crypto;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.watcher.Watcher;
import org.junit.Before;
import javax.crypto.KeyGenerator;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
@ -22,7 +25,7 @@ public class CryptoServiceTests extends ESTestCase {
@Before
public void init() throws Exception {
MockSecureSettings mockSecureSettings = new MockSecureSettings();
mockSecureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), CryptoService.generateKey());
mockSecureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), generateKey());
settings = Settings.builder()
.setSecureSettings(mockSecureSettings)
.build();
@ -49,4 +52,14 @@ public class CryptoServiceTests extends ESTestCase {
assertThat(service.isEncrypted(randomAlphaOfLengthBetween(0, 100).toCharArray()), is(false));
assertThat(service.isEncrypted(service.encrypt(randomAlphaOfLength(10).toCharArray())), is(true));
}
public static byte[] generateKey() {
try {
KeyGenerator generator = KeyGenerator.getInstance(CryptoService.KEY_ALGO);
generator.init(CryptoService.KEY_SIZE);
return generator.generateKey().getEncoded();
} catch (NoSuchAlgorithmException e) {
throw new ElasticsearchException("failed to generate key", e);
}
}
}

10
plugin/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailSecretsIntegrationTests.java
View File

@ -9,13 +9,14 @@ import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
import org.elasticsearch.xpack.watcher.notification.email.support.EmailServer;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.condition.AlwaysCondition;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoServiceTests;
import org.elasticsearch.xpack.watcher.execution.ActionExecutionMode;
import org.elasticsearch.xpack.watcher.notification.email.support.EmailServer;
import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
import org.elasticsearch.xpack.watcher.transport.actions.execute.ExecuteWatchResponse;
@ -32,7 +33,6 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.emailAction;
import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder;
import static org.elasticsearch.xpack.watcher.input.InputBuilders.simpleInput;
import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
@ -64,7 +64,7 @@ public class EmailSecretsIntegrationTests extends AbstractWatcherIntegrationTest
if (encryptSensitiveData == null) {
encryptSensitiveData = randomBoolean();
if (encryptSensitiveData) {
encryptionKey = CryptoService.generateKey();
encryptionKey = CryptoServiceTests.generateKey();
}
}
Settings.Builder builder = Settings.builder()

83
plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.watcher.support;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
@ -18,6 +19,8 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
@ -44,6 +47,7 @@ import static org.mockito.Matchers.same;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verifyZeroInteractions;
public class WatcherIndexTemplateRegistryTests extends ESTestCase {
@ -73,36 +77,18 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
registry = new WatcherIndexTemplateRegistry(Settings.EMPTY, clusterService, threadPool, internalClient);
}
private ClusterChangedEvent createClusterChangedEvent(List<String> existingTemplateNames) {
ClusterChangedEvent event = mock(ClusterChangedEvent.class);
when(event.localNodeMaster()).thenReturn(true);
ClusterState cs = mock(ClusterState.class);
ClusterBlocks clusterBlocks = mock(ClusterBlocks.class);
when(clusterBlocks.hasGlobalBlock(eq(GatewayService.STATE_NOT_RECOVERED_BLOCK))).thenReturn(false);
when(cs.blocks()).thenReturn(clusterBlocks);
when(event.state()).thenReturn(cs);
MetaData metaData = mock(MetaData.class);
ImmutableOpenMap.Builder<String, IndexTemplateMetaData> indexTemplates = ImmutableOpenMap.builder();
for (String name : existingTemplateNames) {
indexTemplates.put(name, mock(IndexTemplateMetaData.class));
}
when(metaData.getTemplates()).thenReturn(indexTemplates.build());
when(cs.metaData()).thenReturn(metaData);
return event;
}
public void testThatNonExistingTemplatesAreAddedImmediately() {
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList());
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes);
registry.clusterChanged(event);
ArgumentCaptor<PutIndexTemplateRequest> argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
verify(client, times(3)).execute(anyObject(), argumentCaptor.capture(), anyObject());
// now delete one template from the cluster state and lets retry
ClusterChangedEvent newEvent = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME,
WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME));
WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME), nodes);
registry.clusterChanged(newEvent);
verify(client, times(4)).execute(anyObject(), argumentCaptor.capture(), anyObject());
}
@ -117,6 +103,57 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
".triggered_watches", ".watches", "whatever", "else")), is(true));
}
// if a node is newer than the master node, the template needs to be applied as well
// otherwise a rolling upgrade would not work as expected, when the node has a .watches shard on it
public void testThatTemplatesAreAppliedOnNewerNodes() {
DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.V_6_0_0);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build();
ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME,
WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes);
registry.clusterChanged(event);
ArgumentCaptor<PutIndexTemplateRequest> argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
verify(client, times(1)).execute(anyObject(), argumentCaptor.capture(), anyObject());
assertThat(argumentCaptor.getValue().name(), is(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME));
}
public void testThatTemplatesAreNotAppliedOnSameVersionNodes() {
DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build();
ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME,
WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes);
registry.clusterChanged(event);
verifyZeroInteractions(client);
}
private ClusterChangedEvent createClusterChangedEvent(List<String> existingTemplateNames, DiscoveryNodes nodes) {
ClusterChangedEvent event = mock(ClusterChangedEvent.class);
when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster());
ClusterState cs = mock(ClusterState.class);
ClusterBlocks clusterBlocks = mock(ClusterBlocks.class);
when(clusterBlocks.hasGlobalBlock(eq(GatewayService.STATE_NOT_RECOVERED_BLOCK))).thenReturn(false);
when(cs.blocks()).thenReturn(clusterBlocks);
when(event.state()).thenReturn(cs);
when(cs.getNodes()).thenReturn(nodes);
MetaData metaData = mock(MetaData.class);
ImmutableOpenMap.Builder<String, IndexTemplateMetaData> indexTemplates = ImmutableOpenMap.builder();
for (String name : existingTemplateNames) {
indexTemplates.put(name, mock(IndexTemplateMetaData.class));
}
when(metaData.getTemplates()).thenReturn(indexTemplates.build());
when(cs.metaData()).thenReturn(metaData);
return event;
}
private ClusterState createClusterState(String ... existingTemplates) {
MetaData.Builder metaDataBuilder = MetaData.builder();
for (String templateName : existingTemplates) {

9
plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
View File

@ -11,13 +11,14 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.test.http.MockResponse;
import org.elasticsearch.test.http.MockWebServer;
import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.ApplicableBasicAuth;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.condition.AlwaysCondition;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoServiceTests;
import org.elasticsearch.xpack.watcher.execution.ActionExecutionMode;
import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@ -72,7 +73,7 @@ public class HttpSecretsIntegrationTests extends AbstractWatcherIntegrationTestC
if (encryptSensitiveData == null) {
encryptSensitiveData = randomBoolean();
if (encryptSensitiveData) {
encryptionKey = CryptoService.generateKey();
encryptionKey = CryptoServiceTests.generateKey();
}
}
if (encryptSensitiveData) {

4
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_monitoring.yml
View File

@ -6,6 +6,10 @@ setup:
---
"Index monitoring data and search on the mixed cluster":
- skip:
version: "all"
reason: "AwaitsFix'ing, see x-pack-elasticsearch #2948"
- do:
search:
index: .monitoring-kibana-*

4
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_monitoring.yml
View File

@ -6,6 +6,10 @@ setup:
---
"Index monitoring data and search on the upgraded cluster":
- skip:
version: "all"
reason: "AwaitsFix'ing, see x-pack-elasticsearch #2948"
- do:
search:
index: .monitoring-kibana-*

5
qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherTemplateIT.java → qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java
View File

@ -6,7 +6,6 @@
package org.elasticsearch.smoketest;
import com.fasterxml.jackson.core.io.JsonStringEncoder;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
@ -15,9 +14,9 @@ import org.elasticsearch.script.ScriptEngine;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.mustache.MustacheScriptEngine;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
import org.elasticsearch.xpack.watcher.Watcher;
import org.junit.Before;
import java.io.IOException;
@ -31,7 +30,7 @@ import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
public class WatcherTemplateIT extends ESTestCase {
public class WatcherTemplateTests extends ESTestCase {
private TextTemplateEngine textTemplateEngine;