honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-26 06:46:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source 
Original commit: elastic/x-pack-elasticsearch@f06f403c04
This commit is contained in:
lcawley 2017-11-17 10:03:54 -08:00
commit 5dfb35e2a8
34 changed files with 406 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java
View File

@ -66,13 +66,10 @@ import org.elasticsearch.xpack.deprecation.Deprecation;
import org.elasticsearch.xpack.extensions.XPackExtension;
import org.elasticsearch.xpack.extensions.XPackExtensionsService;
import org.elasticsearch.xpack.graph.Graph;
import org.elasticsearch.xpack.graph.GraphFeatureSet;
import org.elasticsearch.xpack.logstash.Logstash;
import org.elasticsearch.xpack.logstash.LogstashFeatureSet;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.MachineLearningFeatureSet;
import org.elasticsearch.xpack.monitoring.Monitoring;
import org.elasticsearch.xpack.monitoring.MonitoringFeatureSet;
import org.elasticsearch.xpack.persistent.CompletionPersistentTaskAction;
import org.elasticsearch.xpack.persistent.PersistentTasksClusterService;
import org.elasticsearch.xpack.persistent.PersistentTasksExecutor;
@ -85,50 +82,23 @@ import org.elasticsearch.xpack.rest.action.RestXPackInfoAction;
import org.elasticsearch.xpack.rest.action.RestXPackUsageAction;
import org.elasticsearch.xpack.security.InternalClient;
import org.elasticsearch.xpack.security.Security;
import org.elasticsearch.xpack.security.SecurityFeatureSet;
import org.elasticsearch.xpack.security.authc.AuthenticationService;
import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.ssl.SSLConfigurationReloader;
import org.elasticsearch.xpack.ssl.SSLService;
import org.elasticsearch.xpack.upgrade.Upgrade;
import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.WatcherFeatureSet;
import org.elasticsearch.xpack.watcher.common.http.HttpClient;
import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
import org.elasticsearch.xpack.watcher.common.http.HttpSettings;
import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory;
import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
import org.elasticsearch.xpack.watcher.notification.email.Account;
import org.elasticsearch.xpack.watcher.notification.email.EmailService;
import org.elasticsearch.xpack.watcher.notification.email.attachment.DataAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.EmailAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.EmailAttachmentsParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.HttpEmailAttachementParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.ReportingAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.support.BodyPartSource;
import org.elasticsearch.xpack.watcher.notification.hipchat.HipChatService;
import org.elasticsearch.xpack.watcher.notification.jira.JiraService;
import org.elasticsearch.xpack.watcher.notification.pagerduty.PagerDutyAccount;
import org.elasticsearch.xpack.watcher.notification.pagerduty.PagerDutyService;
import org.elasticsearch.xpack.watcher.notification.slack.SlackService;
import javax.security.auth.DestroyFailedException;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.security.AccessController;
import java.security.GeneralSecurityException;
import java.security.PrivilegedAction;
import java.time.Clock;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -139,8 +109,6 @@ import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.elasticsearch.xpack.watcher.Watcher.ENCRYPT_SENSITIVE_DATA_SETTING;
public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin {
public static final String NAME = "x-pack";
@ -200,9 +168,6 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
throw bogus; // some other bug
}
}
// some classes need to have their own clinit blocks
BodyPartSource.init();
Account.init();
}
protected final Settings settings;
@ -300,29 +265,8 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
}
components.addAll(monitoring.createComponents(internalClient, threadPool, clusterService, licenseService, sslService));
final CryptoService cryptoService;
try {
cryptoService = ENCRYPT_SENSITIVE_DATA_SETTING.get(settings) ? new CryptoService(settings) : null;
} catch (IOException e) {
throw new UncheckedIOException(e);
}
// watcher http stuff
Map<String, HttpAuthFactory> httpAuthFactories = new HashMap<>();
httpAuthFactories.put(BasicAuth.TYPE, new BasicAuthFactory(cryptoService));
// TODO: add more auth types, or remove this indirection
HttpAuthRegistry httpAuthRegistry = new HttpAuthRegistry(httpAuthFactories);
HttpRequestTemplate.Parser httpTemplateParser = new HttpRequestTemplate.Parser(httpAuthRegistry);
components.add(httpTemplateParser);
final HttpClient httpClient = new HttpClient(settings, httpAuthRegistry, sslService);
components.add(httpClient);
Collection<Object> notificationComponents = createNotificationComponents(clusterService.getClusterSettings(), httpClient,
httpTemplateParser, scriptService, httpAuthRegistry, cryptoService);
components.addAll(notificationComponents);
components.addAll(watcher.createComponents(getClock(), scriptService, internalClient, licenseState,
httpClient, httpTemplateParser, threadPool, clusterService, cryptoService, xContentRegistry, components));
components.addAll(watcher.createComponents(getClock(), scriptService, internalClient, licenseState, threadPool, clusterService,
xContentRegistry, sslService));
PersistentTasksService persistentTasksService = new PersistentTasksService(settings, clusterService, threadPool, internalClient);
@ -347,28 +291,6 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
return components;
}
private Collection<Object> createNotificationComponents(ClusterSettings clusterSettings, HttpClient httpClient,
HttpRequestTemplate.Parser httpTemplateParser, ScriptService scriptService,
HttpAuthRegistry httpAuthRegistry, CryptoService cryptoService) {
List<Object> components = new ArrayList<>();
components.add(new EmailService(settings, cryptoService, clusterSettings));
components.add(new HipChatService(settings, httpClient, clusterSettings));
components.add(new JiraService(settings, httpClient, clusterSettings));
components.add(new SlackService(settings, httpClient, clusterSettings));
components.add(new PagerDutyService(settings, httpClient, clusterSettings));
TextTemplateEngine textTemplateEngine = new TextTemplateEngine(settings, scriptService);
components.add(textTemplateEngine);
Map<String, EmailAttachmentParser> parsers = new HashMap<>();
parsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, httpTemplateParser, textTemplateEngine));
parsers.put(DataAttachmentParser.TYPE, new DataAttachmentParser());
parsers.put(ReportingAttachmentParser.TYPE, new ReportingAttachmentParser(settings, httpClient, textTemplateEngine,
httpAuthRegistry));
components.add(new EmailAttachmentsParser(parsers));
return components;
}
@Override
public Settings additionalSettings() {
Settings.Builder builder = Settings.builder();
@ -395,7 +317,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
@Override
public List<ScriptContext> getContexts() {
return Arrays.asList(Watcher.SCRIPT_SEARCH_CONTEXT, Watcher.SCRIPT_EXECUTABLE_CONTEXT, Watcher.SCRIPT_TEMPLATE_CONTEXT);
return watcher.getContexts();
}
@Override
@ -413,30 +335,13 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
// we add the `xpack.version` setting to all internal indices
settings.add(Setting.simpleString("index.xpack.version", Setting.Property.IndexScope));
// notification services
settings.add(SlackService.SLACK_ACCOUNT_SETTING);
settings.add(EmailService.EMAIL_ACCOUNT_SETTING);
settings.add(HipChatService.HIPCHAT_ACCOUNT_SETTING);
settings.add(JiraService.JIRA_ACCOUNT_SETTING);
settings.add(PagerDutyService.PAGERDUTY_ACCOUNT_SETTING);
settings.add(ReportingAttachmentParser.RETRIES_SETTING);
settings.add(ReportingAttachmentParser.INTERVAL_SETTING);
// http settings
settings.addAll(HttpSettings.getSettings());
return settings;
}
@Override
public List<String> getSettingsFilter() {
List<String> filters = new ArrayList<>();
filters.add("xpack.notification.email.account.*.smtp.password");
filters.add("xpack.notification.jira.account.*.password");
filters.add("xpack.notification.slack.account.*.url");
filters.add("xpack.notification.pagerduty.account.*.url");
filters.add("xpack.notification.pagerduty." + PagerDutyAccount.SERVICE_KEY_SETTING);
filters.add("xpack.notification.pagerduty.account.*." + PagerDutyAccount.SERVICE_KEY_SETTING);
filters.add("xpack.notification.hipchat.account.*.auth_token");
filters.addAll(watcher.getSettingsFilter());
filters.addAll(security.getSettingsFilter(extensionsService));
filters.addAll(monitoring.getSettingsFilter());
if (transportClientMode == false) {
@ -522,16 +427,13 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
@Override
public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, SECURITY, SecurityFeatureSet.Usage::new));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, WATCHER, WatcherFeatureSet.Usage::new));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, MONITORING, MonitoringFeatureSet.Usage::new));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, GRAPH, GraphFeatureSet.Usage::new));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, MACHINE_LEARNING, MachineLearningFeatureSet.Usage::new));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, LOGSTASH, LogstashFeatureSet.Usage::new));
entries.addAll(watcher.getNamedWriteables());
entries.addAll(machineLearning.getNamedWriteables());
entries.addAll(licensing.getNamedWriteables());
entries.addAll(Security.getNamedWriteables());
entries.addAll(Monitoring.getNamedWriteables());
entries.addAll(Graph.getNamedWriteables());
return entries;
}

11
plugin/src/main/java/org/elasticsearch/xpack/graph/Graph.java
View File

@ -10,20 +10,22 @@ import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.graph.action.GraphExploreAction;
import org.elasticsearch.xpack.graph.action.TransportGraphExploreAction;
import org.elasticsearch.xpack.graph.rest.action.RestGraphAction;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@ -31,8 +33,9 @@ import java.util.function.Supplier;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.XPackPlugin.GRAPH;
public class Graph extends Plugin implements ActionPlugin {
public class Graph implements ActionPlugin {
public static final String NAME = "graph";
protected final boolean enabled;
@ -65,4 +68,8 @@ public class Graph extends Plugin implements ActionPlugin {
}
return singletonList(new RestGraphAction(settings, restController));
}
public static Collection<? extends NamedWriteableRegistry.Entry> getNamedWriteables() {
return Arrays.asList(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, GRAPH, GraphFeatureSet.Usage::new));
}
}

7
plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ExecutorBuilder;
import org.elasticsearch.threadpool.FixedExecutorBuilder;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.ml.action.CloseJobAction;
@ -145,6 +146,7 @@ import java.util.List;
import java.util.function.Supplier;
import static java.util.Collections.emptyList;
import static org.elasticsearch.xpack.XPackPlugin.MACHINE_LEARNING;
public class MachineLearning implements ActionPlugin {
public static final String NAME = "ml";
@ -270,7 +272,10 @@ public class MachineLearning implements ActionPlugin {
new NamedWriteableRegistry.Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME,
PersistentTasksNodeService.Status::new),
new NamedWriteableRegistry.Entry(Task.Status.class, JobTaskStatus.NAME, JobTaskStatus::new),
new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream)
new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream),
// feature set
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, MACHINE_LEARNING, MachineLearningFeatureSet.Usage::new)
);
}

24
plugin/src/main/java/org/elasticsearch/xpack/ml/action/ForecastJobAction.java
View File

@ -19,6 +19,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -56,12 +57,14 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
public static class Request extends TransportJobTaskAction.JobTaskRequest<Request> implements ToXContentObject {
public static final ParseField END_TIME = new ParseField("end");
public static final ParseField DURATION = new ParseField("duration");
private static final ObjectParser<Request, Void> PARSER = new ObjectParser<>(NAME, Request::new);
static {
PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID);
PARSER.declareString(Request::setEndTime, END_TIME);
PARSER.declareString(Request::setDuration, DURATION);
}
public static Request parseRequest(String jobId, XContentParser parser) {
@ -73,6 +76,7 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
}
private String endTime;
private TimeValue duration;
Request() {
}
@ -89,21 +93,31 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
this.endTime = endTime;
}
public TimeValue getDuration() {
return duration;
}
public void setDuration(String duration) {
this.duration = TimeValue.parseTimeValue(duration, DURATION.getPreferredName());
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
this.endTime = in.readOptionalString();
this.duration = in.readOptionalWriteable(TimeValue::new);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeOptionalString(endTime);
out.writeOptionalWriteable(duration);
}
@Override
public int hashCode() {
return Objects.hash(jobId, endTime);
return Objects.hash(jobId, endTime, duration);
}
@Override
@ -115,7 +129,7 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
return false;
}
Request other = (Request) obj;
return Objects.equals(jobId, other.jobId) && Objects.equals(endTime, other.endTime);
return Objects.equals(jobId, other.jobId) && Objects.equals(endTime, other.endTime) && Objects.equals(duration, other.duration);
}
@Override
@ -125,6 +139,9 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
if (endTime != null) {
builder.field(END_TIME.getPreferredName(), endTime);
}
if (duration != null) {
builder.field(DURATION.getPreferredName(), duration.getStringRep());
}
builder.endObject();
return builder;
}
@ -219,6 +236,9 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
if (request.getEndTime() != null) {
paramsBuilder.endTime(request.getEndTime(), END_TIME);
}
if (request.getDuration() != null) {
paramsBuilder.duration(request.getDuration());
}
ForecastParams params = paramsBuilder.build();
processManager.forecastJob(task, params, e -> {

1
plugin/src/main/java/org/elasticsearch/xpack/ml/job/messages/Messages.java
View File

@ -164,6 +164,7 @@ public final class Messages {
"Model snapshot ''{0}'' is the active snapshot for job ''{1}'', so cannot be deleted";
public static final String REST_INVALID_DATETIME_PARAMS =
"Query param [{0}] with value [{1}] cannot be parsed as a date or converted to a number (epoch).";
public static final String REST_INVALID_DURATION_AND_ENDTIME = "Specify either duration or end time";
public static final String REST_INVALID_FLUSH_PARAMS_MISSING = "Invalid flush parameters: ''{0}'' has not been specified.";
public static final String REST_INVALID_FLUSH_PARAMS_UNEXPECTED = "Invalid flush parameters: unexpected ''{0}''.";
public static final String REST_JOB_NOT_CLOSED_REVERT = "Can only revert to a model snapshot when the job is closed.";

37
plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParams.java
View File

@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ml.job.process.autodetect.params;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.joda.DateMathParser;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.xpack.ml.job.messages.Messages;
@ -16,11 +17,13 @@ import java.util.Objects;
public class ForecastParams {
private final long endTime;
private final long duration;
private final long id;
private ForecastParams(long id, long endTime) {
private ForecastParams(long id, long endTime, long duration) {
this.id = id;
this.endTime = endTime;
this.duration = duration;
}
/**
@ -31,6 +34,14 @@ public class ForecastParams {
return endTime;
}
/**
* The forecast duration in seconds
* @return The duration in seconds
*/
public long getDuration() {
return duration;
}
/**
* The forecast id
*
@ -42,7 +53,7 @@ public class ForecastParams {
@Override
public int hashCode() {
return Objects.hash(id, endTime);
return Objects.hash(id, endTime, duration);
}
@Override
@ -54,27 +65,24 @@ public class ForecastParams {
return false;
}
ForecastParams other = (ForecastParams) obj;
return Objects.equals(id, other.id) && Objects.equals(endTime, other.endTime);
return Objects.equals(id, other.id) && Objects.equals(endTime, other.endTime) && Objects.equals(duration, other.duration);
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private long endTimeEpochSecs;
private long durationSecs;
private long startTime;
private long forecastId;
private Builder() {
startTime = System.currentTimeMillis();
endTimeEpochSecs = tomorrow(startTime);
endTimeEpochSecs = 0;
forecastId = generateId();
}
static long tomorrow(long now) {
return (now / 1000) + (60 * 60 * 24);
durationSecs = 0;
}
private long generateId() {
@ -94,8 +102,17 @@ public class ForecastParams {
return this;
}
public Builder duration(TimeValue duration) {
durationSecs = duration.seconds();
return this;
}
public ForecastParams build() {
return new ForecastParams(forecastId, endTimeEpochSecs);
if (endTimeEpochSecs != 0 && durationSecs != 0) {
throw new ElasticsearchParseException(Messages.getMessage(Messages.REST_INVALID_DURATION_AND_ENDTIME));
}
return new ForecastParams(forecastId, endTimeEpochSecs, durationSecs);
}
}
}

12
plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriter.java
View File

@ -153,9 +153,15 @@ public class ControlMsgToProcessWriter {
public void writeForecastMessage(ForecastParams params) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.field("forecast_id", params.getId())
.field("end_time", params.getEndTime())
.endObject();
.field("forecast_id", params.getId());
if (params.getEndTime() != 0) {
builder.field("end_time", params.getEndTime());
}
if (params.getDuration() != 0) {
builder.field("duration", params.getDuration());
}
builder.endObject();
writeMessage(FORECAST_MESSAGE_CODE + builder.string());
fillCommandBuffer();

3
plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestForecastJobAction.java
View File

@ -43,6 +43,9 @@ public class RestForecastJobAction extends BaseRestHandler {
if (restRequest.hasParam(ForecastJobAction.Request.END_TIME.getPreferredName())) {
request.setEndTime(restRequest.param(ForecastJobAction.Request.END_TIME.getPreferredName()));
}
if (restRequest.hasParam(ForecastJobAction.Request.DURATION.getPreferredName())) {
request.setDuration(restRequest.param(ForecastJobAction.Request.DURATION.getPreferredName()));
}
}
return channel -> client.execute(ForecastJobAction.INSTANCE, request, new RestToXContentListener<>(channel));

7
plugin/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.util.Providers;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Setting;
@ -24,6 +25,7 @@ import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.monitoring.action.MonitoringBulkAction;
@ -59,6 +61,7 @@ import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.common.settings.Setting.boolSetting;
import static org.elasticsearch.common.settings.Setting.timeSetting;
import static org.elasticsearch.xpack.XPackPlugin.MONITORING;
/**
* This class activates/deactivates the monitoring modules depending if we're running a node client, transport client or tribe client:
@ -111,6 +114,10 @@ public class Monitoring implements ActionPlugin {
this.tribeNode = XPackPlugin.isTribeNode(settings);
}
public static Collection<? extends NamedWriteableRegistry.Entry> getNamedWriteables() {
return Arrays.asList(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, MONITORING, MonitoringFeatureSet.Usage::new));
}
boolean isEnabled() {
return enabled;
}

2
plugin/src/main/java/org/elasticsearch/xpack/security/Security.java
View File

@ -72,6 +72,7 @@ import org.elasticsearch.transport.TransportInterceptor;
import org.elasticsearch.transport.TransportRequest;
import org.elasticsearch.transport.TransportRequestHandler;
import org.elasticsearch.watcher.ResourceWatcherService;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.extensions.XPackExtension;
@ -889,6 +890,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
entries.add(new NamedWriteableRegistry.Entry(ClusterState.Custom.class, TokenMetaData.TYPE, TokenMetaData::new));
entries.add(new NamedWriteableRegistry.Entry(NamedDiff.class, TokenMetaData.TYPE, TokenMetaData::readDiffFrom));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackPlugin.SECURITY, SecurityFeatureSet.Usage::new));
entries.addAll(Arrays.asList(ExpressionParser.NAMED_WRITEABLES));
return entries;
}

27
plugin/src/main/java/org/elasticsearch/xpack/security/crypto/tool/SystemKeyTool.java
View File

@ -7,8 +7,9 @@ package org.elasticsearch.xpack.security.crypto.tool;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.elasticsearch.cli.ExitCodes;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cli.EnvironmentAwareCommand;
import org.elasticsearch.cli.ExitCodes;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cli.UserException;
import org.elasticsearch.common.SuppressForbidden;
@ -16,19 +17,24 @@ import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.env.Environment;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.PosixFileAttributeView;
import java.nio.file.attribute.PosixFilePermission;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.Locale;
import java.util.Set;
public class SystemKeyTool extends EnvironmentAwareCommand {
static final String KEY_ALGO = "HmacSHA512";
static final int KEY_SIZE = 1024;
private final OptionSpec<String> arguments;
SystemKeyTool() {
@ -67,7 +73,7 @@ public class SystemKeyTool extends EnvironmentAwareCommand {
// write the key
terminal.println(Terminal.Verbosity.VERBOSE, "generating...");
byte[] key = CryptoService.generateKey();
byte[] key = generateKey();
terminal.println(String.format(Locale.ROOT, "Storing generated key in [%s]...", keyPath.toAbsolutePath()));
Files.write(keyPath, key, StandardOpenOption.CREATE_NEW);
@ -80,6 +86,21 @@ public class SystemKeyTool extends EnvironmentAwareCommand {
}
}
static byte[] generateKey() {
return generateSecretKey(KEY_SIZE).getEncoded();
}
static SecretKey generateSecretKey(int keyLength) {
try {
KeyGenerator generator = KeyGenerator.getInstance(KEY_ALGO);
generator.init(keyLength);
return generator.generateKey();
} catch (NoSuchAlgorithmException e) {
throw new ElasticsearchException("failed to generate key", e);
}
}
@SuppressForbidden(reason = "Parsing command line path")
private static Path parsePath(String path) {
return PathUtils.get(path);

145
plugin/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
View File

@ -47,19 +47,11 @@ import org.elasticsearch.script.TemplateScript;
import org.elasticsearch.threadpool.ExecutorBuilder;
import org.elasticsearch.threadpool.FixedExecutorBuilder;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.watcher.common.http.HttpClient;
import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
import org.elasticsearch.xpack.watcher.notification.email.EmailService;
import org.elasticsearch.xpack.watcher.notification.email.attachment.EmailAttachmentsParser;
import org.elasticsearch.xpack.watcher.notification.hipchat.HipChatService;
import org.elasticsearch.xpack.watcher.notification.jira.JiraService;
import org.elasticsearch.xpack.watcher.notification.pagerduty.PagerDutyService;
import org.elasticsearch.xpack.watcher.notification.slack.SlackService;
import org.elasticsearch.xpack.security.InternalClient;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.ssl.SSLService;
import org.elasticsearch.xpack.watcher.actions.ActionFactory;
import org.elasticsearch.xpack.watcher.actions.ActionRegistry;
import org.elasticsearch.xpack.watcher.actions.email.EmailAction;
@ -79,6 +71,14 @@ import org.elasticsearch.xpack.watcher.actions.slack.SlackActionFactory;
import org.elasticsearch.xpack.watcher.actions.webhook.WebhookAction;
import org.elasticsearch.xpack.watcher.actions.webhook.WebhookActionFactory;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.common.http.HttpClient;
import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
import org.elasticsearch.xpack.watcher.common.http.HttpSettings;
import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory;
import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthRegistry;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuthFactory;
import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
import org.elasticsearch.xpack.watcher.condition.AlwaysCondition;
import org.elasticsearch.xpack.watcher.condition.ArrayCompareCondition;
import org.elasticsearch.xpack.watcher.condition.CompareCondition;
@ -86,6 +86,7 @@ import org.elasticsearch.xpack.watcher.condition.ConditionFactory;
import org.elasticsearch.xpack.watcher.condition.ConditionRegistry;
import org.elasticsearch.xpack.watcher.condition.NeverCondition;
import org.elasticsearch.xpack.watcher.condition.ScriptCondition;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.execution.AsyncTriggerEventConsumer;
import org.elasticsearch.xpack.watcher.execution.ExecutionService;
import org.elasticsearch.xpack.watcher.execution.InternalWatchExecutor;
@ -105,6 +106,19 @@ import org.elasticsearch.xpack.watcher.input.search.SearchInput;
import org.elasticsearch.xpack.watcher.input.search.SearchInputFactory;
import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
import org.elasticsearch.xpack.watcher.input.simple.SimpleInputFactory;
import org.elasticsearch.xpack.watcher.notification.email.Account;
import org.elasticsearch.xpack.watcher.notification.email.EmailService;
import org.elasticsearch.xpack.watcher.notification.email.attachment.DataAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.EmailAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.EmailAttachmentsParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.HttpEmailAttachementParser;
import org.elasticsearch.xpack.watcher.notification.email.attachment.ReportingAttachmentParser;
import org.elasticsearch.xpack.watcher.notification.email.support.BodyPartSource;
import org.elasticsearch.xpack.watcher.notification.hipchat.HipChatService;
import org.elasticsearch.xpack.watcher.notification.jira.JiraService;
import org.elasticsearch.xpack.watcher.notification.pagerduty.PagerDutyAccount;
import org.elasticsearch.xpack.watcher.notification.pagerduty.PagerDutyService;
import org.elasticsearch.xpack.watcher.notification.slack.SlackService;
import org.elasticsearch.xpack.watcher.rest.action.RestAckWatchAction;
import org.elasticsearch.xpack.watcher.rest.action.RestActivateWatchAction;
import org.elasticsearch.xpack.watcher.rest.action.RestDeleteWatchAction;
@ -155,7 +169,9 @@ import org.elasticsearch.xpack.watcher.watch.Watch;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.time.Clock;
import java.util.ArrayList;
import java.util.Arrays;
@ -170,12 +186,17 @@ import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import static java.util.Collections.emptyList;
public class Watcher implements ActionPlugin {
static {
// some classes need to have their own clinit blocks
BodyPartSource.init();
Account.init();
}
public static final Setting<String> INDEX_WATCHER_TEMPLATE_VERSION_SETTING =
new Setting<>("index.xpack.watcher.template.version", "", Function.identity(), Setting.Property.IndexScope);
public static final Setting<Boolean> ENCRYPT_SENSITIVE_DATA_SETTING =
@ -199,6 +220,7 @@ public class Watcher implements ActionPlugin {
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, WatcherMetaData.TYPE, WatcherMetaData::new));
entries.add(new NamedWriteableRegistry.Entry(NamedDiff.class, WatcherMetaData.TYPE, WatcherMetaData::readDiffFrom));
entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackPlugin.WATCHER, WatcherFeatureSet.Usage::new));
return entries;
}
@ -225,13 +247,44 @@ public class Watcher implements ActionPlugin {
public Collection<Object> createComponents(Clock clock, ScriptService scriptService, InternalClient internalClient,
XPackLicenseState licenseState,
HttpClient httpClient, HttpRequestTemplate.Parser httpTemplateParser,
ThreadPool threadPool, ClusterService clusterService, CryptoService cryptoService,
NamedXContentRegistry xContentRegistry, Collection<Object> components) {
ThreadPool threadPool, ClusterService clusterService,
NamedXContentRegistry xContentRegistry, SSLService sslService) {
if (enabled == false) {
return Collections.emptyList();
}
final CryptoService cryptoService;
try {
cryptoService = ENCRYPT_SENSITIVE_DATA_SETTING.get(settings) ? new CryptoService(settings) : null;
} catch (IOException e) {
throw new UncheckedIOException(e);
}
// http client
Map<String, HttpAuthFactory> httpAuthFactories = new HashMap<>();
httpAuthFactories.put(BasicAuth.TYPE, new BasicAuthFactory(cryptoService));
// TODO: add more auth types, or remove this indirection
HttpAuthRegistry httpAuthRegistry = new HttpAuthRegistry(httpAuthFactories);
HttpRequestTemplate.Parser httpTemplateParser = new HttpRequestTemplate.Parser(httpAuthRegistry);
final HttpClient httpClient = new HttpClient(settings, httpAuthRegistry, sslService);
// notification
EmailService emailService = new EmailService(settings, cryptoService, clusterService.getClusterSettings());
HipChatService hipChatService = new HipChatService(settings, httpClient, clusterService.getClusterSettings());
JiraService jiraService = new JiraService(settings, httpClient, clusterService.getClusterSettings());
SlackService slackService = new SlackService(settings, httpClient, clusterService.getClusterSettings());
PagerDutyService pagerDutyService = new PagerDutyService(settings, httpClient, clusterService.getClusterSettings());
TextTemplateEngine templateEngine = new TextTemplateEngine(settings, scriptService);
Map<String, EmailAttachmentParser> emailAttachmentParsers = new HashMap<>();
emailAttachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, httpTemplateParser,
templateEngine));
emailAttachmentParsers.put(DataAttachmentParser.TYPE, new DataAttachmentParser());
emailAttachmentParsers.put(ReportingAttachmentParser.TYPE, new ReportingAttachmentParser(settings, httpClient, templateEngine,
httpAuthRegistry));
EmailAttachmentsParser emailAttachmentsParser = new EmailAttachmentsParser(emailAttachmentParsers);
// conditions
final Map<String, ConditionFactory> parsers = new HashMap<>();
parsers.put(AlwaysCondition.TYPE, (c, id, p) -> AlwaysCondition.parse(id, p));
parsers.put(NeverCondition.TYPE, (c, id, p) -> NeverCondition.parse(id, p));
@ -245,24 +298,19 @@ public class Watcher implements ActionPlugin {
transformFactories.put(SearchTransform.TYPE, new SearchTransformFactory(settings, internalClient, xContentRegistry, scriptService));
final TransformRegistry transformRegistry = new TransformRegistry(settings, Collections.unmodifiableMap(transformFactories));
// actions
final Map<String, ActionFactory> actionFactoryMap = new HashMap<>();
TextTemplateEngine templateEngine = getService(TextTemplateEngine.class, components);
actionFactoryMap.put(EmailAction.TYPE, new EmailActionFactory(settings, getService(EmailService.class, components), templateEngine,
getService(EmailAttachmentsParser.class, components)));
actionFactoryMap.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient,
getService(HttpRequestTemplate.Parser.class, components), templateEngine));
actionFactoryMap.put(EmailAction.TYPE, new EmailActionFactory(settings, emailService, templateEngine, emailAttachmentsParser));
actionFactoryMap.put(WebhookAction.TYPE, new WebhookActionFactory(settings, httpClient, httpTemplateParser, templateEngine));
actionFactoryMap.put(IndexAction.TYPE, new IndexActionFactory(settings, internalClient));
actionFactoryMap.put(LoggingAction.TYPE, new LoggingActionFactory(settings, templateEngine));
actionFactoryMap.put(HipChatAction.TYPE, new HipChatActionFactory(settings, templateEngine,
getService(HipChatService.class, components)));
actionFactoryMap.put(JiraAction.TYPE, new JiraActionFactory(settings, templateEngine,
getService(JiraService.class, components)));
actionFactoryMap.put(SlackAction.TYPE, new SlackActionFactory(settings, templateEngine,
getService(SlackService.class, components)));
actionFactoryMap.put(PagerDutyAction.TYPE, new PagerDutyActionFactory(settings, templateEngine,
getService(PagerDutyService.class, components)));
actionFactoryMap.put(HipChatAction.TYPE, new HipChatActionFactory(settings, templateEngine, hipChatService));
actionFactoryMap.put(JiraAction.TYPE, new JiraActionFactory(settings, templateEngine, jiraService));
actionFactoryMap.put(SlackAction.TYPE, new SlackActionFactory(settings, templateEngine, slackService));
actionFactoryMap.put(PagerDutyAction.TYPE, new PagerDutyActionFactory(settings, templateEngine, pagerDutyService));
final ActionRegistry registry = new ActionRegistry(actionFactoryMap, conditionRegistry, transformRegistry, clock, licenseState);
// inputs
final Map<String, InputFactory> inputFactories = new HashMap<>();
inputFactories.put(SearchInput.TYPE,
new SearchInputFactory(settings, internalClient, xContentRegistry, scriptService));
@ -275,6 +323,8 @@ public class Watcher implements ActionPlugin {
final WatcherClient watcherClient = new WatcherClient(internalClient);
final HistoryStore historyStore = new HistoryStore(settings, internalClient);
// schedulers
final Set<Schedule.Parser> scheduleParsers = new HashSet<>();
scheduleParsers.add(new CronSchedule.Parser());
scheduleParsers.add(new DailySchedule.Parser());
@ -321,7 +371,8 @@ public class Watcher implements ActionPlugin {
return Arrays.asList(registry, watcherClient, inputRegistry, historyStore, triggerService, triggeredWatchParser,
watcherLifeCycleService, executionService, triggerEngineListener, watcherService, watchParser,
configuredTriggerEngine, triggeredWatchStore, watcherSearchTemplateService, watcherIndexTemplateRegistry);
configuredTriggerEngine, triggeredWatchStore, watcherSearchTemplateService, watcherIndexTemplateRegistry,
slackService, pagerDutyService);
}
protected TriggerEngine getTriggerEngine(Clock clock, ScheduleRegistry scheduleRegistry) {
@ -336,16 +387,6 @@ public class Watcher implements ActionPlugin {
return new AsyncTriggerEventConsumer(settings, executionService);
}
private <T> T getService(Class<T> serviceClass, Collection<Object> services) {
List<Object> collect = services.stream().filter(o -> o.getClass() == serviceClass).collect(Collectors.toList());
if (collect.isEmpty()) {
throw new IllegalArgumentException("no service for class " + serviceClass.getName());
} else if (collect.size() > 1) {
throw new IllegalArgumentException("more than one service for class " + serviceClass.getName());
}
return (T) collect.get(0);
}
public Collection<Module> nodeModules() {
List<Module> modules = new ArrayList<>();
modules.add(b -> {
@ -384,6 +425,18 @@ public class Watcher implements ActionPlugin {
settings.add(Setting.simpleString("xpack.watcher.execution.scroll.timeout", Setting.Property.NodeScope));
settings.add(Setting.simpleString("xpack.watcher.start_immediately", Setting.Property.NodeScope));
// notification services
settings.add(SlackService.SLACK_ACCOUNT_SETTING);
settings.add(EmailService.EMAIL_ACCOUNT_SETTING);
settings.add(HipChatService.HIPCHAT_ACCOUNT_SETTING);
settings.add(JiraService.JIRA_ACCOUNT_SETTING);
settings.add(PagerDutyService.PAGERDUTY_ACCOUNT_SETTING);
settings.add(ReportingAttachmentParser.RETRIES_SETTING);
settings.add(ReportingAttachmentParser.INTERVAL_SETTING);
// http settings
settings.addAll(HttpSettings.getSettings());
// encryption settings
CryptoService.addSettings(settings);
return settings;
@ -520,4 +573,20 @@ public class Watcher implements ActionPlugin {
public List<BootstrapCheck> getBootstrapChecks(Environment env) {
return Collections.singletonList(new EncryptSensitiveDataBootstrapCheck(env));
}
public Collection<? extends String> getSettingsFilter() {
List<String> filters = new ArrayList<>();
filters.add("xpack.notification.email.account.*.smtp.password");
filters.add("xpack.notification.jira.account.*.password");
filters.add("xpack.notification.slack.account.*.url");
filters.add("xpack.notification.pagerduty.account.*.url");
filters.add("xpack.notification.pagerduty." + PagerDutyAccount.SERVICE_KEY_SETTING);
filters.add("xpack.notification.pagerduty.account.*." + PagerDutyAccount.SERVICE_KEY_SETTING);
filters.add("xpack.notification.hipchat.account.*.auth_token");
return filters;
}
public List<ScriptContext> getContexts() {
return Arrays.asList(Watcher.SCRIPT_SEARCH_CONTEXT, Watcher.SCRIPT_EXECUTABLE_CONTEXT, Watcher.SCRIPT_TEMPLATE_CONTEXT);
}
}

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/ApplicableBasicAuth.java
View File

@ -9,7 +9,7 @@ import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.elasticsearch.xpack.watcher.common.http.auth.ApplicableHttpAuth;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import java.net.HttpURLConnection;
import java.nio.charset.StandardCharsets;

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/common/http/auth/basic/BasicAuthFactory.java
View File

@ -8,7 +8,7 @@ package org.elasticsearch.xpack.watcher.common.http.auth.basic;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.xpack.watcher.common.http.auth.HttpAuthFactory;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import java.io.IOException;

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/common/secret/Secret.java
View File

@ -5,7 +5,7 @@
*/
package org.elasticsearch.xpack.watcher.common.secret;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import java.io.IOException;
import java.util.Arrays;

35
plugin/src/main/java/org/elasticsearch/xpack/security/crypto/CryptoService.java → plugin/src/main/java/org/elasticsearch/xpack/watcher/crypto/CryptoService.java
View File

@ -3,12 +3,20 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.security.crypto;
package org.elasticsearch.xpack.watcher.crypto;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.security.authc.support.CharArrays;
import org.elasticsearch.xpack.watcher.Watcher;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
@ -21,15 +29,6 @@ import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.security.authc.support.CharArrays;
import org.elasticsearch.xpack.watcher.Watcher;
import static org.elasticsearch.xpack.security.Security.setting;
/**
@ -88,20 +87,6 @@ public class CryptoService extends AbstractComponent {
assert encryptionKey != null : "the encryption key should never be null";
}
public static byte[] generateKey() {
return generateSecretKey(KEY_SIZE).getEncoded();
}
static SecretKey generateSecretKey(int keyLength) {
try {
KeyGenerator generator = KeyGenerator.getInstance(KEY_ALGO);
generator.init(keyLength);
return generator.generateKey();
} catch (NoSuchAlgorithmException e) {
throw new ElasticsearchException("failed to generate key", e);
}
}
private static SecretKey readSystemKey(InputStream in) throws IOException {
final int keySizeBytes = KEY_SIZE / 8;
final byte[] keyBytes = new byte[keySizeBytes];

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Account.java
View File

@ -10,7 +10,7 @@ import org.elasticsearch.SpecialPermission;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import javax.activation.CommandMap;
import javax.activation.MailcapCommandMap;

3
plugin/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
View File

@ -9,8 +9,8 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.notification.NotificationService;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import javax.mail.MessagingException;
@ -36,7 +36,6 @@ public class EmailService extends NotificationService<Account> {
return new Account(config, cryptoService, logger);
}
public EmailSent send(Email email, Authentication auth, Profile profile, String accountName) throws MessagingException {
Account account = getAccount(accountName);
if (account == null) {

15
plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRespo
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
@ -77,13 +78,15 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
return;
}
if (event.localNodeMaster() == false) {
// Only the node that runs or will run Watcher should update the templates. Otherwise unnecessary put template
// calls would happen
return;
}
// if this node is newer than the master node, we probably need to add the history template, which might be newer than the
// history template the master node has, so we need potentially add new templates despite being not the master node
DiscoveryNode localNode = event.state().getNodes().getLocalNode();
DiscoveryNode masterNode = event.state().getNodes().getMasterNode();
boolean localNodeVersionAfterMaster = localNode.getVersion().after(masterNode.getVersion());
addTemplatesIfMissing(state);
if (event.localNodeMaster() || localNodeVersionAfterMaster) {
addTemplatesIfMissing(state);
}
}
private void addTemplatesIfMissing(ClusterState state) {

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/support/xcontent/WatcherXContentParser.java
View File

@ -13,7 +13,7 @@ import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.watcher.common.secret.Secret;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import java.io.IOException;
import java.time.Clock;

2
plugin/src/main/java/org/elasticsearch/xpack/watcher/watch/Watch.java
View File

@ -20,7 +20,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.watcher.common.secret.Secret;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.watch.clock.HaltedClock;
import org.elasticsearch.xpack.watcher.actions.ActionRegistry;
import org.elasticsearch.xpack.watcher.actions.ActionStatus;

11
plugin/src/test/java/org/elasticsearch/xpack/ml/action/ForecastJobActionRequestTests.java
View File

@ -5,10 +5,13 @@
*/
package org.elasticsearch.xpack.ml.action;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractStreamableXContentTestCase;
import org.elasticsearch.xpack.ml.action.ForecastJobAction.Request;
import java.time.Instant;
public class ForecastJobActionRequestTests extends AbstractStreamableXContentTestCase<Request> {
@Override
@ -24,6 +27,13 @@ public class ForecastJobActionRequestTests extends AbstractStreamableXContentTes
@Override
protected Request createTestInstance() {
Request request = new Request(randomAlphaOfLengthBetween(1, 20));
if (randomBoolean()) {
request.setEndTime(Instant.ofEpochMilli(randomNonNegativeLong()).toString());
}
if (randomBoolean()) {
request.setDuration(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)).getStringRep());
}
return request;
}
@ -31,5 +41,4 @@ public class ForecastJobActionRequestTests extends AbstractStreamableXContentTes
protected Request createBlankInstance() {
return new Request();
}
}

50
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java
View File

@ -17,7 +17,7 @@ import org.junit.After;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -42,8 +42,7 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
detector.setPartitionFieldName("user");
TimeValue bucketSpan = TimeValue.timeValueHours(1);
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(
Arrays.asList(detector.build()));
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
analysisConfig.setBucketSpan(bucketSpan);
DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setTimeFormat("epoch");
@ -64,6 +63,9 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
List<String> data = new ArrayList<>();
while (timestamp < now) {
for (int i = 0; i < 10000; i++) {
// It's important that the values used here are either always represented in less than 16 UTF-8 bytes or
// always represented in more than 22 UTF-8 bytes. Otherwise platform differences in when the small string
// optimisation is used will make the results of this test very different for the different platforms.
data.add(createJsonRecord(createRecord(timestamp, String.valueOf(i), "")));
}
timestamp += bucketSpan.seconds();
@ -76,7 +78,6 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
// and a balance of partitions/by fields were created
GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0);
ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
assertThat(modelSizeStats.getModelBytes(), lessThan(35000000L));
assertThat(modelSizeStats.getModelBytes(), greaterThan(30000000L));
@ -87,6 +88,7 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
assertThat(modelSizeStats.getTotalPartitionFieldCount(), greaterThan(600L));
assertThat(modelSizeStats.getTotalByFieldCount(), lessThan(900L));
assertThat(modelSizeStats.getTotalByFieldCount(), greaterThan(600L));
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
}
public void testTooManyByFields() throws Exception {
@ -94,8 +96,7 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
detector.setByFieldName("user");
TimeValue bucketSpan = TimeValue.timeValueHours(1);
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(
Arrays.asList(detector.build()));
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
analysisConfig.setBucketSpan(bucketSpan);
DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setTimeFormat("epoch");
@ -116,6 +117,9 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
List<String> data = new ArrayList<>();
while (timestamp < now) {
for (int i = 0; i < 10000; i++) {
// It's important that the values used here are either always represented in less than 16 UTF-8 bytes or
// always represented in more than 22 UTF-8 bytes. Otherwise platform differences in when the small string
// optimisation is used will make the results of this test very different for the different platforms.
data.add(createJsonRecord(createRecord(timestamp, String.valueOf(i), "")));
}
timestamp += bucketSpan.seconds();
@ -127,11 +131,11 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
// Assert we haven't violated the limit too much
GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0);
ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
assertThat(modelSizeStats.getModelBytes(), lessThan(36000000L));
assertThat(modelSizeStats.getModelBytes(), greaterThan(30000000L));
assertThat(modelSizeStats.getTotalByFieldCount(), lessThan(2100L));
assertThat(modelSizeStats.getTotalByFieldCount(), greaterThan(1500L));
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
}
public void testTooManyByAndOverFields() throws Exception {
@ -140,8 +144,7 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
detector.setOverFieldName("user");
TimeValue bucketSpan = TimeValue.timeValueHours(1);
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(
Arrays.asList(detector.build()));
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
analysisConfig.setBucketSpan(bucketSpan);
DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setTimeFormat("epoch");
@ -163,6 +166,9 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
for (int department = 0; department < 10; department++) {
List<String> data = new ArrayList<>();
for (int user = 0; user < 10000; user++) {
// It's important that the values used here are either always represented in less than 16 UTF-8 bytes or
// always represented in more than 22 UTF-8 bytes. Otherwise platform differences in when the small string
// optimisation is used will make the results of this test very different for the different platforms.
data.add(createJsonRecord(createRecord(
timestamp, String.valueOf(department) + "_" + String.valueOf(user), String.valueOf(department))));
}
@ -176,21 +182,20 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
// Assert we haven't violated the limit too much
GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0);
ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
assertThat(modelSizeStats.getModelBytes(), lessThan(36000000L));
assertThat(modelSizeStats.getModelBytes(), greaterThan(24000000L));
assertThat(modelSizeStats.getTotalByFieldCount(), equalTo(8L));
assertThat(modelSizeStats.getTotalOverFieldCount(), greaterThan(50000L));
assertThat(modelSizeStats.getTotalOverFieldCount(), lessThan(60000L));
assertThat(modelSizeStats.getTotalByFieldCount(), equalTo(7L));
assertThat(modelSizeStats.getTotalOverFieldCount(), greaterThan(40000L));
assertThat(modelSizeStats.getTotalOverFieldCount(), lessThan(50000L));
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
}
public void testTooManyDistinctOverFields() throws Exception {
public void testManyDistinctOverFields() throws Exception {
Detector.Builder detector = new Detector.Builder("sum", "value");
detector.setOverFieldName("user");
TimeValue bucketSpan = TimeValue.timeValueHours(1);
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(
Arrays.asList(detector.build()));
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
analysisConfig.setBucketSpan(bucketSpan);
DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setTimeFormat("epoch");
@ -198,8 +203,8 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
job.setAnalysisConfig(analysisConfig);
job.setDataDescription(dataDescription);
// Set the memory limit to 300MB
AnalysisLimits limits = new AnalysisLimits(70L, null);
// Set the memory limit to 110MB
AnalysisLimits limits = new AnalysisLimits(110L, null);
job.setAnalysisLimits(limits);
registerJob(job);
@ -212,6 +217,9 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
while (timestamp < now) {
List<String> data = new ArrayList<>();
for (int i = 0; i < 10000; i++) {
// It's important that the values used here are either always represented in less than 16 UTF-8 bytes or
// always represented in more than 22 UTF-8 bytes. Otherwise platform differences in when the small string
// optimisation is used will make the results of this test very different for the different platforms.
Map<String, Object> record = new HashMap<>();
record.put("time", timestamp);
record.put("user", user++);
@ -227,10 +235,10 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
// Assert we haven't violated the limit too much
GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0);
ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.OK));
assertThat(modelSizeStats.getModelBytes(), lessThan(45000000L));
assertThat(modelSizeStats.getModelBytes(), greaterThan(35000000L));
assertThat(modelSizeStats.getModelBytes(), lessThan(95000000L));
assertThat(modelSizeStats.getModelBytes(), greaterThan(75000000L));
assertThat(modelSizeStats.getTotalOverFieldCount(), greaterThan(140000L));
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.OK));
}
private static Map<String, Object> createRecord(long timestamp, String user, String department) {

24
plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParamsTests.java
View File

@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ml.job.process.autodetect.params;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.job.messages.Messages;
@ -16,15 +17,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
public class ForecastParamsTests extends ESTestCase {
private static ParseField END = new ParseField("end");
public void testDefault_GivesTomorrowTimeInSeconds() {
long nowSecs = System.currentTimeMillis() / 1000;
nowSecs += 60 * 60 * 24;
ForecastParams params = ForecastParams.builder().build();
assertThat(params.getEndTime(), greaterThanOrEqualTo(nowSecs));
assertThat(params.getEndTime(), lessThanOrEqualTo(nowSecs +1));
}
private static ParseField DURATION = new ParseField("duration");
public void test_UnparseableEndTimeThrows() {
ElasticsearchParseException e =
@ -41,4 +34,17 @@ public class ForecastParamsTests extends ESTestCase {
assertThat(end, greaterThanOrEqualTo(nowSecs + 7200));
assertThat(end, lessThanOrEqualTo(nowSecs + 7200 +1));
}
public void testDurationFormats() {
assertEquals(34678L,
ForecastParams.builder().duration(TimeValue.parseTimeValue("34678s", DURATION.getPreferredName())).build().getDuration());
assertEquals(172800L,
ForecastParams.builder().duration(TimeValue.parseTimeValue("2d", DURATION.getPreferredName())).build().getDuration());
}
public void testDurationEndTimeThrows() {
ElasticsearchParseException e = ESTestCase.expectThrows(ElasticsearchParseException.class, () -> ForecastParams.builder()
.endTime("2016-05-01T10:00:00Z", END).duration(TimeValue.parseTimeValue("33d", DURATION.getPreferredName())).build());
assertEquals(Messages.getMessage(Messages.REST_INVALID_DURATION_AND_ENDTIME), e.getMessage());
}
}

7
plugin/src/test/java/org/elasticsearch/xpack/security/crypto/tool/SystemKeyToolTests.java
View File

@ -17,7 +17,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.junit.After;
import java.nio.file.FileSystem;
@ -68,7 +67,7 @@ public class SystemKeyToolTests extends CommandTestCase {
execute("-Epath.home=" + homeDir, path.toString());
byte[] bytes = Files.readAllBytes(path);
// TODO: maybe we should actually check the key is...i dunno...valid?
assertEquals(CryptoService.KEY_SIZE / 8, bytes.length);
assertEquals(SystemKeyTool.KEY_SIZE / 8, bytes.length);
Set<PosixFilePermission> perms = Files.getPosixFilePermissions(path);
assertTrue(perms.toString(), perms.contains(PosixFilePermission.OWNER_READ));
@ -83,7 +82,7 @@ public class SystemKeyToolTests extends CommandTestCase {
Files.createDirectories(xpackConf);
execute("-Epath.home=" + homeDir.toString());
byte[] bytes = Files.readAllBytes(xpackConf.resolve("system_key"));
assertEquals(CryptoService.KEY_SIZE / 8, bytes.length);
assertEquals(SystemKeyTool.KEY_SIZE / 8, bytes.length);
}
public void testGenerateDefaultPath() throws Exception {
@ -92,7 +91,7 @@ public class SystemKeyToolTests extends CommandTestCase {
Files.createDirectories(keyPath.getParent());
execute("-Epath.home=" + homeDir.toString());
byte[] bytes = Files.readAllBytes(keyPath);
assertEquals(CryptoService.KEY_SIZE / 8, bytes.length);
assertEquals(SystemKeyTool.KEY_SIZE / 8, bytes.length);
}
public void testThatSystemKeyMayOnlyBeReadByOwner() throws Exception {

4
plugin/src/test/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheckTests.java
View File

@ -11,7 +11,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoServiceTests;
public class EncryptSensitiveDataBootstrapCheckTests extends ESTestCase {
@ -35,7 +35,7 @@ public class EncryptSensitiveDataBootstrapCheckTests extends ESTestCase {
public void testKeyInKeystore() {
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), CryptoService.generateKey());
secureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), CryptoServiceTests.generateKey());
Settings settings = Settings.builder()
.put("path.home", createTempDir())
.put(Watcher.ENCRYPT_SENSITIVE_DATA_SETTING.getKey(), true)

2
plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
View File

@ -72,6 +72,6 @@ public class WatcherPluginTests extends ESTestCase {
watcher.onIndexModule(indexModule);
// also no component creation if not enabled
assertThat(watcher.createComponents(null, null, null, null, null, null, null, null, null, null, null), hasSize(0));
assertThat(watcher.createComponents(null, null, null, null, null, null, null, null), hasSize(0));
}
}

21
plugin/src/test/java/org/elasticsearch/xpack/security/crypto/CryptoServiceTests.java → plugin/src/test/java/org/elasticsearch/xpack/watcher/crypto/CryptoServiceTests.java
View File

@ -3,16 +3,19 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.security.crypto;
import java.util.Arrays;
package org.elasticsearch.xpack.watcher.crypto;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.watcher.Watcher;
import org.junit.Before;
import javax.crypto.KeyGenerator;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
@ -22,7 +25,7 @@ public class CryptoServiceTests extends ESTestCase {
@Before
public void init() throws Exception {
MockSecureSettings mockSecureSettings = new MockSecureSettings();
mockSecureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), CryptoService.generateKey());
mockSecureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), generateKey());
settings = Settings.builder()
.setSecureSettings(mockSecureSettings)
.build();
@ -49,4 +52,14 @@ public class CryptoServiceTests extends ESTestCase {
assertThat(service.isEncrypted(randomAlphaOfLengthBetween(0, 100).toCharArray()), is(false));
assertThat(service.isEncrypted(service.encrypt(randomAlphaOfLength(10).toCharArray())), is(true));
}
public static byte[] generateKey() {
try {
KeyGenerator generator = KeyGenerator.getInstance(CryptoService.KEY_ALGO);
generator.init(CryptoService.KEY_SIZE);
return generator.generateKey().getEncoded();
} catch (NoSuchAlgorithmException e) {
throw new ElasticsearchException("failed to generate key", e);
}
}
}

10
plugin/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailSecretsIntegrationTests.java
View File

@ -9,13 +9,14 @@ import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
import org.elasticsearch.xpack.watcher.notification.email.support.EmailServer;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.condition.AlwaysCondition;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoServiceTests;
import org.elasticsearch.xpack.watcher.execution.ActionExecutionMode;
import org.elasticsearch.xpack.watcher.notification.email.support.EmailServer;
import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
import org.elasticsearch.xpack.watcher.transport.actions.execute.ExecuteWatchResponse;
@ -32,7 +33,6 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.emailAction;
import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder;
import static org.elasticsearch.xpack.watcher.input.InputBuilders.simpleInput;
import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
@ -64,7 +64,7 @@ public class EmailSecretsIntegrationTests extends AbstractWatcherIntegrationTest
if (encryptSensitiveData == null) {
encryptSensitiveData = randomBoolean();
if (encryptSensitiveData) {
encryptionKey = CryptoService.generateKey();
encryptionKey = CryptoServiceTests.generateKey();
}
}
Settings.Builder builder = Settings.builder()

83
plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.watcher.support;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
@ -18,6 +19,8 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
@ -44,6 +47,7 @@ import static org.mockito.Matchers.same;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verifyZeroInteractions;
public class WatcherIndexTemplateRegistryTests extends ESTestCase {
@ -73,36 +77,18 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
registry = new WatcherIndexTemplateRegistry(Settings.EMPTY, clusterService, threadPool, internalClient);
}
private ClusterChangedEvent createClusterChangedEvent(List<String> existingTemplateNames) {
ClusterChangedEvent event = mock(ClusterChangedEvent.class);
when(event.localNodeMaster()).thenReturn(true);
ClusterState cs = mock(ClusterState.class);
ClusterBlocks clusterBlocks = mock(ClusterBlocks.class);
when(clusterBlocks.hasGlobalBlock(eq(GatewayService.STATE_NOT_RECOVERED_BLOCK))).thenReturn(false);
when(cs.blocks()).thenReturn(clusterBlocks);
when(event.state()).thenReturn(cs);
MetaData metaData = mock(MetaData.class);
ImmutableOpenMap.Builder<String, IndexTemplateMetaData> indexTemplates = ImmutableOpenMap.builder();
for (String name : existingTemplateNames) {
indexTemplates.put(name, mock(IndexTemplateMetaData.class));
}
when(metaData.getTemplates()).thenReturn(indexTemplates.build());
when(cs.metaData()).thenReturn(metaData);
return event;
}
public void testThatNonExistingTemplatesAreAddedImmediately() {
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList());
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes);
registry.clusterChanged(event);
ArgumentCaptor<PutIndexTemplateRequest> argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
verify(client, times(3)).execute(anyObject(), argumentCaptor.capture(), anyObject());
// now delete one template from the cluster state and lets retry
ClusterChangedEvent newEvent = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME,
WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME));
WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME), nodes);
registry.clusterChanged(newEvent);
verify(client, times(4)).execute(anyObject(), argumentCaptor.capture(), anyObject());
}
@ -117,6 +103,57 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
".triggered_watches", ".watches", "whatever", "else")), is(true));
}
// if a node is newer than the master node, the template needs to be applied as well
// otherwise a rolling upgrade would not work as expected, when the node has a .watches shard on it
public void testThatTemplatesAreAppliedOnNewerNodes() {
DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.V_6_0_0);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build();
ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME,
WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes);
registry.clusterChanged(event);
ArgumentCaptor<PutIndexTemplateRequest> argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
verify(client, times(1)).execute(anyObject(), argumentCaptor.capture(), anyObject());
assertThat(argumentCaptor.getValue().name(), is(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME));
}
public void testThatTemplatesAreNotAppliedOnSameVersionNodes() {
DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build();
ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME,
WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes);
registry.clusterChanged(event);
verifyZeroInteractions(client);
}
private ClusterChangedEvent createClusterChangedEvent(List<String> existingTemplateNames, DiscoveryNodes nodes) {
ClusterChangedEvent event = mock(ClusterChangedEvent.class);
when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster());
ClusterState cs = mock(ClusterState.class);
ClusterBlocks clusterBlocks = mock(ClusterBlocks.class);
when(clusterBlocks.hasGlobalBlock(eq(GatewayService.STATE_NOT_RECOVERED_BLOCK))).thenReturn(false);
when(cs.blocks()).thenReturn(clusterBlocks);
when(event.state()).thenReturn(cs);
when(cs.getNodes()).thenReturn(nodes);
MetaData metaData = mock(MetaData.class);
ImmutableOpenMap.Builder<String, IndexTemplateMetaData> indexTemplates = ImmutableOpenMap.builder();
for (String name : existingTemplateNames) {
indexTemplates.put(name, mock(IndexTemplateMetaData.class));
}
when(metaData.getTemplates()).thenReturn(indexTemplates.build());
when(cs.metaData()).thenReturn(metaData);
return event;
}
private ClusterState createClusterState(String ... existingTemplates) {
MetaData.Builder metaDataBuilder = MetaData.builder();
for (String templateName : existingTemplates) {

9
plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
View File

@ -11,13 +11,14 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.test.http.MockResponse;
import org.elasticsearch.test.http.MockWebServer;
import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.ApplicableBasicAuth;
import org.elasticsearch.xpack.watcher.common.http.auth.basic.BasicAuth;
import org.elasticsearch.xpack.security.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.condition.AlwaysCondition;
import org.elasticsearch.xpack.watcher.crypto.CryptoService;
import org.elasticsearch.xpack.watcher.crypto.CryptoServiceTests;
import org.elasticsearch.xpack.watcher.execution.ActionExecutionMode;
import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@ -72,7 +73,7 @@ public class HttpSecretsIntegrationTests extends AbstractWatcherIntegrationTestC
if (encryptSensitiveData == null) {
encryptSensitiveData = randomBoolean();
if (encryptSensitiveData) {
encryptionKey = CryptoService.generateKey();
encryptionKey = CryptoServiceTests.generateKey();
}
}
if (encryptSensitiveData) {

4
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_monitoring.yml
View File

@ -6,6 +6,10 @@ setup:
---
"Index monitoring data and search on the mixed cluster":
- skip:
version: "all"
reason: "AwaitsFix'ing, see x-pack-elasticsearch #2948"
- do:
search:
index: .monitoring-kibana-*

4
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_monitoring.yml
View File

@ -6,6 +6,10 @@ setup:
---
"Index monitoring data and search on the upgraded cluster":
- skip:
version: "all"
reason: "AwaitsFix'ing, see x-pack-elasticsearch #2948"
- do:
search:
index: .monitoring-kibana-*

5
qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherTemplateIT.java → qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java
View File

@ -6,7 +6,6 @@
package org.elasticsearch.smoketest;
import com.fasterxml.jackson.core.io.JsonStringEncoder;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
@ -15,9 +14,9 @@ import org.elasticsearch.script.ScriptEngine;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.mustache.MustacheScriptEngine;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
import org.elasticsearch.xpack.watcher.Watcher;
import org.junit.Before;
import java.io.IOException;
@ -31,7 +30,7 @@ import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
public class WatcherTemplateIT extends ESTestCase {
public class WatcherTemplateTests extends ESTestCase {
private TextTemplateEngine textTemplateEngine;