honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/sql 

Original commit: elastic/x-pack-elasticsearch@d24c740fe3
This commit is contained in:
Nik Everett 2017-08-03 15:19:43 -04:00
parent 0675d08f25 2d9d4c41d8
commit 8a7a724179
27 changed files with 721 additions and 680 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedUpdate;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.config.JobTaskStatus;
import org.elasticsearch.xpack.ml.job.groups.GroupOrJobLookup;
import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.ml.utils.NameResolver;
@ -66,10 +67,12 @@ public class MlMetadata implements MetaData.Custom {
private final SortedMap<String, Job> jobs;
private final SortedMap<String, DatafeedConfig> datafeeds;
private final GroupOrJobLookup groupOrJobLookup;
private MlMetadata(SortedMap<String, Job> jobs, SortedMap<String, DatafeedConfig> datafeeds) {
this.jobs = Collections.unmodifiableSortedMap(jobs);
this.datafeeds = Collections.unmodifiableSortedMap(datafeeds);
this.groupOrJobLookup = new GroupOrJobLookup(jobs.values());
}
public Map<String, Job> getJobs() {
@ -77,8 +80,7 @@ public class MlMetadata implements MetaData.Custom {
}
public Set<String> expandJobIds(String expression, boolean allowNoJobs) {
return NameResolver.newUnaliased(jobs.keySet(), jobId -> ExceptionsHelper.missingJobException(jobId))
.expand(expression, allowNoJobs);
return groupOrJobLookup.expandJobIds(expression, allowNoJobs);
}
public boolean isJobDeleted(String jobId) {
@ -136,6 +138,8 @@ public class MlMetadata implements MetaData.Custom {
datafeeds.put(in.readString(), new DatafeedConfig(in));
}
this.datafeeds = datafeeds;
this.groupOrJobLookup = new GroupOrJobLookup(jobs.values());
}
@Override

8
plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidator.java
View File

@ -48,5 +48,13 @@ public final class DatafeedJobValidator {
TimeValue.timeValueMillis(histogramIntervalMillis).getStringRep(),
TimeValue.timeValueMillis(bucketSpanMillis).getStringRep()));
}
if (bucketSpanMillis % histogramIntervalMillis != 0) {
throw ExceptionsHelper.badRequestException(Messages.getMessage(
Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_DIVISOR_OF_BUCKET_SPAN,
TimeValue.timeValueMillis(histogramIntervalMillis).getStringRep(),
TimeValue.timeValueMillis(bucketSpanMillis).getStringRep()));
}
}
}

11
plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java
View File

@ -115,13 +115,10 @@ class AggregationDataExtractor implements DataExtractor {
}
private SearchRequestBuilder buildSearchRequest() {
long histogramSearchStartTime = context.start;
if (context.aggs.getPipelineAggregatorFactories().isEmpty() == false) {
// For derivative aggregations the first bucket will always be null
// so query one extra histogram bucket back and hope there is data
// in that bucket
histogramSearchStartTime = Math.max(0, context.start - getHistogramInterval());
}
// For derivative aggregations the first bucket will always be null
// so query one extra histogram bucket back and hope there is data
// in that bucket
long histogramSearchStartTime = Math.max(0, context.start - getHistogramInterval());
SearchRequestBuilder searchRequestBuilder = SearchAction.INSTANCE.newRequestBuilder(client)
.setIndices(context.indices)

57
plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Job.java
View File

@ -16,7 +16,6 @@ import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser.Token;
@ -29,6 +28,7 @@ import org.elasticsearch.xpack.ml.utils.time.TimeUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.EnumMap;
import java.util.HashSet;
@ -57,6 +57,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
public static final ParseField ID = new ParseField("job_id");
public static final ParseField JOB_TYPE = new ParseField("job_type");
public static final ParseField JOB_VERSION = new ParseField("job_version");
public static final ParseField GROUPS = new ParseField("groups");
public static final ParseField ANALYSIS_CONFIG = new ParseField("analysis_config");
public static final ParseField ANALYSIS_LIMITS = new ParseField("analysis_limits");
public static final ParseField CREATE_TIME = new ParseField("create_time");
@ -94,6 +95,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
parser.declareString(Builder::setId, ID);
parser.declareString(Builder::setJobType, JOB_TYPE);
parser.declareString(Builder::setJobVersion, JOB_VERSION);
parser.declareStringArray(Builder::setGroups, GROUPS);
parser.declareStringOrNull(Builder::setDescription, DESCRIPTION);
parser.declareField(Builder::setCreateTime, p -> {
if (p.currentToken() == Token.VALUE_NUMBER) {
@ -148,6 +150,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
@Nullable
private final Version jobVersion;
private final List<String> groups;
private final String description;
// TODO: Use java.time for the Dates here: x-pack-elasticsearch#829
private final Date createTime;
@ -166,16 +169,17 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
private final String resultsIndexName;
private final boolean deleted;
private Job(String jobId, String jobType, Version jobVersion, String description, Date createTime,
Date finishedTime, Date lastDataTime,
AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription,
ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval,
Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map<String, Object> customSettings,
String modelSnapshotId, String resultsIndexName, boolean deleted) {
private Job(String jobId, String jobType, Version jobVersion, List<String> groups, String description, Date createTime,
Date finishedTime, Date lastDataTime,
AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription,
ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval,
Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map<String, Object> customSettings,
String modelSnapshotId, String resultsIndexName, boolean deleted) {
this.jobId = jobId;
this.jobType = jobType;
this.jobVersion = jobVersion;
this.groups = groups;
this.description = description;
this.createTime = createTime;
this.finishedTime = finishedTime;
@ -202,6 +206,11 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
} else {
jobVersion = null;
}
if (in.getVersion().onOrAfter(Version.V_6_1_0)) {
groups = in.readList(StreamInput::readString);
} else {
groups = Collections.emptyList();
}
description = in.readOptionalString();
createTime = new Date(in.readVLong());
finishedTime = in.readBoolean() ? new Date(in.readVLong()) : null;
@ -237,6 +246,10 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
return jobVersion;
}
public List<String> getGroups() {
return groups;
}
/**
* The name of the index storing the job's results and state.
* This defaults to {@link #getId()} if a specific index name is not set.
@ -414,6 +427,9 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
out.writeBoolean(false);
}
}
if (out.getVersion().onOrAfter(Version.V_6_1_0)) {
out.writeStringList(groups);
}
out.writeOptionalString(description);
out.writeVLong(createTime.getTime());
if (finishedTime != null) {
@ -458,6 +474,9 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
if (jobVersion != null) {
builder.field(JOB_VERSION.getPreferredName(), jobVersion);
}
if (groups.isEmpty() == false) {
builder.field(GROUPS.getPreferredName(), groups);
}
if (description != null) {
builder.field(DESCRIPTION.getPreferredName(), description);
}
@ -519,6 +538,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
return Objects.equals(this.jobId, that.jobId)
&& Objects.equals(this.jobType, that.jobType)
&& Objects.equals(this.jobVersion, that.jobVersion)
&& Objects.equals(this.groups, that.groups)
&& Objects.equals(this.description, that.description)
&& Objects.equals(this.createTime, that.createTime)
&& Objects.equals(this.finishedTime, that.finishedTime)
@ -538,7 +558,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
@Override
public int hashCode() {
return Objects.hash(jobId, jobType, jobVersion, description, createTime, finishedTime, lastDataTime, analysisConfig,
return Objects.hash(jobId, jobType, jobVersion, groups, description, createTime, finishedTime, lastDataTime, analysisConfig,
analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
modelSnapshotId, resultsIndexName, deleted);
@ -574,6 +594,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
private String id;
private String jobType = ANOMALY_DETECTOR_JOB_TYPE;
private Version jobVersion;
private List<String> groups = Collections.emptyList();
private String description;
private AnalysisConfig analysisConfig;
private AnalysisLimits analysisLimits;
@ -602,6 +623,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
this.id = job.getId();
this.jobType = job.getJobType();
this.jobVersion = job.getJobVersion();
this.groups = job.getGroups();
this.description = job.getDescription();
this.analysisConfig = job.getAnalysisConfig();
this.analysisLimits = job.getAnalysisLimits();
@ -626,6 +648,11 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
if (in.getVersion().onOrAfter(Version.V_5_5_0)) {
jobVersion = in.readBoolean() ? Version.readVersion(in) : null;
}
if (in.getVersion().onOrAfter(Version.V_6_1_0)) {
groups = in.readList(StreamInput::readString);
} else {
groups = Collections.emptyList();
}
description = in.readOptionalString();
createTime = in.readBoolean() ? new Date(in.readVLong()) : null;
finishedTime = in.readBoolean() ? new Date(in.readVLong()) : null;
@ -665,6 +692,15 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
this.jobType = jobType;
}
public void setGroups(List<String> groups) {
this.groups = groups == null ? Collections.emptyList() : groups;
for (String group : this.groups) {
if (MlStrings.isValidId(group) == false) {
throw new IllegalArgumentException(Messages.getMessage(Messages.INVALID_GROUP, group));
}
}
}
public Date getCreateTime() {
return createTime;
}
@ -797,6 +833,9 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
out.writeBoolean(false);
}
}
if (out.getVersion().onOrAfter(Version.V_6_1_0)) {
out.writeStringList(groups);
}
out.writeOptionalString(description);
if (createTime != null) {
out.writeBoolean(true);
@ -1002,7 +1041,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
}
return new Job(
id, jobType, jobVersion, description, createTime, finishedTime, lastDataTime,
id, jobType, jobVersion, groups, description, createTime, finishedTime, lastDataTime,
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
modelSnapshotId, resultsIndexName, deleted);

50
plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.ml.job.config;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
@ -19,6 +20,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -31,6 +34,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
static {
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), Job.ID);
PARSER.declareStringArray(Builder::setGroups, Job.GROUPS);
PARSER.declareStringOrNull(Builder::setDescription, Job.DESCRIPTION);
PARSER.declareObjectArray(Builder::setDetectorUpdates, DetectorUpdate.PARSER, DETECTORS);
PARSER.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.CONFIG_PARSER, Job.MODEL_PLOT_CONFIG);
@ -46,6 +50,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
}
private final String jobId;
private final List<String> groups;
private final String description;
private final List<DetectorUpdate> detectorUpdates;
private final ModelPlotConfig modelPlotConfig;
@ -58,13 +63,14 @@ public class JobUpdate implements Writeable, ToXContentObject {
private final Map<String, Object> customSettings;
private final String modelSnapshotId;
private JobUpdate(String jobId, @Nullable String description, @Nullable List<DetectorUpdate> detectorUpdates,
@Nullable ModelPlotConfig modelPlotConfig, @Nullable AnalysisLimits analysisLimits,
@Nullable TimeValue backgroundPersistInterval, @Nullable Long renormalizationWindowDays,
@Nullable Long resultsRetentionDays, @Nullable Long modelSnapshotRetentionDays,
@Nullable List<String> categorisationFilters, @Nullable Map<String, Object> customSettings,
@Nullable String modelSnapshotId) {
private JobUpdate(String jobId, @Nullable List<String> groups, @Nullable String description,
@Nullable List<DetectorUpdate> detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig,
@Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval,
@Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays,
@Nullable Long modelSnapshotRetentionDays, @Nullable List<String> categorisationFilters,
@Nullable Map<String, Object> customSettings, @Nullable String modelSnapshotId) {
this.jobId = jobId;
this.groups = groups;
this.description = description;
this.detectorUpdates = detectorUpdates;
this.modelPlotConfig = modelPlotConfig;
@ -80,6 +86,12 @@ public class JobUpdate implements Writeable, ToXContentObject {
public JobUpdate(StreamInput in) throws IOException {
jobId = in.readString();
if (in.getVersion().onOrAfter(Version.V_6_1_0)) {
String[] groupsArray = in.readOptionalStringArray();
groups = groupsArray == null ? null : Arrays.asList(groupsArray);
} else {
groups = null;
}
description = in.readOptionalString();
if (in.readBoolean()) {
detectorUpdates = in.readList(DetectorUpdate::new);
@ -100,9 +112,14 @@ public class JobUpdate implements Writeable, ToXContentObject {
customSettings = in.readMap();
modelSnapshotId = in.readOptionalString();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(jobId);
if (out.getVersion().onOrAfter(Version.V_6_1_0)) {
String[] groupsArray = groups == null ? null : groups.toArray(new String[groups.size()]);
out.writeOptionalStringArray(groupsArray);
}
out.writeOptionalString(description);
out.writeBoolean(detectorUpdates != null);
if (detectorUpdates != null) {
@ -126,6 +143,10 @@ public class JobUpdate implements Writeable, ToXContentObject {
return jobId;
}
public List<String> getGroups() {
return groups;
}
public String getDescription() {
return description;
}
@ -178,6 +199,9 @@ public class JobUpdate implements Writeable, ToXContentObject {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(Job.ID.getPreferredName(), jobId);
if (groups != null) {
builder.field(Job.GROUPS.getPreferredName(), groups);
}
if (description != null) {
builder.field(Job.DESCRIPTION.getPreferredName(), description);
}
@ -223,6 +247,9 @@ public class JobUpdate implements Writeable, ToXContentObject {
*/
public Job mergeWithJob(Job source) {
Job.Builder builder = new Job.Builder(source);
if (groups != null) {
builder.setGroups(groups);
}
if (description != null) {
builder.setDescription(description);
}
@ -294,6 +321,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
JobUpdate that = (JobUpdate) other;
return Objects.equals(this.jobId, that.jobId)
&& Objects.equals(this.groups, that.groups)
&& Objects.equals(this.description, that.description)
&& Objects.equals(this.detectorUpdates, that.detectorUpdates)
&& Objects.equals(this.modelPlotConfig, that.modelPlotConfig)
@ -309,7 +337,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
@Override
public int hashCode() {
return Objects.hash(jobId, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays,
return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, categorizationFilters, customSettings,
modelSnapshotId);
}
@ -410,6 +438,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
public static class Builder {
private String jobId;
private List<String> groups;
private String description;
private List<DetectorUpdate> detectorUpdates;
private ModelPlotConfig modelPlotConfig;
@ -431,6 +460,11 @@ public class JobUpdate implements Writeable, ToXContentObject {
return this;
}
public Builder setGroups(List<String> groups) {
this.groups = groups;
return this;
}
public Builder setDescription(String description) {
this.description = description;
return this;
@ -487,7 +521,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
}
public JobUpdate build() {
return new JobUpdate(jobId, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval,
return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval,
renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, categorizationFilters, customSettings,
modelSnapshotId);
}

59
plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJob.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.job.groups;
import org.elasticsearch.xpack.ml.job.config.Job;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
/**
* An interface to represent either a job or a group of jobs
*/
interface GroupOrJob {
boolean isGroup();
List<Job> jobs();
final class Group implements GroupOrJob {
private final List<Job> jobs;
Group(List<Job> jobs) {
this.jobs = Collections.unmodifiableList(jobs);
}
@Override
public boolean isGroup() {
return true;
}
@Override
public List<Job> jobs() {
return jobs;
}
}
final class SingleJob implements GroupOrJob {
private final Job job;
SingleJob(Job job) {
this.job = Objects.requireNonNull(job);
}
@Override
public boolean isGroup() {
return false;
}
@Override
public List<Job> jobs() {
return Collections.singletonList(job);
}
}
}

87
plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookup.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.job.groups;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.ml.utils.NameResolver;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.stream.Collectors;
/**
* A lookup that allows expanding expressions that may consist of job
* IDs, job group names, wildcard patterns or a comma separated combination
* of the aforementioned to the matching job IDs.
* The lookup is immutable.
*/
public class GroupOrJobLookup {
private final SortedMap<String, GroupOrJob> groupOrJobLookup;
public GroupOrJobLookup(Collection<Job> jobs) {
groupOrJobLookup = new TreeMap<>();
jobs.forEach(this::put);
}
private void put(Job job) {
if (groupOrJobLookup.containsKey(job.getId())) {
throw new ResourceAlreadyExistsException(Messages.getMessage(Messages.JOB_AND_GROUP_NAMES_MUST_BE_UNIQUE, job.getId()));
}
groupOrJobLookup.put(job.getId(), new GroupOrJob.SingleJob(job));
for (String groupName : job.getGroups()) {
GroupOrJob oldGroup = groupOrJobLookup.get(groupName);
if (oldGroup == null) {
groupOrJobLookup.put(groupName, new GroupOrJob.Group(Collections.singletonList(job)));
} else {
if (oldGroup.isGroup() == false) {
throw new ResourceAlreadyExistsException(Messages.getMessage(Messages.JOB_AND_GROUP_NAMES_MUST_BE_UNIQUE, groupName));
}
List<Job> groupJobs = new ArrayList<>(oldGroup.jobs());
groupJobs.add(job);
groupOrJobLookup.put(groupName, new GroupOrJob.Group(groupJobs));
}
}
}
public Set<String> expandJobIds(String expression, boolean allowNoJobs) {
return new GroupOrJobResolver().expand(expression, allowNoJobs);
}
private class GroupOrJobResolver extends NameResolver {
private GroupOrJobResolver() {
super(ExceptionsHelper::missingJobException);
}
@Override
protected Set<String> keys() {
return groupOrJobLookup.keySet();
}
@Override
protected Set<String> nameSet() {
return groupOrJobLookup.values().stream()
.filter(groupOrJob -> groupOrJob.isGroup() == false)
.map(groupOrJob -> groupOrJob.jobs().get(0).getId())
.collect(Collectors.toSet());
}
@Override
protected List<String> lookup(String key) {
GroupOrJob groupOrJob = groupOrJobLookup.get(key);
return groupOrJob == null ? Collections.emptyList() : groupOrJob.jobs().stream().map(Job::getId).collect(Collectors.toList());
}
}
}

7
plugin/src/main/java/org/elasticsearch/xpack/ml/job/messages/Messages.java
View File

@ -30,6 +30,8 @@ public final class Messages {
"The date_histogram (or histogram) aggregation cannot have sibling aggregations";
public static final String DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO =
"Aggregation interval must be greater than 0";
public static final String DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_DIVISOR_OF_BUCKET_SPAN =
"Aggregation interval [{0}] must be a divisor of the bucket_span [{1}]";
public static final String DATAFEED_AGGREGATIONS_INTERVAL_MUST_LESS_OR_EQUAL_TO_BUCKET_SPAN =
"Aggregation interval [{0}] must be less than or equal to the bucket_span [{1}]";
public static final String DATAFEED_DATA_HISTOGRAM_MUST_HAVE_NESTED_MAX_AGGREGATION =
@ -40,6 +42,8 @@ public final class Messages {
"Inconsistent {0}; ''{1}'' specified in the body differs from ''{2}'' specified as a URL argument";
public static final String INVALID_ID = "Invalid {0}; ''{1}'' can contain lowercase alphanumeric (a-z and 0-9), hyphens or " +
"underscores; must start and end with alphanumeric";
public static final String INVALID_GROUP = "Invalid group id ''{0}''; must be non-empty string and may contain lowercase alphanumeric" +
" (a-z and 0-9), hyphens or underscores; must start and end with alphanumeric";
public static final String JOB_AUDIR_DATAFEED_DATA_SEEN_AGAIN = "Datafeed has started retrieving data again";
public static final String JOB_AUDIT_CREATED = "Job created";
@ -149,6 +153,9 @@ public final class Messages {
public static final String JOB_CONFIG_TIME_FIELD_NOT_ALLOWED_IN_ANALYSIS_CONFIG =
"data_description.time_field may not be used in the analysis_config";
public static final String JOB_AND_GROUP_NAMES_MUST_BE_UNIQUE =
"job and group names must be unique but job [{0}] and group [{0}] have the same name";
public static final String JOB_UNKNOWN_ID = "No known job with id ''{0}''";
public static final String REST_CANNOT_DELETE_HIGHEST_PRIORITY =

5
plugin/src/main/java/org/elasticsearch/xpack/ml/utils/NameResolver.java
View File

@ -69,7 +69,7 @@ public abstract class NameResolver {
result.addAll(expanded);
} else {
List<String> matchingNames = lookup(token);
if (matchingNames == null) {
if (matchingNames.isEmpty()) {
throw notFoundExceptionSupplier.apply(token);
}
result.addAll(matchingNames);
@ -97,7 +97,6 @@ public abstract class NameResolver {
* @param key the key to look up
* @return a list of the matching names or {@code null} when no matching names exist
*/
@Nullable
protected abstract List<String> lookup(String key);
/**
@ -120,7 +119,7 @@ public abstract class NameResolver {
@Override
protected List<String> lookup(String key) {
return nameSet.contains(key) ? Collections.singletonList(key) : null;
return nameSet.contains(key) ? Collections.singletonList(key) : Collections.emptyList();
}
};
}

87
plugin/src/test/java/org/elasticsearch/bench/HasherBenchmark.java
View File

@ -1,87 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.bench;
import org.elasticsearch.common.Randomness;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.xpack.security.authc.support.Hasher;
@SuppressForbidden(reason = "benchmark")
public class HasherBenchmark {
private static final int WARMING_ITERS = 1000;
private static final int BENCH_ITERS = 10000;
public static void main(String[] args) throws Exception {
test(Hasher.SSHA256).print();
test(Hasher.MD5).print();
test(Hasher.SHA1).print();
test(Hasher.BCRYPT4).print();
}
protected static Metrics test(Hasher hasher) {
Metrics metrics = new Metrics(hasher);
System.out.print("warming up [" + hasher.name() + "]...");
for (int i = 0; i < WARMING_ITERS; i++) {
SecureString str = new SecureString(RandomStrings.randomAsciiOfLength(Randomness.get(), 8).toCharArray());
char[] hash = hasher.hash(str);
hasher.verify(str, hash);
}
System.out.println("done!");
System.out.print("starting benchmark for [" + hasher.name() + "]...");
long start;
for (int i = 0; i < BENCH_ITERS; i++) {
SecureString str = new SecureString(RandomStrings.randomAsciiOfLength(Randomness.get(), 8).toCharArray());
start = System.nanoTime();
char[] hash = hasher.hash(str);
metrics.hash.inc(System.nanoTime() - start);
start = System.nanoTime();
hasher.verify(str, hash);
metrics.verify.inc(System.nanoTime() - start);
if (i % 1000000 == 0) {
System.out.println("finished " + i + " iterations");
}
}
System.out.println("done!");
return metrics;
}
@SuppressForbidden(reason = "benchmark")
private static class Metrics {
final String name;
final MeanMetric hash = new MeanMetric();
final MeanMetric verify = new MeanMetric();
Metrics(Hasher hasher) {
this.name = hasher.name();
}
void print() {
System.out.println(name);
System.out.println("\tHash (total): " + TimeValue.timeValueNanos(hash.sum()).format());
System.out.println("\tHash (avg): " + hash.mean() + " nanos");
System.out.println("\tVerify (total): " + TimeValue.timeValueNanos(verify.sum()).format());
System.out.println("\tVerify (avg): " + verify.mean() + " nanos");
}
}
}

17
plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java
View File

@ -123,6 +123,23 @@ public class DatafeedJobValidatorTests extends ESTestCase {
assertEquals("Aggregation interval [1800001ms] must be less than or equal to the bucket_span [1800000ms]", e.getMessage());
}
public void testVerify_HistogramIntervalIsDivisorOfBucketSpan() throws IOException {
Job.Builder builder = buildJobBuilder("foo");
AnalysisConfig.Builder ac = createAnalysisConfig();
ac.setSummaryCountFieldName("some_count");
ac.setBucketSpan(TimeValue.timeValueMinutes(5));
builder.setAnalysisConfig(ac);
Job job = builder.build(new Date());
DatafeedConfig datafeedConfig = createValidDatafeedConfigWithAggs(37 * 1000).build();
ElasticsearchStatusException e = ESTestCase.expectThrows(ElasticsearchStatusException.class,
() -> DatafeedJobValidator.validate(datafeedConfig, job));
assertEquals("Aggregation interval [37000ms] must be a divisor of the bucket_span [300000ms]", e.getMessage());
DatafeedConfig goodDatafeedConfig = createValidDatafeedConfigWithAggs(60 * 1000).build();
DatafeedJobValidator.validate(goodDatafeedConfig, job);
}
private static Job.Builder buildJobBuilder(String id) {
Job.Builder builder = new Job.Builder(id);
AnalysisConfig.Builder ac = createAnalysisConfig();

13
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
View File

@ -442,7 +442,10 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ "\"aggs\": {\"timestamp\":{\"max\":{\"field\":\"timestamp\"}},"
+ "\"bytes-delta\":{\"derivative\":{\"buckets_path\":\"avg_bytes_out\"}},"
+ "\"avg_bytes_out\":{\"avg\":{\"field\":\"network_bytes_out\"}} }}}}}";
new DatafeedBuilder(datafeedId, jobId, "network-data", "doc").setAggregations(aggregations).build();
new DatafeedBuilder(datafeedId, jobId, "network-data", "doc")
.setAggregations(aggregations)
.setChunkingTimespan("300s")
.build();
openJob(client(), jobId);
@ -696,6 +699,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
String scriptedFields;
String aggregations;
String authHeader = BASIC_AUTH_VALUE_SUPER_USER;
String chunkingTimespan;
DatafeedBuilder(String datafeedId, String jobId, String index, String type) {
this.datafeedId = datafeedId;
@ -724,12 +728,19 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
return this;
}
DatafeedBuilder setChunkingTimespan(String timespan) {
chunkingTimespan = timespan;
return this;
}
Response build() throws IOException {
String datafeedConfig = "{"
+ "\"job_id\": \"" + jobId + "\",\"indexes\":[\"" + index + "\"],\"types\":[\"" + type + "\"]"
+ (source ? ",\"_source\":true" : "")
+ (scriptedFields == null ? "" : ",\"script_fields\":" + scriptedFields)
+ (aggregations == null ? "" : ",\"aggs\":" + aggregations)
+ (chunkingTimespan == null ? "" :
",\"chunking_config\":{\"mode\":\"MANUAL\",\"time_span\":\"" + chunkingTimespan + "\"}")
+ "}";
return client().performRequest("put", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId, Collections.emptyMap(),
new StringEntity(datafeedConfig, ContentType.APPLICATION_JSON),

25
plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTests.java
View File

@ -21,15 +21,18 @@ import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
@ -465,6 +468,20 @@ public class JobTests extends AbstractSerializingTestCase<Job> {
assertEquals(expected, new HashSet<>(builder.invalidCreateTimeSettings()));
}
public void testEmptyGroup() {
Job.Builder builder = buildJobBuilder("foo");
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> builder.setGroups(Arrays.asList("foo-group", "")));
assertThat(e.getMessage(), containsString("Invalid group id ''"));
}
public void testInvalidGroup() {
Job.Builder builder = buildJobBuilder("foo");
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> builder.setGroups(Arrays.asList("foo-group", "$$$")));
assertThat(e.getMessage(), containsString("Invalid group id '$$$'"));
}
public static Job.Builder buildJobBuilder(String id, Date date) {
Job.Builder builder = new Job.Builder(id);
builder.setCreateTime(date);
@ -500,6 +517,14 @@ public class JobTests extends AbstractSerializingTestCase<Job> {
if (randomBoolean()) {
builder.setJobVersion(Version.CURRENT);
}
if (randomBoolean()) {
int groupsNum = randomIntBetween(0, 10);
List<String> groups = new ArrayList<>(groupsNum);
for (int i = 0; i < groupsNum; i++) {
groups.add(randomValidJobId());
}
builder.setGroups(groups);
}
builder.setCreateTime(new Date(randomNonNegativeLong()));
if (randomBoolean()) {
builder.setFinishedTime(new Date(randomNonNegativeLong()));

11
plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java
View File

@ -24,6 +24,14 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
@Override
protected JobUpdate createTestInstance() {
JobUpdate.Builder update = new JobUpdate.Builder(randomAlphaOfLength(4));
if (randomBoolean()) {
int groupsNum = randomIntBetween(0, 10);
List<String> groups = new ArrayList<>(groupsNum);
for (int i = 0; i < groupsNum; i++) {
groups.add(JobTests.randomValidJobId());
}
update.setGroups(groups);
}
if (randomBoolean()) {
update.setDescription(randomAlphaOfLength(20));
}
@ -106,6 +114,7 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
Map<String, Object> customSettings = Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10));
JobUpdate.Builder updateBuilder = new JobUpdate.Builder("foo");
updateBuilder.setGroups(Arrays.asList("group-1", "group-2"));
updateBuilder.setDescription("updated_description");
updateBuilder.setDetectorUpdates(detectorUpdates);
updateBuilder.setModelPlotConfig(modelPlotConfig);
@ -120,6 +129,7 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
JobUpdate update = updateBuilder.build();
Job.Builder jobBuilder = new Job.Builder("foo");
jobBuilder.setGroups(Arrays.asList("group-1"));
Detector.Builder d1 = new Detector.Builder("info_content", "domain");
d1.setOverFieldName("mlcategory");
Detector.Builder d2 = new Detector.Builder("min", "field");
@ -132,6 +142,7 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
Job updatedJob = update.mergeWithJob(jobBuilder.build());
assertEquals(update.getGroups(), updatedJob.getGroups());
assertEquals(update.getDescription(), updatedJob.getDescription());
assertEquals(update.getModelPlotConfig(), updatedJob.getModelPlotConfig());
assertEquals(update.getAnalysisLimits(), updatedJob.getAnalysisLimits());

98
plugin/src/test/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookupTests.java Normal file
View File

@ -0,0 +1,98 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.job.groups;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.job.config.Job;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.contains;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class GroupOrJobLookupTests extends ESTestCase {
public void testEmptyLookup_GivenAllowNoJobs() {
GroupOrJobLookup lookup = new GroupOrJobLookup(Collections.emptyList());
assertThat(lookup.expandJobIds("_all", true).isEmpty(), is(true));
assertThat(lookup.expandJobIds("*", true).isEmpty(), is(true));
assertThat(lookup.expandJobIds("foo*", true).isEmpty(), is(true));
expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("foo", true));
}
public void testEmptyLookup_GivenNotAllowNoJobs() {
GroupOrJobLookup lookup = new GroupOrJobLookup(Collections.emptyList());
expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("_all", false));
expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("*", false));
expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("foo*", false));
expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("foo", true));
}
public void testAllIsNotExpandedInCommaSeparatedExpression() {
GroupOrJobLookup lookup = new GroupOrJobLookup(Collections.emptyList());
ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("foo-*,_all", true));
assertThat(e.getMessage(), equalTo("No known job with id '_all'"));
}
public void testConstructor_GivenJobWithSameIdAsPreviousGroupName() {
List<Job> jobs = new ArrayList<>();
jobs.add(mockJob("foo", Arrays.asList("foo-group")));
jobs.add(mockJob("foo-group", Collections.emptyList()));
ResourceAlreadyExistsException e = expectThrows(ResourceAlreadyExistsException.class, () -> new GroupOrJobLookup(jobs));
assertThat(e.getMessage(),
equalTo("job and group names must be unique but job [foo-group] and group [foo-group] have the same name"));
}
public void testConstructor_GivenGroupWithSameNameAsPreviousJobId() {
List<Job> jobs = new ArrayList<>();
jobs.add(mockJob("foo", Collections.emptyList()));
jobs.add(mockJob("foo-2", Arrays.asList("foo")));
ResourceAlreadyExistsException e = expectThrows(ResourceAlreadyExistsException.class, () -> new GroupOrJobLookup(jobs));
assertThat(e.getMessage(),
equalTo("job and group names must be unique but job [foo] and group [foo] have the same name"));
}
public void testLookup() {
List<Job> jobs = new ArrayList<>();
jobs.add(mockJob("foo-1", Arrays.asList("foo-group", "ones")));
jobs.add(mockJob("foo-2", Arrays.asList("foo-group", "twos")));
jobs.add(mockJob("bar-1", Arrays.asList("bar-group", "ones")));
jobs.add(mockJob("bar-2", Arrays.asList("bar-group", "twos")));
jobs.add(mockJob("nogroup", Collections.emptyList()));
GroupOrJobLookup groupOrJobLookup = new GroupOrJobLookup(jobs);
assertThat(groupOrJobLookup.expandJobIds("_all", false), contains("bar-1", "bar-2", "foo-1", "foo-2", "nogroup"));
assertThat(groupOrJobLookup.expandJobIds("*", false), contains("bar-1", "bar-2", "foo-1", "foo-2", "nogroup"));
assertThat(groupOrJobLookup.expandJobIds("bar-1", false), contains("bar-1"));
assertThat(groupOrJobLookup.expandJobIds("foo-1", false), contains("foo-1"));
assertThat(groupOrJobLookup.expandJobIds("foo-2, bar-1", false), contains("bar-1", "foo-2"));
assertThat(groupOrJobLookup.expandJobIds("foo-group", false), contains("foo-1", "foo-2"));
assertThat(groupOrJobLookup.expandJobIds("bar-group", false), contains("bar-1", "bar-2"));
assertThat(groupOrJobLookup.expandJobIds("ones", false), contains("bar-1", "foo-1"));
assertThat(groupOrJobLookup.expandJobIds("twos", false), contains("bar-2", "foo-2"));
assertThat(groupOrJobLookup.expandJobIds("foo-group, nogroup", false), contains("foo-1", "foo-2", "nogroup"));
assertThat(groupOrJobLookup.expandJobIds("*-group", false), contains("bar-1", "bar-2", "foo-1", "foo-2"));
assertThat(groupOrJobLookup.expandJobIds("foo-group,foo-1,foo-2", false), contains("foo-1", "foo-2"));
assertThat(groupOrJobLookup.expandJobIds("foo-group,*-2", false), contains("bar-2", "foo-1", "foo-2"));
}
private static Job mockJob(String jobId, List<String> groups) {
Job job = mock(Job.class);
when(job.getId()).thenReturn(jobId);
when(job.getGroups()).thenReturn(groups);
return job;
}
}

35
plugin/src/test/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobTests.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.job.groups;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.job.config.Job;
import java.util.Arrays;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.is;
import static org.mockito.Mockito.mock;
public class GroupOrJobTests extends ESTestCase {
public void testSingleJob() {
Job job = mock(Job.class);
GroupOrJob groupOrJob = new GroupOrJob.SingleJob(job);
assertThat(groupOrJob.isGroup(), is(false));
assertThat(groupOrJob.jobs(), contains(job));
expectThrows(UnsupportedOperationException.class, () -> groupOrJob.jobs().add(mock(Job.class)));
}
public void testGroup() {
Job job1 = mock(Job.class);
Job job2 = mock(Job.class);
GroupOrJob groupOrJob = new GroupOrJob.Group(Arrays.asList(job1, job2));
assertThat(groupOrJob.isGroup(), is(true));
assertThat(groupOrJob.jobs(), contains(job1, job2));
expectThrows(UnsupportedOperationException.class, () -> groupOrJob.jobs().add(mock(Job.class)));
}
}

2
plugin/src/test/java/org/elasticsearch/xpack/ml/utils/NameResolverTests.java
View File

@ -140,7 +140,7 @@ public class NameResolverTests extends ESTestCase {
@Override
protected List<String> lookup(String key) {
return lookup.get(key);
return lookup.containsKey(key) ? lookup.get(key) : Collections.emptyList();
}
}
}

59
plugin/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringF.java
View File

@ -1,59 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.node.MockNode;
import org.elasticsearch.node.Node;
import org.elasticsearch.xpack.XPackPlugin;
import java.io.IOException;
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
/**
* Main class to easily run Monitoring from a IDE.
* <p>
* In order to run this class set configure the following:
* 1) Set `-Des.path.home=` to a directory containing an ES config directory
*
* It accepts collectors names as program arguments.
*/
public class MonitoringF {
public static void main(String[] args) throws Throwable {
Settings.Builder settings = Settings.builder();
settings.put("security.manager.enabled", "false");
settings.put("cluster.name", MonitoringF.class.getSimpleName());
settings.put("xpack.monitoring.collection.interval", "1s");
if (!CollectionUtils.isEmpty(args)) {
settings.putArray("xpack.monitoring.collection.collectors", args);
}
final CountDownLatch latch = new CountDownLatch(1);
final Node node = new MockNode(settings.build(),
Arrays.asList(XPackPlugin.class, XPackPlugin.class, XPackPlugin.class));
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
try {
IOUtils.close(node);
} catch (IOException e) {
throw new ElasticsearchException(e);
} finally {
latch.countDown();
}
}
});
node.start();
latch.await();
}
}

123
plugin/src/test/java/org/elasticsearch/xpack/notification/email/ManualPublicSmtpServersTester.java
View File

@ -1,123 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.notification.email;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import java.util.Collections;
import java.util.Locale;
@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/379")
public class ManualPublicSmtpServersTester {
private static final Terminal terminal = Terminal.DEFAULT;
public static class Gmail {
public static void main(String[] args) throws Exception {
test("gmail", Profile.GMAIL, Settings.builder()
.put("xpack.notification.email.account.gmail.smtp.auth", true)
.put("xpack.notification.email.account.gmail.smtp.starttls.enable", true)
.put("xpack.notification.email.account.gmail.smtp.host", "smtp.gmail.com")
.put("xpack.notification.email.account.gmail.smtp.port", 587)
.put("xpack.notification.email.account.gmail.smtp.user", terminal.readText("username: "))
.put("xpack.notification.email.account.gmail.smtp.password", new String(terminal.readSecret("password: ")))
.put("xpack.notification.email.account.gmail.email_defaults.to", terminal.readText("to: "))
);
}
}
public static class OutlookDotCom {
public static void main(String[] args) throws Exception {
test("outlook", Profile.STANDARD, Settings.builder()
.put("xpack.notification.email.account.outlook.smtp.auth", true)
.put("xpack.notification.email.account.outlook.smtp.starttls.enable", true)
.put("xpack.notification.email.account.outlook.smtp.host", "smtp-mail.outlook.com")
.put("xpack.notification.email.account.outlook.smtp.port", 587)
.put("xpack.notification.email.account.outlook.smtp.user", "elastic.user@outlook.com")
.put("xpack.notification.email.account.outlook.smtp.password", "fantastic42")
.put("xpack.notification.email.account.outlook.email_defaults.to", "elastic.user@outlook.com")
.put()
);
}
}
public static class YahooMail {
public static void main(String[] args) throws Exception {
test("yahoo", Profile.STANDARD, Settings.builder()
.put("xpack.notification.email.account.yahoo.smtp.starttls.enable", true)
.put("xpack.notification.email.account.yahoo.smtp.auth", true)
.put("xpack.notification.email.account.yahoo.smtp.host", "smtp.mail.yahoo.com")
.put("xpack.notification.email.account.yahoo.smtp.port", 587)
.put("xpack.notification.email.account.yahoo.smtp.user", "elastic.user@yahoo.com")
.put("xpack.notification.email.account.yahoo.smtp.password", "fantastic42")
// note: from must be set to the same authenticated user account
.put("xpack.notification.email.account.yahoo.email_defaults.from", "elastic.user@yahoo.com")
.put("xpack.notification.email.account.yahoo.email_defaults.to", "elastic.user@yahoo.com")
);
}
}
// Amazon Simple Email Service
public static class SES {
public static void main(String[] args) throws Exception {
test("ses", Profile.STANDARD, Settings.builder()
.put("xpack.notification.email.account.ses.smtp.auth", true)
.put("xpack.notification.email.account.ses.smtp.starttls.enable", true)
.put("xpack.notification.email.account.ses.smtp.starttls.required", true)
.put("xpack.notification.email.account.ses.smtp.host", "email-smtp.us-east-1.amazonaws.com")
.put("xpack.notification.email.account.ses.smtp.port", 587)
.put("xpack.notification.email.account.ses.smtp.user", terminal.readText("user: "))
.put("xpack.notification.email.account.ses.email_defaults.from", "dummy.user@elasticsearch.com")
.put("xpack.notification.email.account.ses.email_defaults.to", terminal.readText("to: "))
.put("xpack.notification.email.account.ses.smtp.password",
new String(terminal.readSecret("password: ")))
);
}
}
static void test(String accountName, Profile profile, Settings.Builder settingsBuilder) throws Exception {
String path = "/org/elasticsearch/xpack/watcher/actions/email/service/logo.png";
if (EmailServiceTests.class.getResourceAsStream(path) == null) {
throw new ElasticsearchException("Could not find logo at path {}", path);
}
EmailService service = startEmailService(settingsBuilder);
ToXContent content = (xContentBuilder, params) -> xContentBuilder.startObject()
.field("key1", "value1")
.field("key2", "value2")
.field("key3", "value3")
.endObject();
Email email = Email.builder()
.id("_id")
.subject("_subject")
.textBody("_text_body")
.htmlBody("<b>html body</b><p/><p/><img src=\"cid:logo.png\"/>")
.attach(new Attachment.XContent.Yaml("test.yml", content))
.attach(new Attachment.Stream("logo.png", "logo.png", true,
() -> EmailServiceTests.class.getResourceAsStream(path)))
.build();
EmailService.EmailSent sent = service.send(email, null, profile, accountName);
terminal.println(String.format(Locale.ROOT, "email sent via account [%s]", sent.account()));
}
static EmailService startEmailService(Settings.Builder builder) {
Settings settings = builder.build();
return new EmailService(settings, null,
new ClusterSettings(settings, Collections.singleton(EmailService.EMAIL_ACCOUNT_SETTING)));
}
}

81
plugin/src/test/java/org/elasticsearch/xpack/security/SecurityF.java
View File

@ -1,81 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.security;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.node.MockNode;
import org.elasticsearch.node.Node;
import org.elasticsearch.xpack.security.authc.esnative.NativeRealm;
import org.elasticsearch.xpack.security.authc.file.FileRealm;
import org.elasticsearch.xpack.security.test.SecurityTestUtils;
import org.elasticsearch.test.SecuritySettingsSource;
import org.elasticsearch.xpack.XPackPlugin;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import static org.elasticsearch.xpack.security.test.SecurityTestUtils.writeFile;
/**
* Main class to easily run X-Pack Security from a IDE.
*
* During startup an error will be printed that the config directory can't be found, to fix this:
* set `-Des.path.home=` to a location where there is a config directory on your machine.
*/
public class SecurityF {
public static void main(String[] args) throws Throwable {
Settings.Builder settings = Settings.builder();
settings.put("http.cors.enabled", "true");
settings.put("http.cors.allow-origin", "*");
settings.put("xpack.security.enabled", "true");
// Disable Monitoring to prevent cluster activity
settings.put("xpack.monitoring.enabled", "false");
settings.put("cluster.name", SecurityF.class.getSimpleName());
String homeDir = System.getProperty("es.path.home");
if (homeDir == null || Files.exists(PathUtils.get(homeDir)) == false) {
throw new IllegalStateException("es.path.home must be set and exist");
}
final Path config = PathUtils.get(homeDir).resolve("config");
SecurityTestUtils.createFolder(config);
final Path folder = config.resolve("x-pack");
SecurityTestUtils.createFolder(folder);
writeFile(folder, "users", SecuritySettingsSource.CONFIG_STANDARD_USER);
writeFile(folder, "users_roles", SecuritySettingsSource.CONFIG_STANDARD_USER_ROLES);
writeFile(folder, "roles.yml", SecuritySettingsSource.CONFIG_ROLE_ALLOW_ALL);
settings.put("xpack.security.authc.realms.file.type", FileRealm.TYPE);
settings.put("xpack.security.authc.realms.file.order", "0");
settings.put("xpack.security.authc.realms.esnative.type", NativeRealm.TYPE);
settings.put("xpack.security.authc.realms.esnative.order", "1");
final CountDownLatch latch = new CountDownLatch(1);
final Node node = new MockNode(settings.build(), Arrays.asList(XPackPlugin.class), config);
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
try {
IOUtils.close(node);
} catch (IOException ex) {
throw new ElasticsearchException(ex);
} finally {
latch.countDown();
}
}
});
node.start();
latch.await();
}
}

182
plugin/src/test/java/org/elasticsearch/xpack/watcher/OldWatcherIndicesBackwardsCompatibilityTests.java
View File

@ -1,182 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.watcher;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
import org.elasticsearch.AbstractOldXPackIndicesBackwardsCompatibilityTestCase;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.common.text.TextTemplate;
import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction;
import org.elasticsearch.xpack.watcher.client.WatchSourceBuilder;
import org.elasticsearch.xpack.watcher.client.WatcherClient;
import org.elasticsearch.xpack.watcher.condition.AlwaysCondition;
import org.elasticsearch.xpack.watcher.transport.actions.get.GetWatchResponse;
import org.elasticsearch.xpack.watcher.transport.actions.put.PutWatchResponse;
import org.elasticsearch.xpack.watcher.transport.actions.stats.WatcherStatsResponse;
import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule;
import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule.Interval;
import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
import static org.hamcrest.Matchers.everyItem;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.startsWith;
/**
* Tests for watcher indexes created before 5.0.
*/
// This will only work, when the upgrade API is in place!
@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1303")
public class OldWatcherIndicesBackwardsCompatibilityTests extends AbstractOldXPackIndicesBackwardsCompatibilityTestCase {
@Override
public Settings nodeSettings(int ord) {
return Settings.builder()
.put(super.nodeSettings(ord))
.put(XPackSettings.WATCHER_ENABLED.getKey(), true)
.build();
}
@Override
protected void checkVersion(Version version) throws Exception {
// Wait for watcher to actually start....
WatcherClient watcherClient = new WatcherClient(client());
watcherClient.prepareWatchService().start();
assertBusy(() -> {
List<WatcherState> states = watcherClient.prepareWatcherStats().get().getNodes()
.stream().map(WatcherStatsResponse.Node::getWatcherState).collect(Collectors.toList());
assertThat(states, everyItem(is(WatcherState.STARTED)));
});
try {
assertOldTemplatesAreDeleted();
assertWatchIndexContentsWork(version);
assertBasicWatchInteractions();
} finally {
/* Shut down watcher after every test because watcher can be a bit finicky about shutting down when the node shuts down. This
* makes super sure it shuts down *and* causes the test to fail in a sensible spot if it doesn't shut down. */
watcherClient.prepareWatchService().stop().get();
assertBusy(() -> internalCluster().getInstances(WatcherService.class)
.forEach(watcherService -> assertThat(watcherService.state(), is(WatcherState.STOPPED))));
}
}
private void assertOldTemplatesAreDeleted() {
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
List<String> templateNames = response.getIndexTemplates().stream().map(IndexTemplateMetaData::getName).collect(Collectors.toList());
assertThat(templateNames, not(hasItems(is("watches"), startsWith("watch-history"), is("triggered_watches"))));
}
void assertWatchIndexContentsWork(Version version) throws Exception {
WatcherClient watcherClient = new WatcherClient(client());
// Fetch a basic watch
GetWatchResponse bwcWatch = watcherClient.prepareGetWatch("bwc_watch").get();
assertTrue(bwcWatch.isFound());
assertNotNull(bwcWatch.getSource());
Map<String, Object> source = bwcWatch.getSource().getAsMap();
assertEquals(1000, source.get("throttle_period_in_millis"));
Map<?, ?> input = (Map<?, ?>) source.get("input");
Map<?, ?> search = (Map<?, ?>) input.get("search");
// We asked for 100s but 2.x converted that to 1.6m which is actually 96s...
int timeout = (int) (version.onOrAfter(Version.V_5_0_0_alpha1) ? timeValueSeconds(100).millis() : timeValueSeconds(96).millis());
assertEquals(timeout, search.get("timeout_in_millis"));
Map<?, ?> actions = (Map<?, ?>) source.get("actions");
Map<?, ?> indexPayload = (Map<?, ?>) actions.get("index_payload");
Map<?, ?> transform = (Map<?, ?>) indexPayload.get("transform");
search = (Map<?, ?>) transform.get("search");
assertEquals(timeout, search.get("timeout_in_millis"));
Map<?, ?> index = (Map<?, ?>) indexPayload.get("index");
assertEquals("bwc_watch_index", index.get("index"));
assertEquals("bwc_watch_type", index.get("doc_type"));
assertEquals(timeout, index.get("timeout_in_millis"));
// Fetch a watch with "fun" throttle periods
bwcWatch = watcherClient.prepareGetWatch("bwc_throttle_period").get();
assertTrue(bwcWatch.isFound());
assertNotNull(bwcWatch.getSource());
source = bwcWatch.getSource().getAsMap();
assertEquals(timeout, source.get("throttle_period_in_millis"));
actions = (Map<?, ?>) source.get("actions");
indexPayload = (Map<?, ?>) actions.get("index_payload");
assertEquals(timeout, indexPayload.get("throttle_period_in_millis"));
/*
* Fetch a watch with a funny timeout to verify loading fractional time
* values. This watch is only built in >= 2.3 because email attachments
* aren't supported before that.
*/
bwcWatch = watcherClient.prepareGetWatch("bwc_funny_timeout").get();
assertTrue(bwcWatch.isFound());
assertNotNull(bwcWatch.getSource());
source = bwcWatch.getSource().getAsMap();
actions = (Map<?, ?>) source.get("actions");
Map<?, ?> work = (Map<?, ?>) actions.get("work");
Map<?, ?> email = (Map<?, ?>) work.get("email");
Map<?, ?> attachments = (Map<?, ?>) email.get("attachments");
Map<?, ?> attachment = (Map<?, ?>) attachments.get("test_report.pdf");
Map<?, ?> http = (Map<?, ?>) attachment.get("http");
Map<?, ?> request = (Map<?, ?>) http.get("request");
assertEquals(timeout, request.get("read_timeout_millis"));
assertEquals("https", request.get("scheme"));
assertEquals("example.com", request.get("host"));
assertEquals("{{ctx.metadata.report_url}}", request.get("path"));
assertEquals(8443, request.get("port"));
Map<?, ?> auth = (Map<?, ?>) request.get("auth");
Map<?, ?> basic = (Map<?, ?>) auth.get("basic");
assertThat(basic, hasEntry("username", "Aladdin"));
// password doesn't come back because it is hidden
assertThat(basic, not(hasKey("password")));
String watchHistoryPattern = version.onOrAfter(Version.V_5_0_0_alpha1) ? ".watcher-history*" : ".watch_history*";
SearchResponse history = client().prepareSearch(watchHistoryPattern).get();
assertThat(history.getHits().getTotalHits(), greaterThanOrEqualTo(10L));
}
void assertBasicWatchInteractions() throws Exception {
WatcherClient watcherClient = new WatcherClient(client());
PutWatchResponse put = watcherClient.preparePutWatch("new_watch").setSource(new WatchSourceBuilder()
.condition(AlwaysCondition.INSTANCE)
.trigger(ScheduleTrigger.builder(new IntervalSchedule(Interval.seconds(1))))
.addAction("awesome", LoggingAction.builder(new TextTemplate("test")))).get();
assertTrue(put.isCreated());
assertEquals(1, put.getVersion());
put = watcherClient.preparePutWatch("new_watch").setSource(new WatchSourceBuilder()
.condition(AlwaysCondition.INSTANCE)
.trigger(ScheduleTrigger.builder(new IntervalSchedule(Interval.seconds(1))))
.addAction("awesome", LoggingAction.builder(new TextTemplate("test")))).get();
assertFalse(put.isCreated());
assertEquals(2, put.getVersion());
GetWatchResponse get = watcherClient.prepareGetWatch(put.getId()).get();
assertTrue(get.isFound());
{
Map<?, ?> source = get.getSource().getAsMap();
Map<?, ?> actions = (Map<?, ?>) source.get("actions");
Map<?, ?> awesome = (Map<?, ?>) actions.get("awesome");
Map<?, ?> logging = (Map<?, ?>) awesome.get("logging");
assertEquals("info", logging.get("level"));
assertEquals("test", logging.get("text"));
}
}
}

77
plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherF.java
View File

@ -1,77 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.watcher;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.MockNode;
import org.elasticsearch.node.Node;
import org.elasticsearch.xpack.XPackPlugin;
import java.io.IOException;
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
/**
* Main class to easily run Watcher from a IDE.
* It sets all the options to run the Watcher plugin and access it from Sense, but doesn't run with security.
*
* In order to run this class set configure the following:
* 1) Set `-Des.path.home=` to a directory containing an ES config directory
*/
public class WatcherF {
@SuppressForbidden(reason = "not really code or a test")
public static void main(String[] args) throws Throwable {
Settings.Builder settings = Settings.builder();
settings.put("http.cors.enabled", "true");
settings.put("http.cors.allow-origin", "*");
settings.put("xpack.security.enabled", "false");
settings.put("security.manager.enabled", "false");
settings.put("cluster.name", WatcherF.class.getSimpleName());
// this is for the `test-watcher-integration` group level integration in HipChat
settings.put("xpack.notification.hipchat.account.integration.profile", "integration");
settings.put("xpack.notification.hipchat.account.integration.auth_token", "huuS9v7ccuOy3ZBWWWr1vt8Lqu3sQnLUE81nrLZU");
settings.put("xpack.notification.hipchat.account.integration.room", "test-watcher");
// this is for the Watcher Test account in HipChat
settings.put("xpack.notification.hipchat.account.user.profile", "user");
settings.put("xpack.notification.hipchat.account.user.auth_token", "12rNQUuQ0wObfRVeoVD8OeoAnosCT8tSTV5UjsII");
// this is for the `test-watcher-v1` notification token (hipchat)
settings.put("xpack.notification.hipchat.account.v1.profile", "v1");
settings.put("xpack.notification.hipchat.account.v1.auth_token", "a734baf62df618b96dda55b323fc30");
// this is for our test slack incoming webhook (under elasticsearch team)
System.setProperty("es.xpack.notification.slack.account.a1.url",
"https://hooks.slack.com/services/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");
System.setProperty("es.xpack.notification.pagerduty.account.service1.service_api_key",
"fc082467005d4072a914e0bb041882d0");
final CountDownLatch latch = new CountDownLatch(1);
final Node node = new MockNode(settings.build(), Arrays.asList(XPackPlugin.class, XPackPlugin.class));
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
try {
IOUtils.close(node);
} catch (IOException e) {
throw new ElasticsearchException(e);
} finally {
latch.countDown();
}
}
});
node.start();
latch.await();
}
}

32
plugin/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTransformMappingsTests.java
View File

@ -17,7 +17,9 @@ import org.elasticsearch.xpack.watcher.transport.actions.put.PutWatchResponse;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.loggingAction;
@ -26,7 +28,7 @@ import static org.elasticsearch.xpack.watcher.input.InputBuilders.simpleInput;
import static org.elasticsearch.xpack.watcher.transform.TransformBuilders.scriptTransform;
import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.is;
/**
@ -69,20 +71,7 @@ public class HistoryTemplateTransformMappingsTests extends AbstractWatcherIntegr
return true; // just to have better control over the triggers
}
@Override
protected boolean enableSecurity() {
return false; // remove security noise from this test
}
@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1517")
public void testTransformFields() throws Exception {
String index = "the-index";
String type = "the-type";
createIndex(index);
index(index, type, "{}");
flush();
refresh();
PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id1").setSource(watchBuilder()
.trigger(schedule(interval("5s")))
.input(simpleInput())
@ -121,12 +110,15 @@ public class HistoryTemplateTransformMappingsTests extends AbstractWatcherIntegr
.includeDefaults(true)
.get();
for (Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>> map : response.mappings().values()) {
Map<String, GetFieldMappingsResponse.FieldMappingMetaData> watchRecord = map.get("doc");
assertThat(watchRecord, hasKey("result.actions.transform.payload"));
GetFieldMappingsResponse.FieldMappingMetaData fieldMappingMetaData = watchRecord.get("result.actions.transform.payload");
assertThat(fieldMappingMetaData.isNull(), is(true));
}
// time might have rolled over to a new day, thus we need to check that this field exists only in one of the history indices
List<Boolean> payloadNulls = response.mappings().values().stream()
.map(map -> map.get("doc"))
.map(map -> map.get("result.actions.transform.payload"))
.filter(Objects::nonNull)
.map(GetFieldMappingsResponse.FieldMappingMetaData::isNull)
.collect(Collectors.toList());
assertThat(payloadNulls, hasItem(true));
});
}
}

1
plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java
View File

@ -183,7 +183,6 @@ public class BootStrapTests extends AbstractWatcherIntegrationTestCase {
});
}
@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/pull/1544")
public void testMixedTriggeredWatchLoading() throws Exception {
createIndex("output");
client().prepareIndex("my-index", "foo", "bar")

19
plugin/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/tool/EvalCron.java
View File

@ -1,19 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.watcher.trigger.schedule.tool;
import org.elasticsearch.cli.Terminal;
/**
* A small executable tool that can eval crons
*/
public class EvalCron {
public static void main(String[] args) throws Exception {
String expression = Terminal.DEFAULT.readText("cron: ");
CronEvalTool.main(new String[] { expression });
}
}

245
plugin/src/test/resources/rest-api-spec/test/ml/job_groups.yml Normal file
View File

@ -0,0 +1,245 @@
setup:
- do:
xpack.ml.put_job:
job_id: test-job-groups-foo-1
body: >
{
"groups": ["foo-group", "ones"],
"analysis_config" : {
"detectors" :[{"function":"count"}]
},
"data_description" : {}
}
- do:
xpack.ml.put_job:
job_id: test-job-groups-foo-2
body: >
{
"groups": ["foo-group", "twos"],
"analysis_config" : {
"detectors" :[{"function":"count"}]
},
"data_description" : {}
}
- do:
xpack.ml.put_job:
job_id: test-job-groups-bar-1
body: >
{
"groups": ["bar-group", "ones"],
"analysis_config" : {
"detectors" :[{"function":"count"}]
},
"data_description" : {}
}
- do:
xpack.ml.put_job:
job_id: test-job-groups-bar-2
body: >
{
"groups": ["bar-group", "twos"],
"analysis_config" : {
"detectors" :[{"function":"count"}]
},
"data_description" : {}
}
- do:
xpack.ml.put_job:
job_id: test-job-groups-nogroup
body: >
{
"analysis_config" : {
"detectors" :[{"function":"count"}]
},
"data_description" : {}
}
---
"Test put job with id that matches an existing group":
- do:
catch: /resource_already_exists_exception/
xpack.ml.put_job:
job_id: foo-group
body: >
{
"analysis_config" : {
"detectors" :[{"function":"count"}]
},
"data_description" : {}
}
---
"Test put job with group that matches an job id":
- do:
catch: /resource_already_exists_exception/
xpack.ml.put_job:
job_id: test-job-groups-job-with-group-matching-existing-job-id
body: >
{
"groups": ["test-job-groups-nogroup"],
"analysis_config" : {
"detectors" :[{"function":"count"}]
},
"data_description" : {}
}
---
"Test put job with group that matches its id":
- do:
catch: /resource_already_exists_exception/
xpack.ml.put_job:
job_id: test-job-groups-job-with-group-matching-its-id
body: >
{
"groups": ["test-job-groups-job-with-group-matching-its-id"],
"analysis_config" : {
"detectors" :[{"function":"count"}]
},
"data_description" : {}
}
---
"Test put job with empty group":
- do:
catch: /Invalid group id ''; must be non-empty string and may contain lowercase alphanumeric \(a-z and 0-9\), hyphens or underscores; must start and end with alphanumeric/
xpack.ml.put_job:
job_id: test-job-groups-job-with-empty-group
body: >
{
"groups": ["foo-group", ""],
"analysis_config" : {
"detectors" :[{"function":"count"}]
},
"data_description" : {}
}
---
"Test put job with invalid group":
- do:
catch: /Invalid group id '___'; must be non-empty string and may contain lowercase alphanumeric \(a-z and 0-9\), hyphens or underscores; must start and end with alphanumeric/
xpack.ml.put_job:
job_id: test-job-groups-job-with-invalid-group
body: >
{
"groups": ["foo", "___"],
"analysis_config" : {
"detectors" :[{"function":"count"}]
},
"data_description" : {}
}
---
"Test get job API":
- do:
xpack.ml.get_jobs:
job_id: "_all"
- match: { count: 5 }
- match: { jobs.0.job_id: "test-job-groups-bar-1"}
- match: { jobs.1.job_id: "test-job-groups-bar-2"}
- match: { jobs.2.job_id: "test-job-groups-foo-1"}
- match: { jobs.3.job_id: "test-job-groups-foo-2"}
- match: { jobs.4.job_id: "test-job-groups-nogroup"}
- do:
xpack.ml.get_jobs:
job_id: "test-job-groups-bar-1"
- match: { count: 1 }
- match: { jobs.0.job_id: "test-job-groups-bar-1"}
- do:
xpack.ml.get_jobs:
job_id: "foo-group"
- match: { count: 2 }
- match: { jobs.0.job_id: "test-job-groups-foo-1"}
- match: { jobs.1.job_id: "test-job-groups-foo-2"}
- do:
xpack.ml.get_jobs:
job_id: "bar-group"
- match: { count: 2 }
- match: { jobs.0.job_id: "test-job-groups-bar-1"}
- match: { jobs.1.job_id: "test-job-groups-bar-2"}
- do:
xpack.ml.get_jobs:
job_id: "ones"
- match: { count: 2 }
- match: { jobs.0.job_id: "test-job-groups-bar-1"}
- match: { jobs.1.job_id: "test-job-groups-foo-1"}
- do:
xpack.ml.get_jobs:
job_id: "twos"
- match: { count: 2 }
- match: { jobs.0.job_id: "test-job-groups-bar-2"}
- match: { jobs.1.job_id: "test-job-groups-foo-2"}
- do:
xpack.ml.get_jobs:
job_id: "*-group"
- match: { count: 4 }
- match: { jobs.0.job_id: "test-job-groups-bar-1"}
- match: { jobs.1.job_id: "test-job-groups-bar-2"}
- match: { jobs.2.job_id: "test-job-groups-foo-1"}
- match: { jobs.3.job_id: "test-job-groups-foo-2"}
- do:
xpack.ml.get_jobs:
job_id: "bar-group,test-job-groups-nogroup"
- match: { count: 3 }
- match: { jobs.0.job_id: "test-job-groups-bar-1"}
- match: { jobs.1.job_id: "test-job-groups-bar-2"}
- match: { jobs.2.job_id: "test-job-groups-nogroup"}
---
"Test get job stats API":
- do:
xpack.ml.get_job_stats:
job_id: "foo-group"
- match: { count: 2 }
- match: { jobs.0.job_id: "test-job-groups-foo-1"}
- match: { jobs.1.job_id: "test-job-groups-foo-2"}
---
"Test close job API":
- do:
xpack.ml.open_job:
job_id: "test-job-groups-foo-1"
- do:
xpack.ml.open_job:
job_id: "test-job-groups-bar-1"
- do:
xpack.ml.get_job_stats:
job_id: "ones"
- match: { count: 2 }
- match: { jobs.0.job_id: "test-job-groups-bar-1"}
- match: { jobs.0.state: opened}
- match: { jobs.1.job_id: "test-job-groups-foo-1"}
- match: { jobs.1.state: opened}
- do:
xpack.ml.close_job:
job_id: "ones"
- do:
xpack.ml.get_job_stats:
job_id: "ones"
- match: { count: 2 }
- match: { jobs.0.job_id: "test-job-groups-bar-1"}
- match: { jobs.0.state: closed}
- match: { jobs.1.job_id: "test-job-groups-foo-1"}
- match: { jobs.1.state: closed}

2
plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
View File

@ -296,6 +296,7 @@
job_id: jobs-crud-update-job
body: >
{
"groups": ["group-1", "group-2"],
"description":"Post update description",
"detectors": [{"detector_index": 0, "rules": {"target_field_name": "airline",
"rule_conditions": [ { "condition_type": "numerical_actual",
@ -318,6 +319,7 @@
}
}
- match: { job_id: "jobs-crud-update-job" }
- match: { groups: ["group-1", "group-2"] }
- match: { description: "Post update description" }
- match: { model_plot_config.enabled: false }
- match: { model_plot_config.terms: "foobar" }