honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into index-lifecycle

This commit is contained in:
Colin Goodheart-Smithe 2018-09-13 09:46:14 +01:00
parent 5a349ec0e6 7d3b99a9b7
commit 8e59de3eb2
No known key found for this signature in database
GPG Key ID: F975E7BDD739B3C7
289 changed files with 11262 additions and 3482 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View File

@ -831,6 +831,9 @@ class BuildPlugin implements Plugin<Project> {
// TODO: remove this once ctx isn't added to update script params in 7.0
systemProperty 'es.scripting.update.ctx_in_params', 'false'
//TODO: remove this once the cname is prepended to the address by default in 7.0
systemProperty 'es.http.cname_in_publish_address', 'true'
// Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
if (project.inFipsJvm) {
systemProperty 'javax.net.ssl.trustStorePassword', 'password'

2
buildSrc/version.properties
View File

@ -16,7 +16,7 @@ slf4j = 1.6.2
jna = 4.5.1
# test dependencies
randomizedrunner = 2.5.2
randomizedrunner = 2.7.0
junit = 4.12
httpclient = 4.5.2
# When updating httpcore, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy

49
client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
View File

@ -28,10 +28,12 @@ import org.apache.http.entity.ByteArrayEntity;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.client.RequestConverters.EndpointBuilder;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.DeleteForecastRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
import org.elasticsearch.client.ml.FlushJobRequest;
import org.elasticsearch.client.ml.ForecastJobRequest;
import org.elasticsearch.client.ml.GetBucketsRequest;
import org.elasticsearch.client.ml.GetCategoriesRequest;
import org.elasticsearch.client.ml.GetInfluencersRequest;
import org.elasticsearch.client.ml.GetJobRequest;
import org.elasticsearch.client.ml.GetJobStatsRequest;
@ -39,6 +41,7 @@ import org.elasticsearch.client.ml.GetOverallBucketsRequest;
import org.elasticsearch.client.ml.GetRecordsRequest;
import org.elasticsearch.client.ml.OpenJobRequest;
import org.elasticsearch.client.ml.PostDataRequest;
import org.elasticsearch.client.ml.PutDatafeedRequest;
import org.elasticsearch.client.ml.PutJobRequest;
import org.elasticsearch.client.ml.UpdateJobRequest;
import org.elasticsearch.common.Strings;
@ -180,6 +183,38 @@ final class MLRequestConverters {
return request;
}
static Request putDatafeed(PutDatafeedRequest putDatafeedRequest) throws IOException {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_xpack")
.addPathPartAsIs("ml")
.addPathPartAsIs("datafeeds")
.addPathPart(putDatafeedRequest.getDatafeed().getId())
.build();
Request request = new Request(HttpPut.METHOD_NAME, endpoint);
request.setEntity(createEntity(putDatafeedRequest, REQUEST_BODY_CONTENT_TYPE));
return request;
}
static Request deleteForecast(DeleteForecastRequest deleteForecastRequest) throws IOException {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_xpack")
.addPathPartAsIs("ml")
.addPathPartAsIs("anomaly_detectors")
.addPathPart(deleteForecastRequest.getJobId())
.addPathPartAsIs("_forecast")
.addPathPart(Strings.collectionToCommaDelimitedString(deleteForecastRequest.getForecastIds()))
.build();
Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
RequestConverters.Params params = new RequestConverters.Params(request);
if (deleteForecastRequest.isAllowNoForecasts() != null) {
params.putParam("allow_no_forecasts", Boolean.toString(deleteForecastRequest.isAllowNoForecasts()));
}
if (deleteForecastRequest.timeout() != null) {
params.putParam("timeout", deleteForecastRequest.timeout().getStringRep());
}
return request;
}
static Request getBuckets(GetBucketsRequest getBucketsRequest) throws IOException {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_xpack")
@ -194,6 +229,20 @@ final class MLRequestConverters {
return request;
}
static Request getCategories(GetCategoriesRequest getCategoriesRequest) throws IOException {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_xpack")
.addPathPartAsIs("ml")
.addPathPartAsIs("anomaly_detectors")
.addPathPart(getCategoriesRequest.getJobId())
.addPathPartAsIs("results")
.addPathPartAsIs("categories")
.build();
Request request = new Request(HttpGet.METHOD_NAME, endpoint);
request.setEntity(createEntity(getCategoriesRequest, REQUEST_BODY_CONTENT_TYPE));
return request;
}
static Request getOverallBuckets(GetOverallBucketsRequest getOverallBucketsRequest) throws IOException {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_xpack")

145
client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
View File

@ -19,19 +19,20 @@
package org.elasticsearch.client;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.ml.ForecastJobRequest;
import org.elasticsearch.client.ml.ForecastJobResponse;
import org.elasticsearch.client.ml.PostDataRequest;
import org.elasticsearch.client.ml.PostDataResponse;
import org.elasticsearch.client.ml.UpdateJobRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.CloseJobResponse;
import org.elasticsearch.client.ml.DeleteForecastRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
import org.elasticsearch.client.ml.DeleteJobResponse;
import org.elasticsearch.client.ml.FlushJobRequest;
import org.elasticsearch.client.ml.FlushJobResponse;
import org.elasticsearch.client.ml.ForecastJobRequest;
import org.elasticsearch.client.ml.ForecastJobResponse;
import org.elasticsearch.client.ml.GetBucketsRequest;
import org.elasticsearch.client.ml.GetBucketsResponse;
import org.elasticsearch.client.ml.GetCategoriesRequest;
import org.elasticsearch.client.ml.GetCategoriesResponse;
import org.elasticsearch.client.ml.GetInfluencersRequest;
import org.elasticsearch.client.ml.GetInfluencersResponse;
import org.elasticsearch.client.ml.GetJobRequest;
@ -44,13 +45,19 @@ import org.elasticsearch.client.ml.GetRecordsRequest;
import org.elasticsearch.client.ml.GetRecordsResponse;
import org.elasticsearch.client.ml.OpenJobRequest;
import org.elasticsearch.client.ml.OpenJobResponse;
import org.elasticsearch.client.ml.PostDataRequest;
import org.elasticsearch.client.ml.PostDataResponse;
import org.elasticsearch.client.ml.PutDatafeedRequest;
import org.elasticsearch.client.ml.PutDatafeedResponse;
import org.elasticsearch.client.ml.PutJobRequest;
import org.elasticsearch.client.ml.PutJobResponse;
import org.elasticsearch.client.ml.UpdateJobRequest;
import org.elasticsearch.client.ml.job.stats.JobStats;
import java.io.IOException;
import java.util.Collections;
/**
* Machine Learning API client wrapper for the {@link RestHighLevelClient}
*
@ -387,6 +394,11 @@ public final class MachineLearningClient {
/**
* Updates a Machine Learning {@link org.elasticsearch.client.ml.job.config.Job}
*
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-update-job.html"></a>
* </p>
*
* @param request the {@link UpdateJobRequest} object enclosing the desired updates
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return a PutJobResponse object containing the updated job object
@ -425,6 +437,10 @@ public final class MachineLearningClient {
/**
* Updates a Machine Learning {@link org.elasticsearch.client.ml.job.config.Job} asynchronously
*
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-update-job.html"></a>
* </p>
* @param request the {@link UpdateJobRequest} object enclosing the desired updates
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener Listener to be notified upon request completion
@ -438,6 +454,86 @@ public final class MachineLearningClient {
Collections.emptySet());
}
/**
* Creates a new Machine Learning Datafeed
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-datafeed.html">ML PUT datafeed documentation</a>
*
* @param request The PutDatafeedRequest containing the {@link org.elasticsearch.client.ml.datafeed.DatafeedConfig} settings
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return PutDatafeedResponse with enclosed {@link org.elasticsearch.client.ml.datafeed.DatafeedConfig} object
* @throws IOException when there is a serialization issue sending the request or receiving the response
*/
public PutDatafeedResponse putDatafeed(PutDatafeedRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(request,
MLRequestConverters::putDatafeed,
options,
PutDatafeedResponse::fromXContent,
Collections.emptySet());
}
/**
* Creates a new Machine Learning Datafeed asynchronously and notifies listener on completion
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-datafeed.html">ML PUT datafeed documentation</a>
*
* @param request The request containing the {@link org.elasticsearch.client.ml.datafeed.DatafeedConfig} settings
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener Listener to be notified upon request completion
*/
public void putDatafeedAsync(PutDatafeedRequest request, RequestOptions options, ActionListener<PutDatafeedResponse> listener) {
restHighLevelClient.performRequestAsyncAndParseEntity(request,
MLRequestConverters::putDatafeed,
options,
PutDatafeedResponse::fromXContent,
listener,
Collections.emptySet());
}
/**
* Deletes Machine Learning Job Forecasts
*
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-forecast.html"></a>
* </p>
*
* @param request the {@link DeleteForecastRequest} object enclosing the desired jobId, forecastIDs, and other options
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return a AcknowledgedResponse object indicating request success
* @throws IOException when there is a serialization issue sending the request or receiving the response
*/
public AcknowledgedResponse deleteForecast(DeleteForecastRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(request,
MLRequestConverters::deleteForecast,
options,
AcknowledgedResponse::fromXContent,
Collections.emptySet());
}
/**
* Deletes Machine Learning Job Forecasts asynchronously
*
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-forecast.html"></a>
* </p>
*
* @param request the {@link DeleteForecastRequest} object enclosing the desired jobId, forecastIDs, and other options
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener Listener to be notified upon request completion
*/
public void deleteForecastAsync(DeleteForecastRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
restHighLevelClient.performRequestAsyncAndParseEntity(request,
MLRequestConverters::deleteForecast,
options,
AcknowledgedResponse::fromXContent,
listener,
Collections.emptySet());
}
/**
* Gets the buckets for a Machine Learning Job.
* <p>
@ -474,6 +570,45 @@ public final class MachineLearningClient {
Collections.emptySet());
}
/**
* Gets the categories for a Machine Learning Job.
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-category.html">
* ML GET categories documentation</a>
*
* @param request The request
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @throws IOException when there is a serialization issue sending the request or receiving the response
*/
public GetCategoriesResponse getCategories(GetCategoriesRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(request,
MLRequestConverters::getCategories,
options,
GetCategoriesResponse::fromXContent,
Collections.emptySet());
}
/**
* Gets the categories for a Machine Learning Job, notifies listener once the requested buckets are retrieved.
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-category.html">
* ML GET categories documentation</a>
*
* @param request The request
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener Listener to be notified upon request completion
*/
public void getCategoriesAsync(GetCategoriesRequest request, RequestOptions options, ActionListener<GetCategoriesResponse> listener) {
restHighLevelClient.performRequestAsyncAndParseEntity(request,
MLRequestConverters::getCategories,
options,
GetCategoriesResponse::fromXContent,
listener,
Collections.emptySet());
}
/**
* Gets overall buckets for a set of Machine Learning Jobs.
* <p>

183
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteForecastRequest.java Normal file
View File

@ -0,0 +1,183 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.client.ml.job.config.Job;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
/**
* POJO for a delete forecast request
*/
public class DeleteForecastRequest extends ActionRequest implements ToXContentObject {
public static final ParseField FORECAST_ID = new ParseField("forecast_id");
public static final ParseField ALLOW_NO_FORECASTS = new ParseField("allow_no_forecasts");
public static final ParseField TIMEOUT = new ParseField("timeout");
public static final String ALL = "_all";
public static final ConstructingObjectParser<DeleteForecastRequest, Void> PARSER =
new ConstructingObjectParser<>("delete_forecast_request", (a) -> new DeleteForecastRequest((String) a[0]));
static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
PARSER.declareStringOrNull(
(c, p) -> c.setForecastIds(Strings.commaDelimitedListToStringArray(p)), FORECAST_ID);
PARSER.declareBoolean(DeleteForecastRequest::setAllowNoForecasts, ALLOW_NO_FORECASTS);
PARSER.declareString(DeleteForecastRequest::timeout, TIMEOUT);
}
/**
* Create a new {@link DeleteForecastRequest} that explicitly deletes all forecasts
*
* @param jobId the jobId of the Job whose forecasts to delete
*/
public static DeleteForecastRequest deleteAllForecasts(String jobId) {
DeleteForecastRequest request = new DeleteForecastRequest(jobId);
request.setForecastIds(ALL);
return request;
}
private final String jobId;
private List<String> forecastIds = new ArrayList<>();
private Boolean allowNoForecasts;
private TimeValue timeout;
/**
* Create a new DeleteForecastRequest for the given Job ID
*
* @param jobId the jobId of the Job whose forecast(s) to delete
*/
public DeleteForecastRequest(String jobId) {
this.jobId = Objects.requireNonNull(jobId, Job.ID.getPreferredName());
}
public String getJobId() {
return jobId;
}
public List<String> getForecastIds() {
return forecastIds;
}
/**
* The forecast IDs to delete. Can be also be {@link DeleteForecastRequest#ALL} to explicitly delete ALL forecasts
*
* @param forecastIds forecast IDs to delete
*/
public void setForecastIds(String... forecastIds) {
setForecastIds(Arrays.asList(forecastIds));
}
void setForecastIds(List<String> forecastIds) {
if (forecastIds.stream().anyMatch(Objects::isNull)) {
throw new NullPointerException("forecastIds must not contain null values");
}
this.forecastIds = new ArrayList<>(forecastIds);
}
public Boolean isAllowNoForecasts() {
return allowNoForecasts;
}
/**
* Sets the `allow_no_forecasts` field.
*
* @param allowNoForecasts when {@code true} no error is thrown when {@link DeleteForecastRequest#ALL} does not find any forecasts
*/
public void setAllowNoForecasts(boolean allowNoForecasts) {
this.allowNoForecasts = allowNoForecasts;
}
/**
* Allows to set the timeout
* @param timeout timeout as a string (e.g. 1s)
*/
public void timeout(String timeout) {
this.timeout = TimeValue.parseTimeValue(timeout, this.timeout, getClass().getSimpleName() + ".timeout");
}
/**
* Allows to set the timeout
* @param timeout timeout as a {@link TimeValue}
*/
public void timeout(TimeValue timeout) {
this.timeout = timeout;
}
public TimeValue timeout() {
return timeout;
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (other == null || getClass() != other.getClass()) {
return false;
}
DeleteForecastRequest that = (DeleteForecastRequest) other;
return Objects.equals(jobId, that.jobId) &&
Objects.equals(forecastIds, that.forecastIds) &&
Objects.equals(allowNoForecasts, that.allowNoForecasts) &&
Objects.equals(timeout, that.timeout);
}
@Override
public int hashCode() {
return Objects.hash(jobId, forecastIds, allowNoForecasts, timeout);
}
@Override
public ActionRequestValidationException validate() {
return null;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(Job.ID.getPreferredName(), jobId);
if (forecastIds != null) {
builder.field(FORECAST_ID.getPreferredName(), Strings.collectionToCommaDelimitedString(forecastIds));
}
if (allowNoForecasts != null) {
builder.field(ALLOW_NO_FORECASTS.getPreferredName(), allowNoForecasts);
}
if (timeout != null) {
builder.field(TIMEOUT.getPreferredName(), timeout.getStringRep());
}
builder.endObject();
return builder;
}
}

128
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetCategoriesRequest.java Normal file
View File

@ -0,0 +1,128 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.client.ml.job.config.Job;
import org.elasticsearch.client.ml.job.util.PageParams;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Objects;
/**
* A request to retrieve categories of a given job
*/
public class GetCategoriesRequest extends ActionRequest implements ToXContentObject {
public static final ParseField CATEGORY_ID = new ParseField("category_id");
public static final ConstructingObjectParser<GetCategoriesRequest, Void> PARSER = new ConstructingObjectParser<>(
"get_categories_request", a -> new GetCategoriesRequest((String) a[0]));
static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
PARSER.declareLong(GetCategoriesRequest::setCategoryId, CATEGORY_ID);
PARSER.declareObject(GetCategoriesRequest::setPageParams, PageParams.PARSER, PageParams.PAGE);
}
private final String jobId;
private Long categoryId;
private PageParams pageParams;
/**
* Constructs a request to retrieve category information from a given job
* @param jobId id of the job from which to retrieve results
*/
public GetCategoriesRequest(String jobId) {
this.jobId = Objects.requireNonNull(jobId);
}
public String getJobId() {
return jobId;
}
public PageParams getPageParams() {
return pageParams;
}
public Long getCategoryId() {
return categoryId;
}
/**
* Sets the category id
* @param categoryId the category id
*/
public void setCategoryId(Long categoryId) {
this.categoryId = categoryId;
}
/**
* Sets the paging parameters
* @param pageParams the paging parameters
*/
public void setPageParams(PageParams pageParams) {
this.pageParams = pageParams;
}
@Override
public ActionRequestValidationException validate() {
return null;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(Job.ID.getPreferredName(), jobId);
if (categoryId != null) {
builder.field(CATEGORY_ID.getPreferredName(), categoryId);
}
if (pageParams != null) {
builder.field(PageParams.PAGE.getPreferredName(), pageParams);
}
builder.endObject();
return builder;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
GetCategoriesRequest request = (GetCategoriesRequest) obj;
return Objects.equals(jobId, request.jobId)
&& Objects.equals(categoryId, request.categoryId)
&& Objects.equals(pageParams, request.pageParams);
}
@Override
public int hashCode() {
return Objects.hash(jobId, categoryId, pageParams);
}
}

79
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetCategoriesResponse.java Normal file
View File

@ -0,0 +1,79 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.client.ml.job.results.CategoryDefinition;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
/**
* A response containing the requested categories
*/
public class GetCategoriesResponse extends AbstractResultResponse<CategoryDefinition> {
public static final ParseField CATEGORIES = new ParseField("categories");
@SuppressWarnings("unchecked")
public static final ConstructingObjectParser<GetCategoriesResponse, Void> PARSER =
new ConstructingObjectParser<>("get_categories_response", true,
a -> new GetCategoriesResponse((List<CategoryDefinition>) a[0], (long) a[1]));
static {
PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), CategoryDefinition.PARSER, CATEGORIES);
PARSER.declareLong(ConstructingObjectParser.constructorArg(), COUNT);
}
public static GetCategoriesResponse fromXContent(XContentParser parser) throws IOException {
return PARSER.parse(parser, null);
}
GetCategoriesResponse(List<CategoryDefinition> categories, long count) {
super(CATEGORIES, categories, count);
}
/**
* The retrieved categories
* @return the retrieved categories
*/
public List<CategoryDefinition> categories() {
return results;
}
@Override
public int hashCode() {
return Objects.hash(count, results);
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
GetCategoriesResponse other = (GetCategoriesResponse) obj;
return count == other.count && Objects.equals(results, other.results);
}
}

84
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDatafeedRequest.java Normal file
View File

@ -0,0 +1,84 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.client.ml.datafeed.DatafeedConfig;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Objects;
/**
* Request to create a new Machine Learning Datafeed given a {@link DatafeedConfig} configuration
*/
public class PutDatafeedRequest extends ActionRequest implements ToXContentObject {
private final DatafeedConfig datafeed;
/**
* Construct a new PutDatafeedRequest
*
* @param datafeed a {@link DatafeedConfig} configuration to create
*/
public PutDatafeedRequest(DatafeedConfig datafeed) {
this.datafeed = datafeed;
}
public DatafeedConfig getDatafeed() {
return datafeed;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return datafeed.toXContent(builder, params);
}
@Override
public boolean equals(Object object) {
if (this == object) {
return true;
}
if (object == null || getClass() != object.getClass()) {
return false;
}
PutDatafeedRequest request = (PutDatafeedRequest) object;
return Objects.equals(datafeed, request.datafeed);
}
@Override
public int hashCode() {
return Objects.hash(datafeed);
}
@Override
public final String toString() {
return Strings.toString(this);
}
@Override
public ActionRequestValidationException validate() {
return null;
}
}

70
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDatafeedResponse.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.client.ml.datafeed.DatafeedConfig;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
import java.util.Objects;
/**
* Response containing the newly created {@link DatafeedConfig}
*/
public class PutDatafeedResponse implements ToXContentObject {
private DatafeedConfig datafeed;
public static PutDatafeedResponse fromXContent(XContentParser parser) throws IOException {
return new PutDatafeedResponse(DatafeedConfig.PARSER.parse(parser, null).build());
}
PutDatafeedResponse(DatafeedConfig datafeed) {
this.datafeed = datafeed;
}
public DatafeedConfig getResponse() {
return datafeed;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
datafeed.toXContent(builder, params);
return builder;
}
@Override
public boolean equals(Object object) {
if (this == object) {
return true;
}
if (object == null || getClass() != object.getClass()) {
return false;
}
PutDatafeedResponse response = (PutDatafeedResponse) object;
return Objects.equals(datafeed, response.datafeed);
}
@Override
public int hashCode() {
return Objects.hash(datafeed);
}
}

142
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java
View File

@ -20,36 +20,37 @@ package org.elasticsearch.client.ml.datafeed;
import org.elasticsearch.client.ml.job.config.Job;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.AbstractQueryBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
* Datafeed configuration options pojo. Describes where to proactively pull input
* data from.
* <p>
* If a value has not been set it will be <code>null</code>. Object wrappers are
* used around integral types and booleans so they can take <code>null</code>
* values.
* The datafeed configuration object. It specifies which indices
* to get the data from and offers parameters for customizing different
* aspects of the process.
*/
public class DatafeedConfig implements ToXContentObject {
public static final int DEFAULT_SCROLL_SIZE = 1000;
public static final ParseField ID = new ParseField("datafeed_id");
public static final ParseField QUERY_DELAY = new ParseField("query_delay");
public static final ParseField FREQUENCY = new ParseField("frequency");
@ -59,7 +60,6 @@ public class DatafeedConfig implements ToXContentObject {
public static final ParseField QUERY = new ParseField("query");
public static final ParseField SCROLL_SIZE = new ParseField("scroll_size");
public static final ParseField AGGREGATIONS = new ParseField("aggregations");
public static final ParseField AGGS = new ParseField("aggs");
public static final ParseField SCRIPT_FIELDS = new ParseField("script_fields");
public static final ParseField CHUNKING_CONFIG = new ParseField("chunking_config");
@ -77,9 +77,8 @@ public class DatafeedConfig implements ToXContentObject {
builder.setQueryDelay(TimeValue.parseTimeValue(val, QUERY_DELAY.getPreferredName())), QUERY_DELAY);
PARSER.declareString((builder, val) ->
builder.setFrequency(TimeValue.parseTimeValue(val, FREQUENCY.getPreferredName())), FREQUENCY);
PARSER.declareObject(Builder::setQuery, (p, c) -> AbstractQueryBuilder.parseInnerQueryBuilder(p), QUERY);
PARSER.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p), AGGREGATIONS);
PARSER.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p), AGGS);
PARSER.declareField(Builder::setQuery, DatafeedConfig::parseBytes, QUERY, ObjectParser.ValueType.OBJECT);
PARSER.declareField(Builder::setAggregations, DatafeedConfig::parseBytes, AGGREGATIONS, ObjectParser.ValueType.OBJECT);
PARSER.declareObject(Builder::setScriptFields, (p, c) -> {
List<SearchSourceBuilder.ScriptField> parsedScriptFields = new ArrayList<>();
while (p.nextToken() != XContentParser.Token.END_OBJECT) {
@ -91,29 +90,26 @@ public class DatafeedConfig implements ToXContentObject {
PARSER.declareObject(Builder::setChunkingConfig, ChunkingConfig.PARSER, CHUNKING_CONFIG);
}
private static BytesReference parseBytes(XContentParser parser) throws IOException {
XContentBuilder contentBuilder = JsonXContent.contentBuilder();
contentBuilder.generator().copyCurrentStructure(parser);
return BytesReference.bytes(contentBuilder);
}
private final String id;
private final String jobId;
/**
* The delay before starting to query a period of time
*/
private final TimeValue queryDelay;
/**
* The frequency with which queries are executed
*/
private final TimeValue frequency;
private final List<String> indices;
private final List<String> types;
private final QueryBuilder query;
private final AggregatorFactories.Builder aggregations;
private final BytesReference query;
private final BytesReference aggregations;
private final List<SearchSourceBuilder.ScriptField> scriptFields;
private final Integer scrollSize;
private final ChunkingConfig chunkingConfig;
private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List<String> indices, List<String> types,
QueryBuilder query, AggregatorFactories.Builder aggregations, List<SearchSourceBuilder.ScriptField> scriptFields,
BytesReference query, BytesReference aggregations, List<SearchSourceBuilder.ScriptField> scriptFields,
Integer scrollSize, ChunkingConfig chunkingConfig) {
this.id = id;
this.jobId = jobId;
@ -156,11 +152,11 @@ public class DatafeedConfig implements ToXContentObject {
return scrollSize;
}
public QueryBuilder getQuery() {
public BytesReference getQuery() {
return query;
}
public AggregatorFactories.Builder getAggregations() {
public BytesReference getAggregations() {
return aggregations;
}
@ -183,11 +179,17 @@ public class DatafeedConfig implements ToXContentObject {
if (frequency != null) {
builder.field(FREQUENCY.getPreferredName(), frequency.getStringRep());
}
builder.field(INDICES.getPreferredName(), indices);
builder.field(TYPES.getPreferredName(), types);
builder.field(QUERY.getPreferredName(), query);
if (indices != null) {
builder.field(INDICES.getPreferredName(), indices);
}
if (types != null) {
builder.field(TYPES.getPreferredName(), types);
}
if (query != null) {
builder.field(QUERY.getPreferredName(), asMap(query));
}
if (aggregations != null) {
builder.field(AGGREGATIONS.getPreferredName(), aggregations);
builder.field(AGGREGATIONS.getPreferredName(), asMap(aggregations));
}
if (scriptFields != null) {
builder.startObject(SCRIPT_FIELDS.getPreferredName());
@ -196,7 +198,9 @@ public class DatafeedConfig implements ToXContentObject {
}
builder.endObject();
}
builder.field(SCROLL_SIZE.getPreferredName(), scrollSize);
if (scrollSize != null) {
builder.field(SCROLL_SIZE.getPreferredName(), scrollSize);
}
if (chunkingConfig != null) {
builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig);
}
@ -205,10 +209,18 @@ public class DatafeedConfig implements ToXContentObject {
return builder;
}
private static Map<String, Object> asMap(BytesReference bytesReference) {
return bytesReference == null ? null : XContentHelper.convertToMap(bytesReference, true, XContentType.JSON).v2();
}
/**
* The lists of indices and types are compared for equality but they are not
* sorted first so this test could fail simply because the indices and types
* lists are in different orders.
*
* Also note this could be a heavy operation when a query or aggregations
* are set as we need to convert the bytes references into maps to correctly
* compare them.
*/
@Override
public boolean equals(Object other) {
@ -228,31 +240,40 @@ public class DatafeedConfig implements ToXContentObject {
&& Objects.equals(this.queryDelay, that.queryDelay)
&& Objects.equals(this.indices, that.indices)
&& Objects.equals(this.types, that.types)
&& Objects.equals(this.query, that.query)
&& Objects.equals(asMap(this.query), asMap(that.query))
&& Objects.equals(this.scrollSize, that.scrollSize)
&& Objects.equals(this.aggregations, that.aggregations)
&& Objects.equals(asMap(this.aggregations), asMap(that.aggregations))
&& Objects.equals(this.scriptFields, that.scriptFields)
&& Objects.equals(this.chunkingConfig, that.chunkingConfig);
}
/**
* Note this could be a heavy operation when a query or aggregations
* are set as we need to convert the bytes references into maps to
* compute a stable hash code.
*/
@Override
public int hashCode() {
return Objects.hash(id, jobId, frequency, queryDelay, indices, types, query, scrollSize, aggregations, scriptFields,
return Objects.hash(id, jobId, frequency, queryDelay, indices, types, asMap(query), scrollSize, asMap(aggregations), scriptFields,
chunkingConfig);
}
public static Builder builder(String id, String jobId) {
return new Builder(id, jobId);
}
public static class Builder {
private String id;
private String jobId;
private TimeValue queryDelay;
private TimeValue frequency;
private List<String> indices = Collections.emptyList();
private List<String> types = Collections.emptyList();
private QueryBuilder query = QueryBuilders.matchAllQuery();
private AggregatorFactories.Builder aggregations;
private List<String> indices;
private List<String> types;
private BytesReference query;
private BytesReference aggregations;
private List<SearchSourceBuilder.ScriptField> scriptFields;
private Integer scrollSize = DEFAULT_SCROLL_SIZE;
private Integer scrollSize;
private ChunkingConfig chunkingConfig;
public Builder(String id, String jobId) {
@ -279,8 +300,12 @@ public class DatafeedConfig implements ToXContentObject {
return this;
}
public Builder setIndices(String... indices) {
return setIndices(Arrays.asList(indices));
}
public Builder setTypes(List<String> types) {
this.types = Objects.requireNonNull(types, TYPES.getPreferredName());
this.types = types;
return this;
}
@ -294,16 +319,36 @@ public class DatafeedConfig implements ToXContentObject {
return this;
}
public Builder setQuery(QueryBuilder query) {
this.query = Objects.requireNonNull(query, QUERY.getPreferredName());
private Builder setQuery(BytesReference query) {
this.query = query;
return this;
}
public Builder setAggregations(AggregatorFactories.Builder aggregations) {
public Builder setQuery(String queryAsJson) {
this.query = queryAsJson == null ? null : new BytesArray(queryAsJson);
return this;
}
public Builder setQuery(QueryBuilder query) throws IOException {
this.query = query == null ? null : xContentToBytes(query);
return this;
}
private Builder setAggregations(BytesReference aggregations) {
this.aggregations = aggregations;
return this;
}
public Builder setAggregations(String aggsAsJson) {
this.aggregations = aggsAsJson == null ? null : new BytesArray(aggsAsJson);
return this;
}
public Builder setAggregations(AggregatorFactories.Builder aggregations) throws IOException {
this.aggregations = aggregations == null ? null : xContentToBytes(aggregations);
return this;
}
public Builder setScriptFields(List<SearchSourceBuilder.ScriptField> scriptFields) {
List<SearchSourceBuilder.ScriptField> sorted = new ArrayList<>(scriptFields);
sorted.sort(Comparator.comparing(SearchSourceBuilder.ScriptField::fieldName));
@ -325,5 +370,12 @@ public class DatafeedConfig implements ToXContentObject {
return new DatafeedConfig(id, jobId, queryDelay, frequency, indices, types, query, aggregations, scriptFields, scrollSize,
chunkingConfig);
}
private static BytesReference xContentToBytes(ToXContentObject object) throws IOException {
try (XContentBuilder builder = JsonXContent.contentBuilder()) {
object.toXContent(builder, ToXContentObject.EMPTY_PARAMS);
return BytesReference.bytes(builder);
}
}
}
}

98
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java
View File

@ -20,12 +20,17 @@ package org.elasticsearch.client.ml.datafeed;
import org.elasticsearch.client.ml.job.config.Job;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.AbstractQueryBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.builder.SearchSourceBuilder;
@ -35,6 +40,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
@ -58,11 +64,9 @@ public class DatafeedUpdate implements ToXContentObject {
TimeValue.parseTimeValue(val, DatafeedConfig.QUERY_DELAY.getPreferredName())), DatafeedConfig.QUERY_DELAY);
PARSER.declareString((builder, val) -> builder.setFrequency(
TimeValue.parseTimeValue(val, DatafeedConfig.FREQUENCY.getPreferredName())), DatafeedConfig.FREQUENCY);
PARSER.declareObject(Builder::setQuery, (p, c) -> AbstractQueryBuilder.parseInnerQueryBuilder(p), DatafeedConfig.QUERY);
PARSER.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p),
DatafeedConfig.AGGREGATIONS);
PARSER.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p),
DatafeedConfig.AGGS);
PARSER.declareField(Builder::setQuery, DatafeedUpdate::parseBytes, DatafeedConfig.QUERY, ObjectParser.ValueType.OBJECT);
PARSER.declareField(Builder::setAggregations, DatafeedUpdate::parseBytes, DatafeedConfig.AGGREGATIONS,
ObjectParser.ValueType.OBJECT);
PARSER.declareObject(Builder::setScriptFields, (p, c) -> {
List<SearchSourceBuilder.ScriptField> parsedScriptFields = new ArrayList<>();
while (p.nextToken() != XContentParser.Token.END_OBJECT) {
@ -74,20 +78,26 @@ public class DatafeedUpdate implements ToXContentObject {
PARSER.declareObject(Builder::setChunkingConfig, ChunkingConfig.PARSER, DatafeedConfig.CHUNKING_CONFIG);
}
private static BytesReference parseBytes(XContentParser parser) throws IOException {
XContentBuilder contentBuilder = JsonXContent.contentBuilder();
contentBuilder.generator().copyCurrentStructure(parser);
return BytesReference.bytes(contentBuilder);
}
private final String id;
private final String jobId;
private final TimeValue queryDelay;
private final TimeValue frequency;
private final List<String> indices;
private final List<String> types;
private final QueryBuilder query;
private final AggregatorFactories.Builder aggregations;
private final BytesReference query;
private final BytesReference aggregations;
private final List<SearchSourceBuilder.ScriptField> scriptFields;
private final Integer scrollSize;
private final ChunkingConfig chunkingConfig;
private DatafeedUpdate(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List<String> indices, List<String> types,
QueryBuilder query, AggregatorFactories.Builder aggregations, List<SearchSourceBuilder.ScriptField> scriptFields,
BytesReference query, BytesReference aggregations, List<SearchSourceBuilder.ScriptField> scriptFields,
Integer scrollSize, ChunkingConfig chunkingConfig) {
this.id = id;
this.jobId = jobId;
@ -121,9 +131,13 @@ public class DatafeedUpdate implements ToXContentObject {
builder.field(DatafeedConfig.FREQUENCY.getPreferredName(), frequency.getStringRep());
}
addOptionalField(builder, DatafeedConfig.INDICES, indices);
if (query != null) {
builder.field(DatafeedConfig.QUERY.getPreferredName(), asMap(query));
}
if (aggregations != null) {
builder.field(DatafeedConfig.AGGREGATIONS.getPreferredName(), asMap(aggregations));
}
addOptionalField(builder, DatafeedConfig.TYPES, types);
addOptionalField(builder, DatafeedConfig.QUERY, query);
addOptionalField(builder, DatafeedConfig.AGGREGATIONS, aggregations);
if (scriptFields != null) {
builder.startObject(DatafeedConfig.SCRIPT_FIELDS.getPreferredName());
for (SearchSourceBuilder.ScriptField scriptField : scriptFields) {
@ -167,11 +181,11 @@ public class DatafeedUpdate implements ToXContentObject {
return scrollSize;
}
public QueryBuilder getQuery() {
public BytesReference getQuery() {
return query;
}
public AggregatorFactories.Builder getAggregations() {
public BytesReference getAggregations() {
return aggregations;
}
@ -183,10 +197,18 @@ public class DatafeedUpdate implements ToXContentObject {
return chunkingConfig;
}
private static Map<String, Object> asMap(BytesReference bytesReference) {
return bytesReference == null ? null : XContentHelper.convertToMap(bytesReference, true, XContentType.JSON).v2();
}
/**
* The lists of indices and types are compared for equality but they are not
* sorted first so this test could fail simply because the indices and types
* lists are in different orders.
*
* Also note this could be a heavy operation when a query or aggregations
* are set as we need to convert the bytes references into maps to correctly
* compare them.
*/
@Override
public boolean equals(Object other) {
@ -206,19 +228,28 @@ public class DatafeedUpdate implements ToXContentObject {
&& Objects.equals(this.queryDelay, that.queryDelay)
&& Objects.equals(this.indices, that.indices)
&& Objects.equals(this.types, that.types)
&& Objects.equals(this.query, that.query)
&& Objects.equals(asMap(this.query), asMap(that.query))
&& Objects.equals(this.scrollSize, that.scrollSize)
&& Objects.equals(this.aggregations, that.aggregations)
&& Objects.equals(asMap(this.aggregations), asMap(that.aggregations))
&& Objects.equals(this.scriptFields, that.scriptFields)
&& Objects.equals(this.chunkingConfig, that.chunkingConfig);
}
/**
* Note this could be a heavy operation when a query or aggregations
* are set as we need to convert the bytes references into maps to
* compute a stable hash code.
*/
@Override
public int hashCode() {
return Objects.hash(id, jobId, frequency, queryDelay, indices, types, query, scrollSize, aggregations, scriptFields,
return Objects.hash(id, jobId, frequency, queryDelay, indices, types, asMap(query), scrollSize, asMap(aggregations), scriptFields,
chunkingConfig);
}
public static Builder builder(String id) {
return new Builder(id);
}
public static class Builder {
private String id;
@ -227,8 +258,8 @@ public class DatafeedUpdate implements ToXContentObject {
private TimeValue frequency;
private List<String> indices;
private List<String> types;
private QueryBuilder query;
private AggregatorFactories.Builder aggregations;
private BytesReference query;
private BytesReference aggregations;
private List<SearchSourceBuilder.ScriptField> scriptFields;
private Integer scrollSize;
private ChunkingConfig chunkingConfig;
@ -276,16 +307,36 @@ public class DatafeedUpdate implements ToXContentObject {
return this;
}
public Builder setQuery(QueryBuilder query) {
private Builder setQuery(BytesReference query) {
this.query = query;
return this;
}
public Builder setAggregations(AggregatorFactories.Builder aggregations) {
public Builder setQuery(String queryAsJson) {
this.query = queryAsJson == null ? null : new BytesArray(queryAsJson);
return this;
}
public Builder setQuery(QueryBuilder query) throws IOException {
this.query = query == null ? null : xContentToBytes(query);
return this;
}
private Builder setAggregations(BytesReference aggregations) {
this.aggregations = aggregations;
return this;
}
public Builder setAggregations(String aggsAsJson) {
this.aggregations = aggsAsJson == null ? null : new BytesArray(aggsAsJson);
return this;
}
public Builder setAggregations(AggregatorFactories.Builder aggregations) throws IOException {
this.aggregations = aggregations == null ? null : xContentToBytes(aggregations);
return this;
}
public Builder setScriptFields(List<SearchSourceBuilder.ScriptField> scriptFields) {
List<SearchSourceBuilder.ScriptField> sorted = new ArrayList<>(scriptFields);
sorted.sort(Comparator.comparing(SearchSourceBuilder.ScriptField::fieldName));
@ -307,5 +358,12 @@ public class DatafeedUpdate implements ToXContentObject {
return new DatafeedUpdate(id, jobId, queryDelay, frequency, indices, types, query, aggregations, scriptFields, scrollSize,
chunkingConfig);
}
private static BytesReference xContentToBytes(ToXContentObject object) throws IOException {
try (XContentBuilder builder = JsonXContent.contentBuilder()) {
object.toXContent(builder, ToXContentObject.EMPTY_PARAMS);
return BytesReference.bytes(builder);
}
}
}
}

62
client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
View File

@ -24,10 +24,12 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.DeleteForecastRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
import org.elasticsearch.client.ml.FlushJobRequest;
import org.elasticsearch.client.ml.ForecastJobRequest;
import org.elasticsearch.client.ml.GetBucketsRequest;
import org.elasticsearch.client.ml.GetCategoriesRequest;
import org.elasticsearch.client.ml.GetInfluencersRequest;
import org.elasticsearch.client.ml.GetJobRequest;
import org.elasticsearch.client.ml.GetJobStatsRequest;
@ -35,14 +37,18 @@ import org.elasticsearch.client.ml.GetOverallBucketsRequest;
import org.elasticsearch.client.ml.GetRecordsRequest;
import org.elasticsearch.client.ml.OpenJobRequest;
import org.elasticsearch.client.ml.PostDataRequest;
import org.elasticsearch.client.ml.PutDatafeedRequest;
import org.elasticsearch.client.ml.PutJobRequest;
import org.elasticsearch.client.ml.UpdateJobRequest;
import org.elasticsearch.client.ml.datafeed.DatafeedConfig;
import org.elasticsearch.client.ml.datafeed.DatafeedConfigTests;
import org.elasticsearch.client.ml.job.config.AnalysisConfig;
import org.elasticsearch.client.ml.job.config.Detector;
import org.elasticsearch.client.ml.job.config.Job;
import org.elasticsearch.client.ml.job.config.JobUpdate;
import org.elasticsearch.client.ml.job.config.JobUpdateTests;
import org.elasticsearch.client.ml.job.util.PageParams;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
@ -203,6 +209,47 @@ public class MLRequestConvertersTests extends ESTestCase {
}
}
public void testPutDatafeed() throws IOException {
DatafeedConfig datafeed = DatafeedConfigTests.createRandom();
PutDatafeedRequest putDatafeedRequest = new PutDatafeedRequest(datafeed);
Request request = MLRequestConverters.putDatafeed(putDatafeedRequest);
assertEquals(HttpPut.METHOD_NAME, request.getMethod());
assertThat(request.getEndpoint(), equalTo("/_xpack/ml/datafeeds/" + datafeed.getId()));
try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
DatafeedConfig parsedDatafeed = DatafeedConfig.PARSER.apply(parser, null).build();
assertThat(parsedDatafeed, equalTo(datafeed));
}
}
public void testDeleteForecast() throws Exception {
String jobId = randomAlphaOfLength(10);
DeleteForecastRequest deleteForecastRequest = new DeleteForecastRequest(jobId);
Request request = MLRequestConverters.deleteForecast(deleteForecastRequest);
assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
assertEquals("/_xpack/ml/anomaly_detectors/" + jobId + "/_forecast", request.getEndpoint());
assertFalse(request.getParameters().containsKey("timeout"));
assertFalse(request.getParameters().containsKey("allow_no_forecasts"));
deleteForecastRequest.setForecastIds(randomAlphaOfLength(10), randomAlphaOfLength(10));
deleteForecastRequest.timeout("10s");
deleteForecastRequest.setAllowNoForecasts(true);
request = MLRequestConverters.deleteForecast(deleteForecastRequest);
assertEquals(
"/_xpack/ml/anomaly_detectors/" +
jobId +
"/_forecast/" +
Strings.collectionToCommaDelimitedString(deleteForecastRequest.getForecastIds()),
request.getEndpoint());
assertEquals("10s",
request.getParameters().get(DeleteForecastRequest.TIMEOUT.getPreferredName()));
assertEquals(Boolean.toString(true),
request.getParameters().get(DeleteForecastRequest.ALLOW_NO_FORECASTS.getPreferredName()));
}
public void testGetBuckets() throws IOException {
String jobId = randomAlphaOfLength(10);
GetBucketsRequest getBucketsRequest = new GetBucketsRequest(jobId);
@ -220,6 +267,21 @@ public class MLRequestConvertersTests extends ESTestCase {
}
}
public void testGetCategories() throws IOException {
String jobId = randomAlphaOfLength(10);
GetCategoriesRequest getCategoriesRequest = new GetCategoriesRequest(jobId);
getCategoriesRequest.setPageParams(new PageParams(100, 300));
Request request = MLRequestConverters.getCategories(getCategoriesRequest);
assertEquals(HttpGet.METHOD_NAME, request.getMethod());
assertEquals("/_xpack/ml/anomaly_detectors/" + jobId + "/results/categories", request.getEndpoint());
try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
GetCategoriesRequest parsedRequest = GetCategoriesRequest.PARSER.apply(parser, null);
assertThat(parsedRequest, equalTo(getCategoriesRequest));
}
}
public void testGetOverallBuckets() throws IOException {
String jobId = randomAlphaOfLength(10);
GetOverallBucketsRequest getOverallBucketsRequest = new GetOverallBucketsRequest(jobId);

141
client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java
View File

@ -23,6 +23,8 @@ import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.client.ml.GetBucketsRequest;
import org.elasticsearch.client.ml.GetBucketsResponse;
import org.elasticsearch.client.ml.GetCategoriesRequest;
import org.elasticsearch.client.ml.GetCategoriesResponse;
import org.elasticsearch.client.ml.GetInfluencersRequest;
import org.elasticsearch.client.ml.GetInfluencersResponse;
import org.elasticsearch.client.ml.GetOverallBucketsRequest;
@ -126,11 +128,150 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
bulkRequest.add(indexRequest);
}
private void addCategoryIndexRequest(long categoryId, String categoryName, BulkRequest bulkRequest) {
IndexRequest indexRequest = new IndexRequest(RESULTS_INDEX, DOC);
indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"category_id\": " + categoryId + ", \"terms\": \"" +
categoryName + "\", \"regex\": \".*?" + categoryName + ".*\", \"max_matching_length\": 3, \"examples\": [\"" +
categoryName + "\"]}", XContentType.JSON);
bulkRequest.add(indexRequest);
}
private void addCategoriesIndexRequests(BulkRequest bulkRequest) {
List<String> categories = Arrays.asList("AAL", "JZA", "JBU");
for (int i = 0; i < categories.size(); i++) {
addCategoryIndexRequest(i+1, categories.get(i), bulkRequest);
}
}
@After
public void deleteJob() throws IOException {
new MlRestTestStateCleaner(logger, client()).clearMlMetadata();
}
public void testGetCategories() throws IOException {
// index some category results
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
addCategoriesIndexRequests(bulkRequest);
highLevelClient().bulk(bulkRequest, RequestOptions.DEFAULT);
MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
{
GetCategoriesRequest request = new GetCategoriesRequest(JOB_ID);
request.setPageParams(new PageParams(0, 10000));
GetCategoriesResponse response = execute(request, machineLearningClient::getCategories,
machineLearningClient::getCategoriesAsync);
assertThat(response.count(), equalTo(3L));
assertThat(response.categories().size(), equalTo(3));
assertThat(response.categories().get(0).getCategoryId(), equalTo(1L));
assertThat(response.categories().get(0).getGrokPattern(), equalTo(".*?AAL.*"));
assertThat(response.categories().get(0).getRegex(), equalTo(".*?AAL.*"));
assertThat(response.categories().get(0).getTerms(), equalTo("AAL"));
assertThat(response.categories().get(1).getCategoryId(), equalTo(2L));
assertThat(response.categories().get(1).getGrokPattern(), equalTo(".*?JZA.*"));
assertThat(response.categories().get(1).getRegex(), equalTo(".*?JZA.*"));
assertThat(response.categories().get(1).getTerms(), equalTo("JZA"));
assertThat(response.categories().get(2).getCategoryId(), equalTo(3L));
assertThat(response.categories().get(2).getGrokPattern(), equalTo(".*?JBU.*"));
assertThat(response.categories().get(2).getRegex(), equalTo(".*?JBU.*"));
assertThat(response.categories().get(2).getTerms(), equalTo("JBU"));
}
{
GetCategoriesRequest request = new GetCategoriesRequest(JOB_ID);
request.setPageParams(new PageParams(0, 1));
GetCategoriesResponse response = execute(request, machineLearningClient::getCategories,
machineLearningClient::getCategoriesAsync);
assertThat(response.count(), equalTo(3L));
assertThat(response.categories().size(), equalTo(1));
assertThat(response.categories().get(0).getCategoryId(), equalTo(1L));
assertThat(response.categories().get(0).getGrokPattern(), equalTo(".*?AAL.*"));
assertThat(response.categories().get(0).getRegex(), equalTo(".*?AAL.*"));
assertThat(response.categories().get(0).getTerms(), equalTo("AAL"));
}
{
GetCategoriesRequest request = new GetCategoriesRequest(JOB_ID);
request.setPageParams(new PageParams(1, 2));
GetCategoriesResponse response = execute(request, machineLearningClient::getCategories,
machineLearningClient::getCategoriesAsync);
assertThat(response.count(), equalTo(3L));
assertThat(response.categories().size(), equalTo(2));
assertThat(response.categories().get(0).getCategoryId(), equalTo(2L));
assertThat(response.categories().get(0).getGrokPattern(), equalTo(".*?JZA.*"));
assertThat(response.categories().get(0).getRegex(), equalTo(".*?JZA.*"));
assertThat(response.categories().get(0).getTerms(), equalTo("JZA"));
assertThat(response.categories().get(1).getCategoryId(), equalTo(3L));
assertThat(response.categories().get(1).getGrokPattern(), equalTo(".*?JBU.*"));
assertThat(response.categories().get(1).getRegex(), equalTo(".*?JBU.*"));
assertThat(response.categories().get(1).getTerms(), equalTo("JBU"));
}
{
GetCategoriesRequest request = new GetCategoriesRequest(JOB_ID);
request.setCategoryId(0L); // request a non-existent category
GetCategoriesResponse response = execute(request, machineLearningClient::getCategories,
machineLearningClient::getCategoriesAsync);
assertThat(response.count(), equalTo(0L));
assertThat(response.categories().size(), equalTo(0));
}
{
GetCategoriesRequest request = new GetCategoriesRequest(JOB_ID);
request.setCategoryId(1L);
GetCategoriesResponse response = execute(request, machineLearningClient::getCategories,
machineLearningClient::getCategoriesAsync);
assertThat(response.count(), equalTo(1L));
assertThat(response.categories().size(), equalTo(1));
assertThat(response.categories().get(0).getCategoryId(), equalTo(1L));
assertThat(response.categories().get(0).getGrokPattern(), equalTo(".*?AAL.*"));
assertThat(response.categories().get(0).getRegex(), equalTo(".*?AAL.*"));
assertThat(response.categories().get(0).getTerms(), equalTo("AAL"));
}
{
GetCategoriesRequest request = new GetCategoriesRequest(JOB_ID);
request.setCategoryId(2L);
GetCategoriesResponse response = execute(request, machineLearningClient::getCategories,
machineLearningClient::getCategoriesAsync);
assertThat(response.count(), equalTo(1L));
assertThat(response.categories().get(0).getCategoryId(), equalTo(2L));
assertThat(response.categories().get(0).getGrokPattern(), equalTo(".*?JZA.*"));
assertThat(response.categories().get(0).getRegex(), equalTo(".*?JZA.*"));
assertThat(response.categories().get(0).getTerms(), equalTo("JZA"));
}
{
GetCategoriesRequest request = new GetCategoriesRequest(JOB_ID);
request.setCategoryId(3L);
GetCategoriesResponse response = execute(request, machineLearningClient::getCategories,
machineLearningClient::getCategoriesAsync);
assertThat(response.count(), equalTo(1L));
assertThat(response.categories().get(0).getCategoryId(), equalTo(3L));
assertThat(response.categories().get(0).getGrokPattern(), equalTo(".*?JBU.*"));
assertThat(response.categories().get(0).getRegex(), equalTo(".*?JBU.*"));
assertThat(response.categories().get(0).getTerms(), equalTo("JBU"));
}
}
public void testGetBuckets() throws IOException {
MachineLearningClient machineLearningClient = highLevelClient().machineLearning();

119
client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
View File

@ -20,33 +20,40 @@ package org.elasticsearch.client;
import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.client.ml.ForecastJobRequest;
import org.elasticsearch.client.ml.ForecastJobResponse;
import org.elasticsearch.client.ml.PostDataRequest;
import org.elasticsearch.client.ml.PostDataResponse;
import org.elasticsearch.client.ml.UpdateJobRequest;
import org.elasticsearch.client.ml.job.config.JobUpdate;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.client.ml.GetJobStatsRequest;
import org.elasticsearch.client.ml.GetJobStatsResponse;
import org.elasticsearch.client.ml.job.config.JobState;
import org.elasticsearch.client.ml.job.stats.JobStats;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.CloseJobResponse;
import org.elasticsearch.client.ml.DeleteForecastRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
import org.elasticsearch.client.ml.DeleteJobResponse;
import org.elasticsearch.client.ml.FlushJobRequest;
import org.elasticsearch.client.ml.FlushJobResponse;
import org.elasticsearch.client.ml.ForecastJobRequest;
import org.elasticsearch.client.ml.ForecastJobResponse;
import org.elasticsearch.client.ml.GetJobRequest;
import org.elasticsearch.client.ml.GetJobResponse;
import org.elasticsearch.client.ml.GetJobStatsRequest;
import org.elasticsearch.client.ml.GetJobStatsResponse;
import org.elasticsearch.client.ml.OpenJobRequest;
import org.elasticsearch.client.ml.OpenJobResponse;
import org.elasticsearch.client.ml.PostDataRequest;
import org.elasticsearch.client.ml.PostDataResponse;
import org.elasticsearch.client.ml.PutDatafeedRequest;
import org.elasticsearch.client.ml.PutDatafeedResponse;
import org.elasticsearch.client.ml.PutJobRequest;
import org.elasticsearch.client.ml.PutJobResponse;
import org.elasticsearch.client.ml.UpdateJobRequest;
import org.elasticsearch.client.ml.datafeed.DatafeedConfig;
import org.elasticsearch.client.ml.job.config.AnalysisConfig;
import org.elasticsearch.client.ml.job.config.DataDescription;
import org.elasticsearch.client.ml.job.config.Detector;
import org.elasticsearch.client.ml.job.config.Job;
import org.elasticsearch.client.ml.FlushJobRequest;
import org.elasticsearch.client.ml.FlushJobResponse;
import org.elasticsearch.client.ml.job.config.JobState;
import org.elasticsearch.client.ml.job.config.JobUpdate;
import org.elasticsearch.client.ml.job.stats.JobStats;
import org.elasticsearch.common.unit.TimeValue;
import org.junit.After;
import java.io.IOException;
@ -288,6 +295,92 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
assertEquals("Updated description", getResponse.jobs().get(0).getDescription());
}
public void testPutDatafeed() throws Exception {
String jobId = randomValidJobId();
Job job = buildJob(jobId);
MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
execute(new PutJobRequest(job), machineLearningClient::putJob, machineLearningClient::putJobAsync);
String datafeedId = "datafeed-" + jobId;
DatafeedConfig datafeedConfig = DatafeedConfig.builder(datafeedId, jobId).setIndices("some_data_index").build();
PutDatafeedResponse response = execute(new PutDatafeedRequest(datafeedConfig), machineLearningClient::putDatafeed,
machineLearningClient::putDatafeedAsync);
DatafeedConfig createdDatafeed = response.getResponse();
assertThat(createdDatafeed.getId(), equalTo(datafeedId));
assertThat(createdDatafeed.getIndices(), equalTo(datafeedConfig.getIndices()));
}
public void testDeleteForecast() throws Exception {
String jobId = "test-delete-forecast";
Job job = buildJob(jobId);
MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
machineLearningClient.putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
machineLearningClient.openJob(new OpenJobRequest(jobId), RequestOptions.DEFAULT);
Job noForecastsJob = buildJob("test-delete-forecast-none");
machineLearningClient.putJob(new PutJobRequest(noForecastsJob), RequestOptions.DEFAULT);
PostDataRequest.JsonBuilder builder = new PostDataRequest.JsonBuilder();
for(int i = 0; i < 30; i++) {
Map<String, Object> hashMap = new HashMap<>();
hashMap.put("total", randomInt(1000));
hashMap.put("timestamp", (i+1)*1000);
builder.addDoc(hashMap);
}
PostDataRequest postDataRequest = new PostDataRequest(jobId, builder);
machineLearningClient.postData(postDataRequest, RequestOptions.DEFAULT);
machineLearningClient.flushJob(new FlushJobRequest(jobId), RequestOptions.DEFAULT);
ForecastJobResponse forecastJobResponse1 = machineLearningClient.forecastJob(new ForecastJobRequest(jobId), RequestOptions.DEFAULT);
ForecastJobResponse forecastJobResponse2 = machineLearningClient.forecastJob(new ForecastJobRequest(jobId), RequestOptions.DEFAULT);
waitForForecastToComplete(jobId, forecastJobResponse1.getForecastId());
waitForForecastToComplete(jobId, forecastJobResponse2.getForecastId());
{
DeleteForecastRequest request = new DeleteForecastRequest(jobId);
request.setForecastIds(forecastJobResponse1.getForecastId(), forecastJobResponse2.getForecastId());
AcknowledgedResponse response = execute(request, machineLearningClient::deleteForecast,
machineLearningClient::deleteForecastAsync);
assertTrue(response.isAcknowledged());
assertFalse(forecastExists(jobId, forecastJobResponse1.getForecastId()));
assertFalse(forecastExists(jobId, forecastJobResponse2.getForecastId()));
}
{
DeleteForecastRequest request = DeleteForecastRequest.deleteAllForecasts(noForecastsJob.getId());
request.setAllowNoForecasts(true);
AcknowledgedResponse response = execute(request, machineLearningClient::deleteForecast,
machineLearningClient::deleteForecastAsync);
assertTrue(response.isAcknowledged());
}
{
DeleteForecastRequest request = DeleteForecastRequest.deleteAllForecasts(noForecastsJob.getId());
request.setAllowNoForecasts(false);
ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class,
() -> execute(request, machineLearningClient::deleteForecast, machineLearningClient::deleteForecastAsync));
assertThat(exception.status().getStatus(), equalTo(404));
}
}
private void waitForForecastToComplete(String jobId, String forecastId) throws Exception {
GetRequest request = new GetRequest(".ml-anomalies-" + jobId);
request.id(jobId + "_model_forecast_request_stats_" + forecastId);
assertBusy(() -> {
GetResponse getResponse = highLevelClient().get(request, RequestOptions.DEFAULT);
assertTrue(getResponse.isExists());
assertTrue(getResponse.getSourceAsString().contains("finished"));
}, 30, TimeUnit.SECONDS);
}
private boolean forecastExists(String jobId, String forecastId) throws Exception {
GetRequest getRequest = new GetRequest(".ml-anomalies-" + jobId);
getRequest.id(jobId + "_model_forecast_request_stats_" + forecastId);
GetResponse getResponse = highLevelClient().get(getRequest, RequestOptions.DEFAULT);
return getResponse.isExists();
}
public static String randomValidJobId() {
CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz0123456789".toCharArray());
return generator.ofCodePointsLength(random(), 10, 10);

266
client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
View File

@ -21,8 +21,11 @@ package org.elasticsearch.client.documentation;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.LatchedActionListener;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ESRestHighLevelClientTestCase;
import org.elasticsearch.client.MachineLearningGetResultsIT;
import org.elasticsearch.client.MachineLearningIT;
@ -31,6 +34,7 @@ import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.CloseJobResponse;
import org.elasticsearch.client.ml.DeleteForecastRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
import org.elasticsearch.client.ml.DeleteJobResponse;
import org.elasticsearch.client.ml.FlushJobRequest;
@ -39,6 +43,8 @@ import org.elasticsearch.client.ml.ForecastJobRequest;
import org.elasticsearch.client.ml.ForecastJobResponse;
import org.elasticsearch.client.ml.GetBucketsRequest;
import org.elasticsearch.client.ml.GetBucketsResponse;
import org.elasticsearch.client.ml.GetCategoriesRequest;
import org.elasticsearch.client.ml.GetCategoriesResponse;
import org.elasticsearch.client.ml.GetInfluencersRequest;
import org.elasticsearch.client.ml.GetInfluencersResponse;
import org.elasticsearch.client.ml.GetJobRequest;
@ -53,28 +59,36 @@ import org.elasticsearch.client.ml.OpenJobRequest;
import org.elasticsearch.client.ml.OpenJobResponse;
import org.elasticsearch.client.ml.PostDataRequest;
import org.elasticsearch.client.ml.PostDataResponse;
import org.elasticsearch.client.ml.PutDatafeedRequest;
import org.elasticsearch.client.ml.PutDatafeedResponse;
import org.elasticsearch.client.ml.PutJobRequest;
import org.elasticsearch.client.ml.PutJobResponse;
import org.elasticsearch.client.ml.UpdateJobRequest;
import org.elasticsearch.client.ml.datafeed.ChunkingConfig;
import org.elasticsearch.client.ml.datafeed.DatafeedConfig;
import org.elasticsearch.client.ml.job.config.AnalysisConfig;
import org.elasticsearch.client.ml.job.config.AnalysisLimits;
import org.elasticsearch.client.ml.job.config.DataDescription;
import org.elasticsearch.client.ml.job.config.DetectionRule;
import org.elasticsearch.client.ml.job.config.Detector;
import org.elasticsearch.client.ml.job.config.Job;
import org.elasticsearch.client.ml.job.process.DataCounts;
import org.elasticsearch.client.ml.job.config.JobUpdate;
import org.elasticsearch.client.ml.job.config.ModelPlotConfig;
import org.elasticsearch.client.ml.job.config.Operator;
import org.elasticsearch.client.ml.job.config.RuleCondition;
import org.elasticsearch.client.ml.job.process.DataCounts;
import org.elasticsearch.client.ml.job.results.AnomalyRecord;
import org.elasticsearch.client.ml.job.results.Bucket;
import org.elasticsearch.client.ml.job.results.CategoryDefinition;
import org.elasticsearch.client.ml.job.results.Influencer;
import org.elasticsearch.client.ml.job.results.OverallBucket;
import org.elasticsearch.client.ml.job.stats.JobStats;
import org.elasticsearch.client.ml.job.util.PageParams;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.After;
import java.io.IOException;
@ -90,6 +104,7 @@ import java.util.stream.Collectors;
import static org.hamcrest.Matchers.closeTo;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.core.Is.is;
@ -182,8 +197,6 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
public void testGetJob() throws Exception {
RestHighLevelClient client = highLevelClient();
String jobId = "get-machine-learning-job1";
Job job = MachineLearningIT.buildJob("get-machine-learning-job1");
client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
@ -474,6 +487,106 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
}
}
public void testPutDatafeed() throws Exception {
RestHighLevelClient client = highLevelClient();
{
// We need to create a job for the datafeed request to be valid
String jobId = "put-datafeed-job-1";
Job job = MachineLearningIT.buildJob(jobId);
client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
String id = "datafeed-1";
//tag::x-pack-ml-create-datafeed-config
DatafeedConfig.Builder datafeedBuilder = new DatafeedConfig.Builder(id, jobId) // <1>
.setIndices("index_1", "index_2"); // <2>
//end::x-pack-ml-create-datafeed-config
AggregatorFactories.Builder aggs = AggregatorFactories.builder();
//tag::x-pack-ml-create-datafeed-config-set-aggregations
datafeedBuilder.setAggregations(aggs); // <1>
//end::x-pack-ml-create-datafeed-config-set-aggregations
// Clearing aggregation to avoid complex validation rules
datafeedBuilder.setAggregations((String) null);
//tag::x-pack-ml-create-datafeed-config-set-chunking-config
datafeedBuilder.setChunkingConfig(ChunkingConfig.newAuto()); // <1>
//end::x-pack-ml-create-datafeed-config-set-chunking-config
//tag::x-pack-ml-create-datafeed-config-set-frequency
datafeedBuilder.setFrequency(TimeValue.timeValueSeconds(30)); // <1>
//end::x-pack-ml-create-datafeed-config-set-frequency
//tag::x-pack-ml-create-datafeed-config-set-query
datafeedBuilder.setQuery(QueryBuilders.matchAllQuery()); // <1>
//end::x-pack-ml-create-datafeed-config-set-query
//tag::x-pack-ml-create-datafeed-config-set-query-delay
datafeedBuilder.setQueryDelay(TimeValue.timeValueMinutes(1)); // <1>
//end::x-pack-ml-create-datafeed-config-set-query-delay
List<SearchSourceBuilder.ScriptField> scriptFields = Collections.emptyList();
//tag::x-pack-ml-create-datafeed-config-set-script-fields
datafeedBuilder.setScriptFields(scriptFields); // <1>
//end::x-pack-ml-create-datafeed-config-set-script-fields
//tag::x-pack-ml-create-datafeed-config-set-scroll-size
datafeedBuilder.setScrollSize(1000); // <1>
//end::x-pack-ml-create-datafeed-config-set-scroll-size
//tag::x-pack-ml-put-datafeed-request
PutDatafeedRequest request = new PutDatafeedRequest(datafeedBuilder.build()); // <1>
//end::x-pack-ml-put-datafeed-request
//tag::x-pack-ml-put-datafeed-execute
PutDatafeedResponse response = client.machineLearning().putDatafeed(request, RequestOptions.DEFAULT);
//end::x-pack-ml-put-datafeed-execute
//tag::x-pack-ml-put-datafeed-response
DatafeedConfig datafeed = response.getResponse(); // <1>
//end::x-pack-ml-put-datafeed-response
assertThat(datafeed.getId(), equalTo("datafeed-1"));
}
{
// We need to create a job for the datafeed request to be valid
String jobId = "put-datafeed-job-2";
Job job = MachineLearningIT.buildJob(jobId);
client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
String id = "datafeed-2";
DatafeedConfig datafeed = new DatafeedConfig.Builder(id, jobId).setIndices("index_1", "index_2").build();
PutDatafeedRequest request = new PutDatafeedRequest(datafeed);
// tag::x-pack-ml-put-datafeed-execute-listener
ActionListener<PutDatafeedResponse> listener = new ActionListener<PutDatafeedResponse>() {
@Override
public void onResponse(PutDatafeedResponse response) {
// <1>
}
@Override
public void onFailure(Exception e) {
// <2>
}
};
// end::x-pack-ml-put-datafeed-execute-listener
// Replace the empty listener by a blocking listener in test
final CountDownLatch latch = new CountDownLatch(1);
listener = new LatchedActionListener<>(listener, latch);
// tag::x-pack-ml-put-datafeed-execute-async
client.machineLearning().putDatafeedAsync(request, RequestOptions.DEFAULT, listener); // <1>
// end::x-pack-ml-put-datafeed-execute-async
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}
public void testGetBuckets() throws IOException, InterruptedException {
RestHighLevelClient client = highLevelClient();
@ -637,6 +750,83 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
}
}
public void testDeleteForecast() throws Exception {
RestHighLevelClient client = highLevelClient();
Job job = MachineLearningIT.buildJob("deleting-forecast-for-job");
client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
client.machineLearning().openJob(new OpenJobRequest(job.getId()), RequestOptions.DEFAULT);
PostDataRequest.JsonBuilder builder = new PostDataRequest.JsonBuilder();
for(int i = 0; i < 30; i++) {
Map<String, Object> hashMap = new HashMap<>();
hashMap.put("total", randomInt(1000));
hashMap.put("timestamp", (i+1)*1000);
builder.addDoc(hashMap);
}
PostDataRequest postDataRequest = new PostDataRequest(job.getId(), builder);
client.machineLearning().postData(postDataRequest, RequestOptions.DEFAULT);
client.machineLearning().flushJob(new FlushJobRequest(job.getId()), RequestOptions.DEFAULT);
ForecastJobResponse forecastJobResponse = client.machineLearning().
forecastJob(new ForecastJobRequest(job.getId()), RequestOptions.DEFAULT);
String forecastId = forecastJobResponse.getForecastId();
GetRequest request = new GetRequest(".ml-anomalies-" + job.getId());
request.id(job.getId() + "_model_forecast_request_stats_" + forecastId);
assertBusy(() -> {
GetResponse getResponse = highLevelClient().get(request, RequestOptions.DEFAULT);
assertTrue(getResponse.isExists());
assertTrue(getResponse.getSourceAsString().contains("finished"));
}, 30, TimeUnit.SECONDS);
{
//tag::x-pack-ml-delete-forecast-request
DeleteForecastRequest deleteForecastRequest = new DeleteForecastRequest("deleting-forecast-for-job"); //<1>
//end::x-pack-ml-delete-forecast-request
//tag::x-pack-ml-delete-forecast-request-options
deleteForecastRequest.setForecastIds(forecastId); //<1>
deleteForecastRequest.timeout("30s"); //<2>
deleteForecastRequest.setAllowNoForecasts(true); //<3>
//end::x-pack-ml-delete-forecast-request-options
//tag::x-pack-ml-delete-forecast-execute
AcknowledgedResponse deleteForecastResponse = client.machineLearning().deleteForecast(deleteForecastRequest,
RequestOptions.DEFAULT);
//end::x-pack-ml-delete-forecast-execute
//tag::x-pack-ml-delete-forecast-response
boolean isAcknowledged = deleteForecastResponse.isAcknowledged(); //<1>
//end::x-pack-ml-delete-forecast-response
}
{
//tag::x-pack-ml-delete-forecast-listener
ActionListener<AcknowledgedResponse> listener = new ActionListener<AcknowledgedResponse>() {
@Override
public void onResponse(AcknowledgedResponse DeleteForecastResponse) {
//<1>
}
@Override
public void onFailure(Exception e) {
// <2>
}
};
//end::x-pack-ml-delete-forecast-listener
DeleteForecastRequest deleteForecastRequest = DeleteForecastRequest.deleteAllForecasts(job.getId());
deleteForecastRequest.setAllowNoForecasts(true);
// Replace the empty listener by a blocking listener in test
final CountDownLatch latch = new CountDownLatch(1);
listener = new LatchedActionListener<>(listener, latch);
// tag::x-pack-ml-delete-forecast-execute-async
client.machineLearning().deleteForecastAsync(deleteForecastRequest, RequestOptions.DEFAULT, listener); //<1>
// end::x-pack-ml-delete-forecast-execute-async
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}
public void testGetJobStats() throws Exception {
RestHighLevelClient client = highLevelClient();
@ -1111,4 +1301,74 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}
public void testGetCategories() throws IOException, InterruptedException {
RestHighLevelClient client = highLevelClient();
String jobId = "test-get-categories";
Job job = MachineLearningIT.buildJob(jobId);
client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
// Let us index a category
IndexRequest indexRequest = new IndexRequest(".ml-anomalies-shared", "doc");
indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
indexRequest.source("{\"job_id\": \"test-get-categories\", \"category_id\": 1, \"terms\": \"AAL\"," +
" \"regex\": \".*?AAL.*\", \"max_matching_length\": 3, \"examples\": [\"AAL\"]}", XContentType.JSON);
client.index(indexRequest, RequestOptions.DEFAULT);
{
// tag::x-pack-ml-get-categories-request
GetCategoriesRequest request = new GetCategoriesRequest(jobId); // <1>
// end::x-pack-ml-get-categories-request
// tag::x-pack-ml-get-categories-category-id
request.setCategoryId(1L); // <1>
// end::x-pack-ml-get-categories-category-id
// tag::x-pack-ml-get-categories-page
request.setPageParams(new PageParams(100, 200)); // <1>
// end::x-pack-ml-get-categories-page
// Set page params back to null so the response contains the category we indexed
request.setPageParams(null);
// tag::x-pack-ml-get-categories-execute
GetCategoriesResponse response = client.machineLearning().getCategories(request, RequestOptions.DEFAULT);
// end::x-pack-ml-get-categories-execute
// tag::x-pack-ml-get-categories-response
long count = response.count(); // <1>
List<CategoryDefinition> categories = response.categories(); // <2>
// end::x-pack-ml-get-categories-response
assertEquals(1, categories.size());
}
{
GetCategoriesRequest request = new GetCategoriesRequest(jobId);
// tag::x-pack-ml-get-categories-listener
ActionListener<GetCategoriesResponse> listener =
new ActionListener<GetCategoriesResponse>() {
@Override
public void onResponse(GetCategoriesResponse getcategoriesResponse) {
// <1>
}
@Override
public void onFailure(Exception e) {
// <2>
}
};
// end::x-pack-ml-get-categories-listener
// Replace the empty listener by a blocking listener in test
final CountDownLatch latch = new CountDownLatch(1);
listener = new LatchedActionListener<>(listener, latch);
// tag::x-pack-ml-get-categories-execute-async
client.machineLearning().getCategoriesAsync(request, RequestOptions.DEFAULT, listener); // <1>
// end::x-pack-ml-get-categories-execute-async
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}
}

62
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteForecastRequestTests.java Normal file
View File

@ -0,0 +1,62 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.client.ml.job.config.JobTests;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class DeleteForecastRequestTests extends AbstractXContentTestCase<DeleteForecastRequest> {
@Override
protected DeleteForecastRequest createTestInstance() {
DeleteForecastRequest deleteForecastRequest = new DeleteForecastRequest(JobTests.randomValidJobId());
if (randomBoolean()) {
int length = randomInt(10);
List<String> ids = new ArrayList<>(length);
for(int i = 0; i < length; i++) {
ids.add(randomAlphaOfLength(10));
}
deleteForecastRequest.setForecastIds(ids);
}
if (randomBoolean()) {
deleteForecastRequest.setAllowNoForecasts(randomBoolean());
}
if (randomBoolean()) {
deleteForecastRequest.timeout(randomTimeValue());
}
return deleteForecastRequest;
}
@Override
protected DeleteForecastRequest doParseInstance(XContentParser parser) throws IOException {
return DeleteForecastRequest.PARSER.apply(parser, null);
}
@Override
protected boolean supportsUnknownFields() {
return false;
}
}

51
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetCategoriesRequestTests.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.client.ml.job.util.PageParams;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
public class GetCategoriesRequestTests extends AbstractXContentTestCase<GetCategoriesRequest> {
@Override
protected GetCategoriesRequest createTestInstance() {
GetCategoriesRequest request = new GetCategoriesRequest(randomAlphaOfLengthBetween(1, 20));
if (randomBoolean()) {
request.setCategoryId(randomNonNegativeLong());
} else {
int from = randomInt(10000);
int size = randomInt(10000);
request.setPageParams(new PageParams(from, size));
}
return request;
}
@Override
protected GetCategoriesRequest doParseInstance(XContentParser parser) throws IOException {
return GetCategoriesRequest.PARSER.apply(parser, null);
}
@Override
protected boolean supportsUnknownFields() {
return false;
}
}

53
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetCategoriesResponseTests.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.client.ml.job.results.CategoryDefinition;
import org.elasticsearch.client.ml.job.results.CategoryDefinitionTests;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class GetCategoriesResponseTests extends AbstractXContentTestCase<GetCategoriesResponse> {
@Override
protected GetCategoriesResponse createTestInstance() {
String jobId = randomAlphaOfLength(20);
int listSize = randomInt(10);
List<CategoryDefinition> categories = new ArrayList<>(listSize);
for (int j = 0; j < listSize; j++) {
CategoryDefinition category = CategoryDefinitionTests.createTestInstance(jobId);
categories.add(category);
}
return new GetCategoriesResponse(categories, listSize);
}
@Override
protected GetCategoriesResponse doParseInstance(XContentParser parser) throws IOException {
return GetCategoriesResponse.fromXContent(parser);
}
@Override
protected boolean supportsUnknownFields() {
return true;
}
}

43
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutDatafeedRequestTests.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.client.ml.datafeed.DatafeedConfig;
import org.elasticsearch.client.ml.datafeed.DatafeedConfigTests;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase;
public class PutDatafeedRequestTests extends AbstractXContentTestCase<PutDatafeedRequest> {
@Override
protected PutDatafeedRequest createTestInstance() {
return new PutDatafeedRequest(DatafeedConfigTests.createRandom());
}
@Override
protected PutDatafeedRequest doParseInstance(XContentParser parser) {
return new PutDatafeedRequest(DatafeedConfig.PARSER.apply(parser, null).build());
}
@Override
protected boolean supportsUnknownFields() {
return false;
}
}

49
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutDatafeedResponseTests.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.client.ml.datafeed.DatafeedConfigTests;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.util.function.Predicate;
public class PutDatafeedResponseTests extends AbstractXContentTestCase<PutDatafeedResponse> {
@Override
protected PutDatafeedResponse createTestInstance() {
return new PutDatafeedResponse(DatafeedConfigTests.createRandom());
}
@Override
protected PutDatafeedResponse doParseInstance(XContentParser parser) throws IOException {
return PutDatafeedResponse.fromXContent(parser);
}
@Override
protected boolean supportsUnknownFields() {
return true;
}
@Override
protected Predicate<String> getRandomFieldsExcludeFilter() {
return field -> !field.isEmpty();
}
}

40
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedConfigTests.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.client.ml.datafeed;
import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@ -27,7 +26,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
@ -36,19 +34,26 @@ import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class DatafeedConfigTests extends AbstractXContentTestCase<DatafeedConfig> {
@Override
protected DatafeedConfig createTestInstance() {
return createRandom();
}
public static DatafeedConfig createRandom() {
long bucketSpanMillis = 3600000;
DatafeedConfig.Builder builder = constructBuilder();
builder.setIndices(randomStringList(1, 10));
builder.setTypes(randomStringList(0, 10));
if (randomBoolean()) {
builder.setQuery(QueryBuilders.termQuery(randomAlphaOfLength(10), randomAlphaOfLength(10)));
try {
builder.setQuery(QueryBuilders.termQuery(randomAlphaOfLength(10), randomAlphaOfLength(10)));
} catch (IOException e) {
throw new RuntimeException("Failed to serialize query", e);
}
}
boolean addScriptFields = randomBoolean();
if (addScriptFields) {
@ -72,7 +77,11 @@ public class DatafeedConfigTests extends AbstractXContentTestCase<DatafeedConfig
MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
aggs.addAggregator(AggregationBuilders.dateHistogram("buckets")
.interval(aggHistogramInterval).subAggregation(maxTime).field("time"));
builder.setAggregations(aggs);
try {
builder.setAggregations(aggs);
} catch (IOException e) {
throw new RuntimeException("failed to serialize aggs", e);
}
}
if (randomBoolean()) {
builder.setScrollSize(randomIntBetween(0, Integer.MAX_VALUE));
@ -93,12 +102,6 @@ public class DatafeedConfigTests extends AbstractXContentTestCase<DatafeedConfig
return builder.build();
}
@Override
protected NamedXContentRegistry xContentRegistry() {
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
return new NamedXContentRegistry(searchModule.getNamedXContents());
}
public static List<String> randomStringList(int min, int max) {
int size = scaledRandomIntBetween(min, max);
List<String> list = new ArrayList<>();
@ -150,21 +153,6 @@ public class DatafeedConfigTests extends AbstractXContentTestCase<DatafeedConfig
expectThrows(NullPointerException.class, () -> new DatafeedConfig.Builder(randomValidDatafeedId(), null));
}
public void testCheckValid_GivenNullIndices() {
DatafeedConfig.Builder conf = constructBuilder();
expectThrows(NullPointerException.class, () -> conf.setIndices(null));
}
public void testCheckValid_GivenNullType() {
DatafeedConfig.Builder conf = constructBuilder();
expectThrows(NullPointerException.class, () -> conf.setTypes(null));
}
public void testCheckValid_GivenNullQuery() {
DatafeedConfig.Builder conf = constructBuilder();
expectThrows(NullPointerException.class, () -> conf.setQuery(null));
}
public static String randomValidDatafeedId() {
CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray());
return generator.ofCodePointsLength(random(), 10, 10);

24
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdateTests.java
View File

@ -18,19 +18,16 @@
*/
package org.elasticsearch.client.ml.datafeed;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class DatafeedUpdateTests extends AbstractXContentTestCase<DatafeedUpdate> {
@ -54,7 +51,11 @@ public class DatafeedUpdateTests extends AbstractXContentTestCase<DatafeedUpdate
builder.setTypes(DatafeedConfigTests.randomStringList(1, 10));
}
if (randomBoolean()) {
builder.setQuery(QueryBuilders.termQuery(randomAlphaOfLength(10), randomAlphaOfLength(10)));
try {
builder.setQuery(QueryBuilders.termQuery(randomAlphaOfLength(10), randomAlphaOfLength(10)));
} catch (IOException e) {
throw new RuntimeException("Failed to serialize query", e);
}
}
if (randomBoolean()) {
int scriptsSize = randomInt(3);
@ -71,7 +72,11 @@ public class DatafeedUpdateTests extends AbstractXContentTestCase<DatafeedUpdate
// Testing with a single agg is ok as we don't have special list xcontent logic
AggregatorFactories.Builder aggs = new AggregatorFactories.Builder();
aggs.addAggregator(AggregationBuilders.avg(randomAlphaOfLength(10)).field(randomAlphaOfLength(10)));
builder.setAggregations(aggs);
try {
builder.setAggregations(aggs);
} catch (IOException e) {
throw new RuntimeException("Failed to serialize aggs", e);
}
}
if (randomBoolean()) {
builder.setScrollSize(randomIntBetween(0, Integer.MAX_VALUE));
@ -91,11 +96,4 @@ public class DatafeedUpdateTests extends AbstractXContentTestCase<DatafeedUpdate
protected boolean supportsUnknownFields() {
return false;
}
@Override
protected NamedXContentRegistry xContentRegistry() {
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
return new NamedXContentRegistry(searchModule.getNamedXContents());
}
}

2
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/results/CategoryDefinitionTests.java
View File

@ -25,7 +25,7 @@ import java.util.Arrays;
public class CategoryDefinitionTests extends AbstractXContentTestCase<CategoryDefinition> {
public CategoryDefinition createTestInstance(String jobId) {
public static CategoryDefinition createTestInstance(String jobId) {
CategoryDefinition categoryDefinition = new CategoryDefinition(jobId);
categoryDefinition.setCategoryId(randomLong());
categoryDefinition.setTerms(randomAlphaOfLength(10));

2
distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeNameInLogsIT.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.unconfigurednodename;
package org.elasticsearch.test.rest;
import org.elasticsearch.common.logging.NodeNameInLogsIntegTestCase;

2
docs/build.gradle
View File

@ -57,6 +57,8 @@ integTestCluster {
// TODO: remove this for 7.0, this exists to allow the doc examples in 6.x to continue using the defaults
systemProperty 'es.scripting.use_java_time', 'false'
systemProperty 'es.scripting.update.ctx_in_params', 'false'
//TODO: remove this once the cname is prepended to the address by default in 7.0
systemProperty 'es.http.cname_in_publish_address', 'true'
}
// remove when https://github.com/elastic/elasticsearch/issues/31305 is fixed

78
docs/java-rest/high-level/ml/delete-forecast.asciidoc Normal file
View File

@ -0,0 +1,78 @@
[[java-rest-high-x-pack-ml-delete-forecast]]
=== Delete Forecast API
The Delete Forecast API provides the ability to delete a {ml} job's
forecast in the cluster.
It accepts a `DeleteForecastRequest` object and responds
with an `AcknowledgedResponse` object.
[[java-rest-high-x-pack-ml-delete-forecast-request]]
==== Delete Forecast Request
A `DeleteForecastRequest` object gets created with an existing non-null `jobId`.
All other fields are optional for the request.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-forecast-request]
--------------------------------------------------
<1> Constructing a new request referencing an existing `jobId`
==== Optional Arguments
The following arguments are optional.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-forecast-request-options]
--------------------------------------------------
<1> Sets the specific forecastIds to delete, can be set to `_all` to indicate ALL forecasts for the given
`jobId`
<2> Set the timeout for the request to respond, default is 30 seconds
<3> Set the `allow_no_forecasts` option. When `true` no error will be returned if an `_all`
request finds no forecasts. It defaults to `true`
[[java-rest-high-x-pack-ml-delete-forecast-execution]]
==== Execution
The request can be executed through the `MachineLearningClient` contained
in the `RestHighLevelClient` object, accessed via the `machineLearningClient()` method.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-forecast-execute]
--------------------------------------------------
[[java-rest-high-x-pack-ml-delete-forecast-execution-async]]
==== Asynchronous Execution
The request can also be executed asynchronously:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-forecast-execute-async]
--------------------------------------------------
<1> The `DeleteForecastRequest` to execute and the `ActionListener` to use when
the execution completes
The method does not block and returns immediately. The passed `ActionListener` is used
to notify the caller of completion. A typical `ActionListener` for `AcknowledgedResponse` may
look like
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-forecast-listener]
--------------------------------------------------
<1> `onResponse` is called back when the action is completed successfully
<2> `onFailure` is called back when some unexpected error occurs
[[java-rest-high-x-pack-ml-delete-forecast-response]]
==== Delete Forecast Response
An `AcknowledgedResponse` contains an acknowledgement of the forecast(s) deletion
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-forecast-response]
--------------------------------------------------
<1> `isAcknowledged()` indicates if the forecast was successfully deleted or not.

83
docs/java-rest/high-level/ml/get-categories.asciidoc Normal file
View File

@ -0,0 +1,83 @@
[[java-rest-high-x-pack-ml-get-categories]]
=== Get Categories API
The Get Categories API retrieves one or more category results.
It accepts a `GetCategoriesRequest` object and responds
with a `GetCategoriesResponse` object.
[[java-rest-high-x-pack-ml-get-categories-request]]
==== Get Categories Request
A `GetCategoriesRequest` object gets created with an existing non-null `jobId`.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-categories-request]
--------------------------------------------------
<1> Constructing a new request referencing an existing `jobId`
==== Optional Arguments
The following arguments are optional:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-categories-category-id]
--------------------------------------------------
<1> The id of the category to get. Otherwise it will return all categories.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-categories-page]
--------------------------------------------------
<1> The page parameters `from` and `size`. `from` specifies the number of categories to skip.
`size` specifies the maximum number of categories to get. Defaults to `0` and `100` respectively.
[[java-rest-high-x-pack-ml-get-categories-execution]]
==== Execution
The request can be executed through the `MachineLearningClient` contained
in the `RestHighLevelClient` object, accessed via the `machineLearningClient()` method.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-categories-execute]
--------------------------------------------------
[[java-rest-high-x-pack-ml-get-categories-execution-async]]
==== Asynchronous Execution
The request can also be executed asynchronously:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-categories-execute-async]
--------------------------------------------------
<1> The `GetCategoriesRequest` to execute and the `ActionListener` to use when
the execution completes
The asynchronous method does not block and returns immediately. Once it is
completed the `ActionListener` is called back with the `onResponse` method
if the execution is successful or the `onFailure` method if the execution
failed.
A typical listener for `GetCategoriesResponse` looks like:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-categories-listener]
--------------------------------------------------
<1> `onResponse` is called back when the action is completed successfully
<2> `onFailure` is called back when some unexpected error occurs
[[java-rest-high-snapshot-ml-get-categories-response]]
==== Get Categories Response
The returned `GetCategoriesResponse` contains the requested categories:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-categories-response]
--------------------------------------------------
<1> The count of categories that were matched
<2> The categories retrieved

124
docs/java-rest/high-level/ml/put-datafeed.asciidoc Normal file
View File

@ -0,0 +1,124 @@
[[java-rest-high-x-pack-ml-put-datafeed]]
=== Put Datafeed API
The Put Datafeed API can be used to create a new {ml} datafeed
in the cluster. The API accepts a `PutDatafeedRequest` object
as a request and returns a `PutDatafeedResponse`.
[[java-rest-high-x-pack-ml-put-datafeed-request]]
==== Put Datafeed Request
A `PutDatafeedRequest` requires the following argument:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-datafeed-request]
--------------------------------------------------
<1> The configuration of the {ml} datafeed to create
[[java-rest-high-x-pack-ml-put-datafeed-config]]
==== Datafeed Configuration
The `DatafeedConfig` object contains all the details about the {ml} datafeed
configuration.
A `DatafeedConfig` requires the following arguments:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-create-datafeed-config]
--------------------------------------------------
<1> The datafeed ID and the job ID
<2> The indices that contain the data to retrieve and feed into the job
==== Optional Arguments
The following arguments are optional:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-create-datafeed-config-set-chunking-config]
--------------------------------------------------
<1> Specifies how data searches are split into time chunks.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-create-datafeed-config-set-frequency]
--------------------------------------------------
<1> The interval at which scheduled queries are made while the datafeed runs in real time.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-create-datafeed-config-set-query]
--------------------------------------------------
<1> A query to filter the search results by. Defaults to the `match_all` query.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-create-datafeed-config-set-query-delay]
--------------------------------------------------
<1> The time interval behind real time that data is queried.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-create-datafeed-config-set-script-fields]
--------------------------------------------------
<1> Allows the use of script fields.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-create-datafeed-config-set-scroll-size]
--------------------------------------------------
<1> The `size` parameter used in the searches.
[[java-rest-high-x-pack-ml-put-datafeed-execution]]
==== Execution
The Put Datafeed API can be executed through a `MachineLearningClient`
instance. Such an instance can be retrieved from a `RestHighLevelClient`
using the `machineLearning()` method:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-datafeed-execute]
--------------------------------------------------
[[java-rest-high-x-pack-ml-put-datafeed-response]]
==== Response
The returned `PutDatafeedResponse` returns the full representation of
the new {ml} datafeed if it has been successfully created. This will
contain the creation time and other fields initialized using
default values:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-datafeed-response]
--------------------------------------------------
<1> The created datafeed
[[java-rest-high-x-pack-ml-put-datafeed-async]]
==== Asynchronous Execution
This request can be executed asynchronously:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-datafeed-execute-async]
--------------------------------------------------
<1> The `PutDatafeedRequest` to execute and the `ActionListener` to use when
the execution completes
The asynchronous method does not block and returns immediately. Once it is
completed the `ActionListener` is called back using the `onResponse` method
if the execution successfully completed or using the `onFailure` method if
it failed.
A typical listener for `PutDatafeedResponse` looks like:
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-datafeed-execute-listener]
--------------------------------------------------
<1> Called when the execution is successfully completed. The response is
provided as an argument
<2> Called in case of failure. The raised exception is provided as an argument

2
docs/java-rest/high-level/ml/put-job.asciidoc
View File

@ -142,7 +142,7 @@ This request can be executed asynchronously:
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-job-execute-async]
--------------------------------------------------
<1> The `PutMlJobRequest` to execute and the `ActionListener` to use when
<1> The `PutJobRequest` to execute and the `ActionListener` to use when
the execution completes
The asynchronous method does not block and returns immediately. Once it is

6
docs/java-rest/high-level/supported-apis.asciidoc
View File

@ -220,12 +220,15 @@ The Java High Level REST Client supports the following Machine Learning APIs:
* <<java-rest-high-x-pack-ml-flush-job>>
* <<java-rest-high-x-pack-ml-update-job>>
* <<java-rest-high-x-pack-ml-get-job-stats>>
* <<java-rest-high-x-pack-ml-put-datafeed>>
* <<java-rest-high-x-pack-ml-forecast-job>>
* <<java-rest-high-x-pack-ml-delete-forecast>>
* <<java-rest-high-x-pack-ml-get-buckets>>
* <<java-rest-high-x-pack-ml-get-overall-buckets>>
* <<java-rest-high-x-pack-ml-get-records>>
* <<java-rest-high-x-pack-ml-post-data>>
* <<java-rest-high-x-pack-ml-get-influencers>>
* <<java-rest-high-x-pack-ml-get-categories>>
include::ml/put-job.asciidoc[]
include::ml/get-job.asciidoc[]
@ -234,13 +237,16 @@ include::ml/open-job.asciidoc[]
include::ml/close-job.asciidoc[]
include::ml/update-job.asciidoc[]
include::ml/flush-job.asciidoc[]
include::ml/put-datafeed.asciidoc[]
include::ml/get-job-stats.asciidoc[]
include::ml/forecast-job.asciidoc[]
include::ml/delete-forecast.asciidoc[]
include::ml/get-buckets.asciidoc[]
include::ml/get-overall-buckets.asciidoc[]
include::ml/get-records.asciidoc[]
include::ml/post-data.asciidoc[]
include::ml/get-influencers.asciidoc[]
include::ml/get-categories.asciidoc[]
== Migration APIs

2
docs/reference/aggregations/bucket/composite-aggregation.asciidoc
View File

@ -348,7 +348,7 @@ GET /_search
\... will sort the composite bucket in descending order when comparing values from the `date_histogram` source
and in ascending order when comparing values from the `terms` source.
====== Missing bucket
==== Missing bucket
By default documents without a value for a given source are ignored.
It is possible to include them in the response by setting `missing_bucket` to

2
docs/reference/analysis/tokenfilters.asciidoc
View File

@ -37,6 +37,8 @@ include::tokenfilters/multiplexer-tokenfilter.asciidoc[]
include::tokenfilters/condition-tokenfilter.asciidoc[]
include::tokenfilters/predicate-tokenfilter.asciidoc[]
include::tokenfilters/stemmer-tokenfilter.asciidoc[]
include::tokenfilters/stemmer-override-tokenfilter.asciidoc[]

79
docs/reference/analysis/tokenfilters/predicate-tokenfilter.asciidoc Normal file
View File

@ -0,0 +1,79 @@
[[analysis-predicatefilter-tokenfilter]]
=== Predicate Token Filter Script
The predicate_token_filter token filter takes a predicate script, and removes tokens that do
not match the predicate.
[float]
=== Options
[horizontal]
script:: a predicate script that determines whether or not the current token will
be emitted. Note that only inline scripts are supported.
[float]
=== Settings example
You can set it up like:
[source,js]
--------------------------------------------------
PUT /condition_example
{
"settings" : {
"analysis" : {
"analyzer" : {
"my_analyzer" : {
"tokenizer" : "standard",
"filter" : [ "my_script_filter" ]
}
},
"filter" : {
"my_script_filter" : {
"type" : "predicate_token_filter",
"script" : {
"source" : "token.getTerm().length() > 5" <1>
}
}
}
}
}
}
--------------------------------------------------
// CONSOLE
<1> This will emit tokens that are more than 5 characters long
And test it like:
[source,js]
--------------------------------------------------
POST /condition_example/_analyze
{
"analyzer" : "my_analyzer",
"text" : "What Flapdoodle"
}
--------------------------------------------------
// CONSOLE
// TEST[continued]
And it'd respond:
[source,js]
--------------------------------------------------
{
"tokens": [
{
"token": "Flapdoodle", <1>
"start_offset": 5,
"end_offset": 15,
"type": "<ALPHANUM>",
"position": 1 <2>
}
]
}
--------------------------------------------------
// TESTRESPONSE
<1> The token 'What' has been removed from the tokenstream because it does not
match the predicate.
<2> The position and offset values are unaffected by the removal of earlier tokens

13
docs/reference/migration/migrate_7_0/settings.asciidoc
View File

@ -40,3 +40,16 @@ will be removed in the future, thus requiring HTTP to always be enabled.
This setting has been removed, as disabling http pipelining support on the server
provided little value. The setting `http.pipelining.max_events` can still be used to
limit the number of pipelined requests in-flight.
==== Cross-cluster search settings renamed
The cross-cluster search remote cluster connection infrastructure is also used
in cross-cluster replication. This means that the setting names
`search.remote.*` used for configuring cross-cluster search belie the fact that
they also apply to other situations where a connection to a remote cluster as
used. Therefore, these settings have been renamed from `search.remote.*` to
`cluster.remote.*`. For backwards compatibility purposes, we will fallback to
`search.remote.*` if `cluster.remote.*` is not set. For any such settings stored
in the cluster state, or set on dynamic settings updates, we will automatically
upgrade the setting from `search.remote.*` to `cluster.remote.*`. The fallback
settings will be removed in 8.0.0.

45
docs/reference/modules/snapshots.asciidoc
View File

@ -207,6 +207,51 @@ repositories.url.allowed_urls: ["http://www.example.org/root/*", "https://*.mydo
URL repositories with `file:` URLs can only point to locations registered in the `path.repo` setting similar to
shared file system repository.
[float]
[role="xpack"]
[testenv="basic"]
===== Source Only Repository
A source repository enables you to create minimal, source-only snapshots that take up to 50% less space on disk.
Source only snapshots contain stored fields and index metadata. They do not include index or doc values structures
and are not searchable when restored. After restoring a source-only snapshot, you must <<docs-reindex,reindex>>
the data into a new index.
Source repositories delegate to another snapshot repository for storage.
[IMPORTANT]
==================================================
Source only snapshots are only supported if the `_source` field is enabled and no source-filtering is applied.
When you restore a source only snapshot:
* The restored index is read-only and can only serve `match_all` search or scroll requests to enable reindexing.
* Queries other than `match_all` and `_get` requests are not supported.
* The mapping of the restored index is empty, but the original mapping is available from the types top
level `meta` element.
==================================================
When you create a source repository, you must specify the type and name of the delegate repository
where the snapshots will be stored:
[source,js]
-----------------------------------
PUT _snapshot/my_src_only_repository
{
"type": "source",
"settings": {
"delegate_type": "fs",
"location": "my_backup_location"
}
}
-----------------------------------
// CONSOLE
// TEST[continued]
[float]
===== Repository plugins

2
docs/reference/query-dsl/match-query.asciidoc
View File

@ -172,7 +172,7 @@ GET /_search
The example above creates a boolean query:
`(ny OR (new AND york)) city)`
`(ny OR (new AND york)) city`
that matches documents with the term `ny` or the conjunction `new AND york`.
By default the parameter `auto_generate_synonyms_phrase_query` is set to `true`.

23
docs/reference/search/suggesters/context-suggest.asciidoc
View File

@ -13,6 +13,9 @@ Every context mapping has a unique name and a type. There are two types: `catego
and `geo`. Context mappings are configured under the `contexts` parameter in
the field mapping.
NOTE: It is mandatory to provide a context when indexing and querying
a context enabled completion field.
The following defines types, each with two context mappings for a completion
field:
@ -84,10 +87,6 @@ PUT place_path_category
NOTE: Adding context mappings increases the index size for completion field. The completion index
is entirely heap resident, you can monitor the completion field index size using <<indices-stats>>.
NOTE: deprecated[7.0.0, Indexing a suggestion without context on a context enabled completion field is deprecated
and will be removed in the next major release. If you want to index a suggestion that matches all contexts you should
add a special context for it.]
[[suggester-context-category]]
[float]
==== Category Context
@ -160,9 +159,9 @@ POST place/_search?pretty
// CONSOLE
// TEST[continued]
Note: deprecated[7.0.0, When no categories are provided at query-time, all indexed documents are considered.
Querying with no categories on a category enabled completion field is deprecated and will be removed in the next major release
as it degrades search performance considerably.]
NOTE: If multiple categories or category contexts are set on the query
they are merged as a disjunction. This means that suggestions match
if they contain at least one of the provided context values.
Suggestions with certain categories can be boosted higher than others.
The following filters suggestions by categories and additionally boosts
@ -218,6 +217,9 @@ multiple category context clauses. The following parameters are supported for a
so on, by specifying a category prefix of 'type'.
Defaults to `false`
NOTE: If a suggestion entry matches multiple contexts the final score is computed as the
maximum score produced by any matching contexts.
[[suggester-context-geo]]
[float]
==== Geo location Context
@ -307,6 +309,10 @@ POST place/_search
NOTE: When a location with a lower precision at query time is specified, all suggestions
that fall within the area will be considered.
NOTE: If multiple categories or category contexts are set on the query
they are merged as a disjunction. This means that suggestions match
if they contain at least one of the provided context values.
Suggestions that are within an area represented by a geohash can also be boosted higher
than others, as shown by the following:
@ -349,6 +355,9 @@ POST place/_search?pretty
that fall under the geohash representation of '(43.6624803, -79.3863353)'
with a default precision of '6' by a factor of `2`
NOTE: If a suggestion entry matches multiple contexts the final score is computed as the
maximum score produced by any matching contexts.
In addition to accepting context values, a context query can be composed of
multiple context clauses. The following parameters are supported for a
`category` context clause:

5
docs/reference/setup/install/windows.asciidoc
View File

@ -295,8 +295,9 @@ as _properties_ within Windows Installer documentation) that can be passed to `m
`SKIPSETTINGPASSWORDS`::
When installing with a `Trial` license and X-Pack Security enabled, whether the
installation should skip setting up the built-in users `elastic`, `kibana` and `logstash_system`.
When installing with a `Trial` license and {security} enabled, whether the
installation should skip setting up the built-in users `elastic`, `kibana`,
`logstash_system`, `apm_system`, and `beats_system`.
Defaults to `false`
`ELASTICUSERPASSWORD`::

9
libs/core/src/main/java/org/elasticsearch/core/internal/io/IOUtils.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.core.internal.io;
import java.io.Closeable;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
@ -36,6 +37,14 @@ import java.util.Map;
*/
public final class IOUtils {
/**
* UTF-8 charset string.
* <p>Where possible, use {@link StandardCharsets#UTF_8} instead,
* as using the String constant may slow things down.
* @see StandardCharsets#UTF_8
*/
public static final String UTF_8 = StandardCharsets.UTF_8.name();
private IOUtils() {
// Static utils methods
}

56
modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java
View File

@ -19,6 +19,13 @@
package org.elasticsearch.analysis.common;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeSource;
import org.elasticsearch.script.ScriptContext;
/**
@ -30,21 +37,40 @@ public abstract class AnalysisPredicateScript {
* Encapsulation of the state of the current token
*/
public static class Token {
public CharSequence term;
public int pos;
public int posInc;
public int posLen;
public int startOffset;
public int endOffset;
public String type;
public boolean isKeyword;
private final CharTermAttribute termAtt;
private final PositionIncrementAttribute posIncAtt;
private final PositionLengthAttribute posLenAtt;
private final OffsetAttribute offsetAtt;
private final TypeAttribute typeAtt;
private final KeywordAttribute keywordAtt;
// posInc is always 1 at the beginning of a tokenstream and the convention
// from the _analyze endpoint is that tokenstream positions are 0-based
private int pos = -1;
/**
* Create a token exposing values from an AttributeSource
*/
public Token(AttributeSource source) {
this.termAtt = source.addAttribute(CharTermAttribute.class);
this.posIncAtt = source.addAttribute(PositionIncrementAttribute.class);
this.posLenAtt = source.addAttribute(PositionLengthAttribute.class);
this.offsetAtt = source.addAttribute(OffsetAttribute.class);
this.typeAtt = source.addAttribute(TypeAttribute.class);
this.keywordAtt = source.addAttribute(KeywordAttribute.class);
}
public void updatePosition() {
this.pos = this.pos + posIncAtt.getPositionIncrement();
}
public CharSequence getTerm() {
return term;
return termAtt;
}
public int getPositionIncrement() {
return posInc;
return posIncAtt.getPositionIncrement();
}
public int getPosition() {
@ -52,23 +78,23 @@ public abstract class AnalysisPredicateScript {
}
public int getPositionLength() {
return posLen;
return posLenAtt.getPositionLength();
}
public int getStartOffset() {
return startOffset;
return offsetAtt.startOffset();
}
public int getEndOffset() {
return endOffset;
return offsetAtt.endOffset();
}
public String getType() {
return type;
return typeAtt.type();
}
public boolean isKeyword() {
return isKeyword;
return keywordAtt.isKeyword();
}
}

2
modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
View File

@ -264,6 +264,8 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, Scri
filters.put("pattern_replace", requiresAnalysisSettings(PatternReplaceTokenFilterFactory::new));
filters.put("persian_normalization", PersianNormalizationFilterFactory::new);
filters.put("porter_stem", PorterStemTokenFilterFactory::new);
filters.put("predicate_token_filter",
requiresAnalysisSettings((i, e, n, s) -> new PredicateTokenFilterScriptFactory(i, n, s, scriptService.get())));
filters.put("remove_duplicates", RemoveDuplicatesTokenFilterFactory::new);
filters.put("reverse", ReverseTokenFilterFactory::new);
filters.put("russian_stem", RussianStemTokenFilterFactory::new);

73
modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PredicateTokenFilterScriptFactory.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.analysis.common;
import org.apache.lucene.analysis.FilteringTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;
import java.io.IOException;
/**
* A factory for creating FilteringTokenFilters that determine whether or not to
* accept their underlying token by consulting a script
*/
public class PredicateTokenFilterScriptFactory extends AbstractTokenFilterFactory {
private final AnalysisPredicateScript.Factory factory;
public PredicateTokenFilterScriptFactory(IndexSettings indexSettings, String name, Settings settings, ScriptService scriptService) {
super(indexSettings, name, settings);
Settings scriptSettings = settings.getAsSettings("script");
Script script = Script.parse(scriptSettings);
if (script.getType() != ScriptType.INLINE) {
throw new IllegalArgumentException("Cannot use stored scripts in tokenfilter [" + name + "]");
}
this.factory = scriptService.compile(script, AnalysisPredicateScript.CONTEXT);
}
@Override
public TokenStream create(TokenStream tokenStream) {
return new ScriptFilteringTokenFilter(tokenStream, factory.newInstance());
}
private static class ScriptFilteringTokenFilter extends FilteringTokenFilter {
final AnalysisPredicateScript script;
final AnalysisPredicateScript.Token token;
ScriptFilteringTokenFilter(TokenStream in, AnalysisPredicateScript script) {
super(in);
this.script = script;
this.token = new AnalysisPredicateScript.Token(this);
}
@Override
protected boolean accept() throws IOException {
token.updatePosition();
return script.execute(token);
}
}
}

47
modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java
View File

@ -21,12 +21,6 @@ package org.elasticsearch.analysis.common;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
@ -36,6 +30,7 @@ import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -76,30 +71,26 @@ public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFact
}
return in;
};
AnalysisPredicateScript script = factory.newInstance();
final AnalysisPredicateScript.Token token = new AnalysisPredicateScript.Token();
return new ConditionalTokenFilter(tokenStream, filter) {
return new ScriptedConditionTokenFilter(tokenStream, filter, factory.newInstance());
}
CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
PositionLengthAttribute posLenAtt = addAttribute(PositionLengthAttribute.class);
OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
private static class ScriptedConditionTokenFilter extends ConditionalTokenFilter {
@Override
protected boolean shouldFilter() {
token.term = termAtt;
token.posInc = posIncAtt.getPositionIncrement();
token.pos += token.posInc;
token.posLen = posLenAtt.getPositionLength();
token.startOffset = offsetAtt.startOffset();
token.endOffset = offsetAtt.endOffset();
token.type = typeAtt.type();
token.isKeyword = keywordAtt.isKeyword();
return script.execute(token);
}
};
private final AnalysisPredicateScript script;
private final AnalysisPredicateScript.Token token;
ScriptedConditionTokenFilter(TokenStream input, Function<TokenStream, TokenStream> inputFactory,
AnalysisPredicateScript script) {
super(input, inputFactory);
this.script = script;
this.token = new AnalysisPredicateScript.Token(this);
}
@Override
protected boolean shouldFilter() throws IOException {
token.updatePosition();
return script.execute(token);
}
}
@Override

89
modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.analysis.common;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.test.ESTokenStreamTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import java.io.IOException;
import java.util.Collections;
public class PredicateTokenScriptFilterTests extends ESTokenStreamTestCase {
public void testSimpleFilter() throws IOException {
Settings settings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build();
Settings indexSettings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put("index.analysis.filter.f.type", "predicate_token_filter")
.put("index.analysis.filter.f.script.source", "token.getTerm().length() > 5")
.put("index.analysis.analyzer.myAnalyzer.type", "custom")
.put("index.analysis.analyzer.myAnalyzer.tokenizer", "standard")
.putList("index.analysis.analyzer.myAnalyzer.filter", "f")
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
AnalysisPredicateScript.Factory factory = () -> new AnalysisPredicateScript() {
@Override
public boolean execute(Token token) {
return token.getTerm().length() > 5;
}
};
@SuppressWarnings("unchecked")
ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()){
@Override
public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryType> context) {
assertEquals(context, AnalysisPredicateScript.CONTEXT);
assertEquals(new Script("token.getTerm().length() > 5"), script);
return (FactoryType) factory;
}
};
CommonAnalysisPlugin plugin = new CommonAnalysisPlugin();
plugin.createComponents(null, null, null, null, scriptService, null, null, null, null);
AnalysisModule module
= new AnalysisModule(TestEnvironment.newEnvironment(settings), Collections.singletonList(plugin));
IndexAnalyzers analyzers = module.getAnalysisRegistry().build(idxSettings);
try (NamedAnalyzer analyzer = analyzers.get("myAnalyzer")) {
assertNotNull(analyzer);
assertAnalyzesTo(analyzer, "Vorsprung Durch Technik", new String[]{
"Vorsprung", "Technik"
});
}
}
}

37
modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/60_analysis_scripting.yml
View File

@ -28,9 +28,44 @@
- type: condition
filter: [ "lowercase" ]
script:
source: "token.position > 1 && token.positionIncrement > 0 && token.startOffset > 0 && token.endOffset > 0 && (token.positionLength == 1 || token.type == \"a\" || token.keyword)"
source: "token.position >= 1 && token.positionIncrement > 0 && token.startOffset > 0 && token.endOffset > 0 && (token.positionLength == 1 || token.type == \"a\" || token.keyword)"
- length: { tokens: 3 }
- match: { tokens.0.token: "Vorsprung" }
- match: { tokens.1.token: "durch" }
- match: { tokens.2.token: "technik" }
---
"script_filter":
- do:
indices.analyze:
body:
text: "Vorsprung Durch Technik"
tokenizer: "whitespace"
filter:
- type: predicate_token_filter
script:
source: "token.term.length() > 5"
- length: { tokens: 2 }
- match: { tokens.0.token: "Vorsprung" }
- match: { tokens.1.token: "Technik" }
---
"script_filter_position":
- do:
indices.analyze:
body:
text: "a b c d e f g h"
tokenizer: "whitespace"
filter:
- type: predicate_token_filter
script:
source: "token.position >= 4"
- length: { tokens: 4 }
- match: { tokens.0.token: "e" }
- match: { tokens.1.token: "f" }
- match: { tokens.2.token: "g" }
- match: { tokens.3.token: "h" }

1
modules/lang-painless/build.gradle
View File

@ -26,6 +26,7 @@ integTestCluster {
module project.project(':modules:mapper-extras')
systemProperty 'es.scripting.use_java_time', 'true'
systemProperty 'es.scripting.update.ctx_in_params', 'false'
systemProperty 'es.http.cname_in_publish_address', 'true'
}
dependencies {

7
plugins/repository-s3/build.gradle
View File

@ -32,19 +32,23 @@ esplugin {
}
versions << [
'aws': '1.11.223'
'aws': '1.11.406'
]
dependencies {
compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}"
compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
compile "com.amazonaws:jmespath-java:${versions.aws}"
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
compile "commons-logging:commons-logging:${versions.commonslogging}"
compile "commons-codec:commons-codec:${versions.commonscodec}"
compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7.1'
compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0'
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
compile 'joda-time:joda-time:2.10'
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
// and whitelist this hack in JarHell
@ -53,6 +57,7 @@ dependencies {
dependencyLicenses {
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
mapping from: /jackson-.*/, to: 'jackson'
mapping from: /jaxb-.*/, to: 'jaxb'
}

1
plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1
View File

@ -1 +0,0 @@
c3993cb44f5856fa721b7b7ccfc266377c0bf9c0

1
plugins/repository-s3/licenses/aws-java-sdk-core-1.11.406.jar.sha1 Normal file
View File

@ -0,0 +1 @@
43f3b7332d4d527bbf34d4ac6be094f3dabec6de

1
plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1
View File

@ -1 +0,0 @@
c24e6ebe108c60a08098aeaad5ae0b6a5a77b618

1
plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.406.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e29854e58dc20f5453c1da7e580a5921b1e9714a

1
plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1
View File

@ -1 +0,0 @@
c2ef96732e22d97952fbcd0a94f1dc376d157eda

1
plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.406.jar.sha1 Normal file
View File

@ -0,0 +1 @@
5c3c2c57b076602b3aeef841c63e5848ec52b00d

1
plugins/repository-s3/licenses/jmespath-java-1.11.406.jar.sha1 Normal file
View File

@ -0,0 +1 @@
06c291d1029943d4968a36fadffa3b71a6d8b4e4

35
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
View File

@ -23,10 +23,12 @@ import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.http.IdleConnectionReaper;
import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.internal.Constants;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.MapBuilder;
@ -93,19 +95,26 @@ class S3Service extends AbstractComponent implements Closeable {
}
}
private AmazonS3 buildClient(S3ClientSettings clientSettings) {
final AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings);
final ClientConfiguration configuration = buildConfiguration(clientSettings);
final AmazonS3 client = buildClient(credentials, configuration);
if (Strings.hasText(clientSettings.endpoint)) {
client.setEndpoint(clientSettings.endpoint);
}
return client;
}
// proxy for testing
AmazonS3 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
return new AmazonS3Client(credentials, configuration);
AmazonS3 buildClient(final S3ClientSettings clientSettings) {
final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
builder.withCredentials(buildCredentials(logger, clientSettings));
builder.withClientConfiguration(buildConfiguration(clientSettings));
final String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME;
logger.debug("using endpoint [{}]", endpoint);
// If the endpoint configuration isn't set on the builder then the default behaviour is to try
// and work out what region we are in and use an appropriate endpoint - see AwsClientBuilder#setRegion.
// In contrast, directly-constructed clients use s3.amazonaws.com unless otherwise instructed. We currently
// use a directly-constructed client, and need to keep the existing behaviour to avoid a breaking change,
// so to move to using the builder we must set it explicitly to keep the existing behaviour.
//
// We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too)
// so this change removes that usage of a deprecated API.
builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));
return builder.build();
}
// pkg private for tests

7
plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.repositories.s3;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
@ -70,9 +69,9 @@ public class RepositoryCredentialsTests extends ESTestCase {
}
@Override
AmazonS3 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
final AmazonS3 client = super.buildClient(credentials, configuration);
return new ClientAndCredentials(client, credentials);
AmazonS3 buildClient(final S3ClientSettings clientSettings) {
final AmazonS3 client = super.buildClient(clientSettings);
return new ClientAndCredentials(client, buildCredentials(logger, clientSettings));
}
}

6
qa/full-cluster-restart/build.gradle
View File

@ -53,9 +53,6 @@ for (Version version : bwcVersions.indexCompatible) {
// some tests rely on the translog not being flushed
setting 'indices.memory.shard_inactive_time', '20m'
// debug logging for testRecovery
setting 'logger.level', 'DEBUG'
if (version.onOrAfter('5.3.0')) {
setting 'http.content_type.required', 'true'
}
@ -75,9 +72,6 @@ for (Version version : bwcVersions.indexCompatible) {
// some tests rely on the translog not being flushed
setting 'indices.memory.shard_inactive_time', '20m'
// debug logging for testRecovery
setting 'logger.level', 'DEBUG'
numNodes = 2
dataDir = { nodeNum -> oldClusterTest.nodes[nodeNum].dataDir }
cleanShared = false // We want to keep snapshots made by the old cluster!

92
qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
View File

@ -68,10 +68,8 @@ import static org.hamcrest.Matchers.notNullValue;
* version is started with the same data directories and then this is rerun
* with {@code tests.is_old_cluster} set to {@code false}.
*/
public class FullClusterRestartIT extends ESRestTestCase {
private final boolean runningAgainstOldCluster = Booleans.parseBoolean(System.getProperty("tests.is_old_cluster"));
private final Version oldClusterVersion = Version.fromString(System.getProperty("tests.old_cluster_version"));
private final boolean supportsLenientBooleans = oldClusterVersion.before(Version.V_6_0_0_alpha1);
public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
private final boolean supportsLenientBooleans = getOldClusterVersion().before(Version.V_6_0_0_alpha1);
private static final Version VERSION_5_1_0_UNRELEASED = Version.fromString("5.1.0");
private String index;
@ -81,29 +79,9 @@ public class FullClusterRestartIT extends ESRestTestCase {
index = getTestName().toLowerCase(Locale.ROOT);
}
@Override
protected boolean preserveIndicesUponCompletion() {
return true;
}
@Override
protected boolean preserveSnapshotsUponCompletion() {
return true;
}
@Override
protected boolean preserveReposUponCompletion() {
return true;
}
@Override
protected boolean preserveTemplatesUponCompletion() {
return true;
}
public void testSearch() throws Exception {
int count;
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
XContentBuilder mappingsAndSettings = jsonBuilder();
mappingsAndSettings.startObject();
{
@ -169,7 +147,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
}
public void testNewReplicasWork() throws Exception {
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
XContentBuilder mappingsAndSettings = jsonBuilder();
mappingsAndSettings.startObject();
{
@ -237,10 +215,10 @@ public class FullClusterRestartIT extends ESRestTestCase {
*/
public void testAliasWithBadName() throws Exception {
assumeTrue("Can only test bad alias name if old cluster is on 5.1.0 or before",
oldClusterVersion.before(VERSION_5_1_0_UNRELEASED));
getOldClusterVersion().before(VERSION_5_1_0_UNRELEASED));
int count;
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
XContentBuilder mappingsAndSettings = jsonBuilder();
mappingsAndSettings.startObject();
{
@ -291,7 +269,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
Map<String, Object> searchRsp = entityAsMap(client().performRequest(new Request("GET", "/" + aliasName + "/_search")));
int totalHits = (int) XContentMapValues.extractValue("hits.total", searchRsp);
assertEquals(count, totalHits);
if (runningAgainstOldCluster == false) {
if (isRunningAgainstOldCluster() == false) {
// We can remove the alias.
Response response = client().performRequest(new Request("DELETE", "/" + index + "/_alias/" + aliasName));
assertEquals(200, response.getStatusLine().getStatusCode());
@ -302,7 +280,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
}
public void testClusterState() throws Exception {
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
XContentBuilder mappingsAndSettings = jsonBuilder();
mappingsAndSettings.startObject();
mappingsAndSettings.field("template", index);
@ -341,14 +319,14 @@ public class FullClusterRestartIT extends ESRestTestCase {
assertEquals("0", numberOfReplicas);
Version version = Version.fromId(Integer.valueOf((String) XContentMapValues.extractValue("metadata.indices." + index +
".settings.index.version.created", clusterState)));
assertEquals(oldClusterVersion, version);
assertEquals(getOldClusterVersion(), version);
}
public void testShrink() throws IOException {
String shrunkenIndex = index + "_shrunk";
int numDocs;
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
XContentBuilder mappingsAndSettings = jsonBuilder();
mappingsAndSettings.startObject();
{
@ -413,7 +391,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
public void testShrinkAfterUpgrade() throws IOException {
String shrunkenIndex = index + "_shrunk";
int numDocs;
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
XContentBuilder mappingsAndSettings = jsonBuilder();
mappingsAndSettings.startObject();
{
@ -465,7 +443,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
int totalHits = (int) XContentMapValues.extractValue("hits.total", response);
assertEquals(numDocs, totalHits);
if (runningAgainstOldCluster == false) {
if (isRunningAgainstOldCluster() == false) {
response = entityAsMap(client().performRequest(new Request("GET", "/" + shrunkenIndex + "/_search")));
assertNoFailures(response);
totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
@ -490,7 +468,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
* </ol>
*/
public void testRollover() throws IOException {
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
Request createIndex = new Request("PUT", "/" + index + "-000001");
createIndex.setJsonEntity("{"
+ " \"aliases\": {"
@ -511,7 +489,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
bulkRequest.addParameter("refresh", "");
assertThat(EntityUtils.toString(client().performRequest(bulkRequest).getEntity()), containsString("\"errors\":false"));
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
Request rolloverRequest = new Request("POST", "/" + index + "_write/_rollover");
rolloverRequest.setJsonEntity("{"
+ " \"conditions\": {"
@ -529,7 +507,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
Map<String, Object> count = entityAsMap(client().performRequest(countRequest));
assertNoFailures(count);
int expectedCount = bulkCount + (runningAgainstOldCluster ? 0 : bulkCount);
int expectedCount = bulkCount + (isRunningAgainstOldCluster() ? 0 : bulkCount);
assertEquals(expectedCount, (int) XContentMapValues.extractValue("hits.total", count));
}
@ -688,7 +666,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
String docLocation = "/" + index + "/doc/1";
String doc = "{\"test\": \"test\"}";
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
Request createDoc = new Request("PUT", docLocation);
createDoc.setJsonEntity(doc);
client().performRequest(createDoc);
@ -703,7 +681,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
public void testEmptyShard() throws IOException {
final String index = "test_empty_shard";
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
Settings.Builder settings = Settings.builder()
.put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
@ -726,7 +704,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
public void testRecovery() throws Exception {
int count;
boolean shouldHaveTranslog;
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
count = between(200, 300);
/* We've had bugs in the past where we couldn't restore
* an index without a translog so we randomize whether
@ -772,7 +750,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
String countResponse = toStr(client().performRequest(countRequest));
assertThat(countResponse, containsString("\"total\":" + count));
if (false == runningAgainstOldCluster) {
if (false == isRunningAgainstOldCluster()) {
boolean restoredFromTranslog = false;
boolean foundPrimary = false;
Request recoveryRequest = new Request("GET", "/_cat/recovery/" + index);
@ -800,7 +778,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
assertEquals("mismatch while checking for translog recovery\n" + recoveryResponse, shouldHaveTranslog, restoredFromTranslog);
String currentLuceneVersion = Version.CURRENT.luceneVersion.toString();
String bwcLuceneVersion = oldClusterVersion.luceneVersion.toString();
String bwcLuceneVersion = getOldClusterVersion().luceneVersion.toString();
if (shouldHaveTranslog && false == currentLuceneVersion.equals(bwcLuceneVersion)) {
int numCurrentVersion = 0;
int numBwcVersion = 0;
@ -840,7 +818,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
*/
public void testSnapshotRestore() throws IOException {
int count;
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
// Create the index
count = between(200, 300);
indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject());
@ -860,7 +838,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
// Stick a routing attribute into to cluster settings so we can see it after the restore
Request addRoutingSettings = new Request("PUT", "/_cluster/settings");
addRoutingSettings.setJsonEntity(
"{\"persistent\": {\"cluster.routing.allocation.exclude.test_attr\": \"" + oldClusterVersion + "\"}}");
"{\"persistent\": {\"cluster.routing.allocation.exclude.test_attr\": \"" + getOldClusterVersion() + "\"}}");
client().performRequest(addRoutingSettings);
// Stick a template into the cluster so we can see it after the restore
@ -885,7 +863,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
templateBuilder.startObject("alias2"); {
templateBuilder.startObject("filter"); {
templateBuilder.startObject("term"); {
templateBuilder.field("version", runningAgainstOldCluster ? oldClusterVersion : Version.CURRENT);
templateBuilder.field("version", isRunningAgainstOldCluster() ? getOldClusterVersion() : Version.CURRENT);
}
templateBuilder.endObject();
}
@ -898,7 +876,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
createTemplateRequest.setJsonEntity(Strings.toString(templateBuilder));
client().performRequest(createTemplateRequest);
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
// Create the repo
XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject(); {
repoConfig.field("type", "fs");
@ -914,19 +892,19 @@ public class FullClusterRestartIT extends ESRestTestCase {
client().performRequest(createRepoRequest);
}
Request createSnapshot = new Request("PUT", "/_snapshot/repo/" + (runningAgainstOldCluster ? "old_snap" : "new_snap"));
Request createSnapshot = new Request("PUT", "/_snapshot/repo/" + (isRunningAgainstOldCluster() ? "old_snap" : "new_snap"));
createSnapshot.addParameter("wait_for_completion", "true");
createSnapshot.setJsonEntity("{\"indices\": \"" + index + "\"}");
client().performRequest(createSnapshot);
checkSnapshot("old_snap", count, oldClusterVersion);
if (false == runningAgainstOldCluster) {
checkSnapshot("old_snap", count, getOldClusterVersion());
if (false == isRunningAgainstOldCluster()) {
checkSnapshot("new_snap", count, Version.CURRENT);
}
}
public void testHistoryUUIDIsAdded() throws Exception {
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
XContentBuilder mappingsAndSettings = jsonBuilder();
mappingsAndSettings.startObject();
{
@ -1019,20 +997,14 @@ public class FullClusterRestartIT extends ESRestTestCase {
Request clusterSettingsRequest = new Request("GET", "/_cluster/settings");
clusterSettingsRequest.addParameter("flat_settings", "true");
Map<String, Object> clusterSettingsResponse = entityAsMap(client().performRequest(clusterSettingsRequest));
Map<String, Object> expectedClusterSettings = new HashMap<>();
expectedClusterSettings.put("transient", emptyMap());
expectedClusterSettings.put("persistent",
singletonMap("cluster.routing.allocation.exclude.test_attr", oldClusterVersion.toString()));
if (expectedClusterSettings.equals(clusterSettingsResponse) == false) {
NotEqualMessageBuilder builder = new NotEqualMessageBuilder();
builder.compareMaps(clusterSettingsResponse, expectedClusterSettings);
fail("settings don't match:\n" + builder.toString());
}
@SuppressWarnings("unchecked") final Map<String, Object> persistentSettings =
(Map<String, Object>)clusterSettingsResponse.get("persistent");
assertThat(persistentSettings.get("cluster.routing.allocation.exclude.test_attr"), equalTo(getOldClusterVersion().toString()));
// Check that the template was restored successfully
Map<String, Object> getTemplateResponse = entityAsMap(client().performRequest(new Request("GET", "/_template/test_template")));
Map<String, Object> expectedTemplate = new HashMap<>();
if (runningAgainstOldCluster && oldClusterVersion.before(Version.V_6_0_0_beta1)) {
if (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_6_0_0_beta1)) {
expectedTemplate.put("template", "evil_*");
} else {
expectedTemplate.put("index_patterns", singletonList("evil_*"));

101
qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.java Normal file
View File

@ -0,0 +1,101 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.upgrades;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.transport.RemoteClusterService;
import java.io.IOException;
import java.util.Collections;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.transport.RemoteClusterAware.SEARCH_REMOTE_CLUSTERS_SEEDS;
import static org.elasticsearch.transport.RemoteClusterService.SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE;
import static org.hamcrest.Matchers.equalTo;
public class FullClusterRestartSettingsUpgradeIT extends AbstractFullClusterRestartTestCase {
public void testRemoteClusterSettingsUpgraded() throws IOException {
assumeTrue("settings automatically upgraded since 6.5.0", getOldClusterVersion().before(Version.V_6_5_0));
if (isRunningAgainstOldCluster()) {
final Request putSettingsRequest = new Request("PUT", "/_cluster/settings");
try (XContentBuilder builder = jsonBuilder()) {
builder.startObject();
{
builder.startObject("persistent");
{
builder.field("search.remote.foo.skip_unavailable", true);
builder.field("search.remote.foo.seeds", Collections.singletonList("localhost:9200"));
}
builder.endObject();
}
builder.endObject();
putSettingsRequest.setJsonEntity(Strings.toString(builder));
}
client().performRequest(putSettingsRequest);
final Request getSettingsRequest = new Request("GET", "/_cluster/settings");
final Response response = client().performRequest(getSettingsRequest);
try (XContentParser parser = createParser(JsonXContent.jsonXContent, response.getEntity().getContent())) {
final ClusterGetSettingsResponse clusterGetSettingsResponse = ClusterGetSettingsResponse.fromXContent(parser);
final Settings settings = clusterGetSettingsResponse.getPersistentSettings();
assertTrue(SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE.getConcreteSettingForNamespace("foo").exists(settings));
assertTrue(SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE.getConcreteSettingForNamespace("foo").get(settings));
assertTrue(SEARCH_REMOTE_CLUSTERS_SEEDS.getConcreteSettingForNamespace("foo").exists(settings));
assertThat(
SEARCH_REMOTE_CLUSTERS_SEEDS.getConcreteSettingForNamespace("foo").get(settings),
equalTo(Collections.singletonList("localhost:9200")));
}
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE.getConcreteSettingForNamespace("foo"),
SEARCH_REMOTE_CLUSTERS_SEEDS.getConcreteSettingForNamespace("foo")});
} else {
final Request getSettingsRequest = new Request("GET", "/_cluster/settings");
final Response getSettingsResponse = client().performRequest(getSettingsRequest);
try (XContentParser parser = createParser(JsonXContent.jsonXContent, getSettingsResponse.getEntity().getContent())) {
final ClusterGetSettingsResponse clusterGetSettingsResponse = ClusterGetSettingsResponse.fromXContent(parser);
final Settings settings = clusterGetSettingsResponse.getPersistentSettings();
assertFalse(SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE.getConcreteSettingForNamespace("foo").exists(settings));
assertTrue(
settings.toString(),
RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE.getConcreteSettingForNamespace("foo").exists(settings));
assertTrue(RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE.getConcreteSettingForNamespace("foo").get(settings));
assertFalse(SEARCH_REMOTE_CLUSTERS_SEEDS.getConcreteSettingForNamespace("foo").exists(settings));
assertTrue(RemoteClusterService.REMOTE_CLUSTERS_SEEDS.getConcreteSettingForNamespace("foo").exists(settings));
assertThat(
RemoteClusterService.REMOTE_CLUSTERS_SEEDS.getConcreteSettingForNamespace("foo").get(settings),
equalTo(Collections.singletonList("localhost:9200")));
}
}
}
}

32
qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
View File

@ -20,10 +20,8 @@
package org.elasticsearch.upgrades;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.InputStreamStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@ -48,7 +46,6 @@ import org.elasticsearch.index.query.SpanTermQueryBuilder;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.test.rest.ESRestTestCase;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -71,7 +68,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
* The queries to test are specified in json format, which turns out to work because we tend break here rarely. If the
* json format of a query being tested here then feel free to change this.
*/
public class QueryBuilderBWCIT extends ESRestTestCase {
public class QueryBuilderBWCIT extends AbstractFullClusterRestartTestCase {
private static final List<Object[]> CANDIDATES = new ArrayList<>();
@ -145,32 +142,9 @@ public class QueryBuilderBWCIT extends ESRestTestCase {
CANDIDATES.add(new Object[]{"{\"query\": {" + querySource + "}}", expectedQb});
}
private final Version oldClusterVersion = Version.fromString(System.getProperty("tests.old_cluster_version"));
private final boolean runningAgainstOldCluster = Booleans.parseBoolean(System.getProperty("tests.is_old_cluster"));
@Override
protected boolean preserveIndicesUponCompletion() {
return true;
}
@Override
protected boolean preserveSnapshotsUponCompletion() {
return true;
}
@Override
protected boolean preserveReposUponCompletion() {
return true;
}
@Override
protected boolean preserveTemplatesUponCompletion() {
return true;
}
public void testQueryBuilderBWC() throws Exception {
String index = "queries";
if (runningAgainstOldCluster) {
if (isRunningAgainstOldCluster()) {
XContentBuilder mappingsAndSettings = jsonBuilder();
mappingsAndSettings.startObject();
{
@ -230,7 +204,7 @@ public class QueryBuilderBWCIT extends ESRestTestCase {
byte[] qbSource = Base64.getDecoder().decode(queryBuilderStr);
try (InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length)) {
try (StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry)) {
input.setVersion(oldClusterVersion);
input.setVersion(getOldClusterVersion());
QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
assert in.read() == -1;
assertEquals(expectedQueryBuilder, queryBuilder);

18
rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml
View File

@ -139,12 +139,26 @@ setup:
features: warnings
- do:
warnings:
- 'Doc-value field [count] is not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with the doc value field in order to opt in for the future behaviour and ease the migration to 7.0.'
- 'There are doc-value fields which are not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with a doc value field in order to opt in for the future behaviour and ease the migration to 7.0: [count]'
search:
body:
docvalue_fields: [ "count" ]
- match: { hits.hits.0.fields.count: [1] }
---
"multiple docvalue_fields":
- skip:
version: " - 6.3.99"
reason: format option was added in 6.4
features: warnings
- do:
warnings:
- 'There are doc-value fields which are not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with a doc value field in order to opt in for the future behaviour and ease the migration to 7.0: [count, include.field1.keyword]'
search:
body:
docvalue_fields: [ "count", "include.field1.keyword" ]
- match: { hits.hits.0.fields.count: [1] }
---
"docvalue_fields as url param":
- skip:
@ -153,7 +167,7 @@ setup:
features: warnings
- do:
warnings:
- 'Doc-value field [count] is not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with the doc value field in order to opt in for the future behaviour and ease the migration to 7.0.'
- 'There are doc-value fields which are not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with a doc value field in order to opt in for the future behaviour and ease the migration to 7.0: [count]'
search:
docvalue_fields: [ "count" ]
- match: { hits.hits.0.fields.count: [1] }

7
server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
View File

@ -39,6 +39,7 @@ import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.threadpool.ThreadPool;
@ -171,6 +172,12 @@ public class TransportResizeAction extends TransportMasterNodeAction<ResizeReque
throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize");
}
}
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(metaData.getSettings()) &&
IndexSettings.INDEX_SOFT_DELETES_SETTING.get(metaData.getSettings()) &&
IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(targetIndexSettings) &&
IndexSettings.INDEX_SOFT_DELETES_SETTING.get(targetIndexSettings) == false) {
throw new IllegalArgumentException("Can't disable [index.soft_deletes.enabled] setting on resize");
}
String cause = resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT) + "_index";
targetIndex.cause(cause);
Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings);

8
server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java
View File

@ -22,7 +22,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.TransportActions;
import org.elasticsearch.action.support.replication.ReplicationOperation;
import org.elasticsearch.action.support.replication.ReplicationResponse;
import org.elasticsearch.action.support.replication.TransportReplicationAction;
@ -171,12 +170,7 @@ public class TransportResyncReplicationAction extends TransportWriteAction<Resyn
@Override
public void handleException(TransportException exp) {
final Throwable cause = exp.unwrapCause();
if (TransportActions.isShardNotAvailableException(cause)) {
logger.trace("primary became unavailable during resync, ignoring", exp);
} else {
listener.onFailure(exp);
}
listener.onFailure(exp);
}
});
}

3
server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -749,7 +749,8 @@ public class MetaDataCreateIndexService extends AbstractComponent {
}
} else {
final Predicate<String> sourceSettingsPredicate =
(s) -> (s.startsWith("index.similarity.") || s.startsWith("index.analysis.") || s.startsWith("index.sort."))
(s) -> (s.startsWith("index.similarity.") || s.startsWith("index.analysis.") ||
s.startsWith("index.sort.") || s.equals("index.soft_deletes.enabled"))
&& indexSettingsBuilder.keys().contains(s) == false;
builder.put(sourceMetaData.getSettings().filter(sourceSettingsPredicate));
}

27
server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
View File

@ -27,7 +27,6 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.regex.Regex;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -54,7 +53,7 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
private final List<SettingUpdater<?>> settingUpdaters = new CopyOnWriteArrayList<>();
private final Map<String, Setting<?>> complexMatchers;
private final Map<String, Setting<?>> keySettings;
private final Map<Setting<?>, Function<Map.Entry<String, String>, Map.Entry<String, String>>> settingUpgraders;
private final Map<Setting<?>, SettingUpgrader<?>> settingUpgraders;
private final Setting.Property scope;
private static final Pattern KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])*[-\\w]+$");
private static final Pattern GROUP_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+$");
@ -70,12 +69,8 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
this.settingUpgraders =
Collections.unmodifiableMap(
settingUpgraders
.stream()
.collect(
Collectors.toMap(
SettingUpgrader::getSetting,
u -> e -> new AbstractMap.SimpleEntry<>(u.getKey(e.getKey()), u.getValue(e.getValue())))));
settingUpgraders.stream().collect(Collectors.toMap(SettingUpgrader::getSetting, Function.identity())));
this.scope = scope;
Map<String, Setting<?>> complexMatchers = new HashMap<>();
@ -786,15 +781,25 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
boolean changed = false; // track if any settings were upgraded
for (final String key : settings.keySet()) {
final Setting<?> setting = getRaw(key);
final Function<Map.Entry<String, String>, Map.Entry<String, String>> upgrader = settingUpgraders.get(setting);
final SettingUpgrader<?> upgrader = settingUpgraders.get(setting);
if (upgrader == null) {
// the setting does not have an upgrader, copy the setting
builder.copy(key, settings);
} else {
// the setting has an upgrader, so mark that we have changed a setting and apply the upgrade logic
changed = true;
final Map.Entry<String, String> upgrade = upgrader.apply(new Entry(key, settings));
builder.put(upgrade.getKey(), upgrade.getValue());
// noinspection ConstantConditions
if (setting.getConcreteSetting(key).isListSetting()) {
final List<String> value = settings.getAsList(key);
final String upgradedKey = upgrader.getKey(key);
final List<String> upgradedValue = upgrader.getListValue(value);
builder.putList(upgradedKey, upgradedValue);
} else {
final String value = settings.get(key);
final String upgradedKey = upgrader.getKey(key);
final String upgradedValue = upgrader.getValue(value);
builder.put(upgradedKey, upgradedValue);
}
}
}
// we only return a new instance if there was an upgrade

5
server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View File

@ -443,6 +443,9 @@ public final class ClusterSettings extends AbstractScopedSettings {
EnableAssignmentDecider.CLUSTER_TASKS_ALLOCATION_ENABLE_SETTING
)));
public static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList();
public static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.unmodifiableList(Arrays.asList(
RemoteClusterAware.SEARCH_REMOTE_CLUSTER_SEEDS_UPGRADER,
RemoteClusterAware.SEARCH_REMOTE_CLUSTERS_PROXY_UPGRADER,
RemoteClusterService.SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE_UPGRADER));
}

8
server/src/main/java/org/elasticsearch/common/settings/Setting.java
View File

@ -345,6 +345,11 @@ public class Setting<T> implements ToXContentObject {
return false;
}
final boolean isListSetting() {
return this instanceof ListSetting;
}
boolean hasComplexMatcher() {
return isGroupSetting();
}
@ -453,7 +458,7 @@ public class Setting<T> implements ToXContentObject {
* @return the raw string representation of the setting value
*/
String innerGetRaw(final Settings settings) {
return settings.get(getKey(), defaultValue.apply(settings));
return settings.get(getKey(), defaultValue.apply(settings), isListSetting());
}
/** Logs a deprecation warning if the setting is deprecated and used. */
@ -1305,7 +1310,6 @@ public class Setting<T> implements ToXContentObject {
}
}
}
}
static void logSettingUpdate(Setting setting, Settings current, Settings previous, Logger logger) {

6
server/src/main/java/org/elasticsearch/common/settings/SettingUpgrader.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.common.settings;
import java.util.List;
/**
* Represents the logic to upgrade a setting.
*
@ -51,4 +53,8 @@ public interface SettingUpgrader<T> {
return value;
}
default List<String> getListValue(final List<String> value) {
return value;
}
}

24
server/src/main/java/org/elasticsearch/common/settings/Settings.java
View File

@ -245,6 +245,30 @@ public final class Settings implements ToXContentFragment {
return retVal == null ? defaultValue : retVal;
}
/**
* Returns the setting value associated with the setting key. If it does not exists,
* returns the default value provided.
*/
String get(String setting, String defaultValue, boolean isList) {
Object value = settings.get(setting);
if (value != null) {
if (value instanceof List) {
if (isList == false) {
throw new IllegalArgumentException(
"Found list type value for setting [" + setting + "] but but did not expect a list for it."
);
}
} else if (isList) {
throw new IllegalArgumentException(
"Expected list type value for setting [" + setting + "] but found [" + value.getClass() + ']'
);
}
return toString(value);
} else {
return defaultValue;
}
}
/**
* Returns the setting value (as float) associated with the setting key. If it does not exists,
* returns the default value provided.

9
server/src/main/java/org/elasticsearch/common/util/concurrent/FutureUtils.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.common.util.concurrent;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.SuppressForbidden;
import java.util.concurrent.ExecutionException;
@ -30,8 +31,14 @@ import java.util.concurrent.TimeoutException;
public class FutureUtils {
/**
* Cancel execution of this future without interrupting a running thread. See {@link Future#cancel(boolean)} for details.
*
* @param toCancel the future to cancel
* @return false if the future could not be cancelled, otherwise true
*/
@SuppressForbidden(reason = "Future#cancel()")
public static boolean cancel(Future<?> toCancel) {
public static boolean cancel(@Nullable final Future<?> toCancel) {
if (toCancel != null) {
return toCancel.cancel(false); // this method is a forbidden API since it interrupts threads
}

2
server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
View File

@ -131,7 +131,7 @@ public class DiscoveryModule {
if (discoverySupplier == null) {
throw new IllegalArgumentException("Unknown discovery type [" + discoveryType + "]");
}
Loggers.getLogger(getClass(), settings).info("using discovery type [{}]", discoveryType);
Loggers.getLogger(getClass(), settings).info("using discovery type [{}] and host providers {}", discoveryType, hostsProviderNames);
discovery = Objects.requireNonNull(discoverySupplier.get());
}

47
server/src/main/java/org/elasticsearch/http/HttpInfo.java
View File

@ -19,24 +19,46 @@
package org.elasticsearch.http;
import org.apache.logging.log4j.LogManager;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.common.transport.BoundTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import static org.elasticsearch.common.Booleans.parseBoolean;
public class HttpInfo implements Writeable, ToXContentFragment {
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(HttpInfo.class));
/** Whether to add hostname to publish host field when serializing. */
private static final boolean CNAME_IN_PUBLISH_HOST =
parseBoolean(System.getProperty("es.http.cname_in_publish_address"), false);
private final BoundTransportAddress address;
private final long maxContentLength;
private final boolean cnameInPublishHost;
public HttpInfo(StreamInput in) throws IOException {
address = BoundTransportAddress.readBoundTransportAddress(in);
maxContentLength = in.readLong();
this(BoundTransportAddress.readBoundTransportAddress(in), in.readLong(), CNAME_IN_PUBLISH_HOST);
}
public HttpInfo(BoundTransportAddress address, long maxContentLength) {
this(address, maxContentLength, CNAME_IN_PUBLISH_HOST);
}
HttpInfo(BoundTransportAddress address, long maxContentLength, boolean cnameInPublishHost) {
this.address = address;
this.maxContentLength = maxContentLength;
this.cnameInPublishHost = cnameInPublishHost;
}
@Override
@ -45,11 +67,6 @@ public class HttpInfo implements Writeable, ToXContentFragment {
out.writeLong(maxContentLength);
}
public HttpInfo(BoundTransportAddress address, long maxContentLength) {
this.address = address;
this.maxContentLength = maxContentLength;
}
static final class Fields {
static final String HTTP = "http";
static final String BOUND_ADDRESS = "bound_address";
@ -62,7 +79,21 @@ public class HttpInfo implements Writeable, ToXContentFragment {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.HTTP);
builder.array(Fields.BOUND_ADDRESS, (Object[]) address.boundAddresses());
builder.field(Fields.PUBLISH_ADDRESS, address.publishAddress().toString());
TransportAddress publishAddress = address.publishAddress();
String publishAddressString = publishAddress.toString();
String hostString = publishAddress.address().getHostString();
if (InetAddresses.isInetAddress(hostString) == false) {
if (cnameInPublishHost) {
publishAddressString = hostString + '/' + publishAddress.toString();
} else {
DEPRECATION_LOGGER.deprecated(
"[http.publish_host] was printed as [ip:port] instead of [hostname/ip:port]. "
+ "This format is deprecated and will change to [hostname/ip:port] in a future version. "
+ "Use -Des.http.cname_in_publish_address=true to enforce non-deprecated formatting."
);
}
}
builder.field(Fields.PUBLISH_ADDRESS, publishAddressString);
builder.humanReadableField(Fields.MAX_CONTENT_LENGTH_IN_BYTES, Fields.MAX_CONTENT_LENGTH, maxContentLength());
builder.endObject();
return builder;

21
server/src/main/java/org/elasticsearch/index/engine/Engine.java
View File

@ -661,7 +661,7 @@ public abstract class Engine implements Closeable {
}
/** get commits stats for the last commit */
public CommitStats commitStats() {
public final CommitStats commitStats() {
return new CommitStats(getLastCommittedSegmentInfos());
}
@ -678,12 +678,6 @@ public abstract class Engine implements Closeable {
*/
public abstract void waitForOpsToComplete(long seqNo) throws InterruptedException;
/**
* Reset the local checkpoint in the tracker to the given local checkpoint
* @param localCheckpoint the new checkpoint to be set
*/
public abstract void resetLocalCheckpoint(long localCheckpoint);
/**
* @return a {@link SeqNoStats} object, using local state and the supplied global checkpoint
*/
@ -951,7 +945,9 @@ public abstract class Engine implements Closeable {
*
* @return the commit Id for the resulting commit
*/
public abstract CommitId flush() throws EngineException;
public final CommitId flush() throws EngineException {
return flush(false, false);
}
/**
@ -1163,11 +1159,16 @@ public abstract class Engine implements Closeable {
PRIMARY,
REPLICA,
PEER_RECOVERY,
LOCAL_TRANSLOG_RECOVERY;
LOCAL_TRANSLOG_RECOVERY,
LOCAL_RESET;
public boolean isRecovery() {
return this == PEER_RECOVERY || this == LOCAL_TRANSLOG_RECOVERY;
}
boolean isFromTranslog() {
return this == LOCAL_TRANSLOG_RECOVERY || this == LOCAL_RESET;
}
}
public Origin origin() {
@ -1593,7 +1594,7 @@ public abstract class Engine implements Closeable {
private final CheckedRunnable<IOException> onClose;
private final IndexCommit indexCommit;
IndexCommitRef(IndexCommit indexCommit, CheckedRunnable<IOException> onClose) {
public IndexCommitRef(IndexCommit indexCommit, CheckedRunnable<IOException> onClose) {
this.indexCommit = indexCommit;
this.onClose = onClose;
}

1
server/src/main/java/org/elasticsearch/index/engine/EngineFactory.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.engine;
/**
* Simple Engine Factory
*/
@FunctionalInterface
public interface EngineFactory {
Engine newReadWriteEngine(EngineConfig config);

30
server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
View File

@ -152,12 +152,6 @@ public class InternalEngine extends Engine {
private final SoftDeletesPolicy softDeletesPolicy;
private final LastRefreshedCheckpointListener lastRefreshedCheckpointListener;
/**
* How many bytes we are currently moving to disk, via either IndexWriter.flush or refresh. IndexingMemoryController polls this
* across all shards to decide if throttling is necessary because moving bytes to disk is falling behind vs incoming documents
* being indexed/deleted.
*/
private final AtomicLong writingBytes = new AtomicLong();
private final AtomicBoolean trackTranslogLocation = new AtomicBoolean(false);
@Nullable
@ -530,7 +524,7 @@ public class InternalEngine extends Engine {
/** Returns how many bytes we are currently moving from indexing buffer to segments on disk */
@Override
public long getWritingBytes() {
return writingBytes.get();
return indexWriter.getFlushingBytes() + versionMap.getRefreshingBytes();
}
/**
@ -735,6 +729,7 @@ public class InternalEngine extends Engine {
: "version: " + index.version() + " type: " + index.versionType();
return true;
case LOCAL_TRANSLOG_RECOVERY:
case LOCAL_RESET:
assert index.isRetry();
return true; // allow to optimize in order to update the max safe time stamp
default:
@ -833,7 +828,7 @@ public class InternalEngine extends Engine {
indexResult = new IndexResult(
plan.versionForIndexing, getPrimaryTerm(), plan.seqNoForIndexing, plan.currentNotFoundOrDeleted);
}
if (index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
if (index.origin().isFromTranslog() == false) {
final Translog.Location location;
if (indexResult.getResultType() == Result.Type.SUCCESS) {
location = translog.add(new Translog.Index(index, indexResult));
@ -1173,7 +1168,7 @@ public class InternalEngine extends Engine {
deleteResult = new DeleteResult(
plan.versionOfDeletion, getPrimaryTerm(), plan.seqNoOfDeletion, plan.currentlyDeleted == false);
}
if (delete.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
if (delete.origin().isFromTranslog() == false) {
final Translog.Location location;
if (deleteResult.getResultType() == Result.Type.SUCCESS) {
location = translog.add(new Translog.Delete(delete, deleteResult));
@ -1411,7 +1406,7 @@ public class InternalEngine extends Engine {
}
}
final NoOpResult noOpResult = failure != null ? new NoOpResult(getPrimaryTerm(), noOp.seqNo(), failure) : new NoOpResult(getPrimaryTerm(), noOp.seqNo());
if (noOp.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
if (noOp.origin().isFromTranslog() == false) {
final Translog.Location location = translog.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason()));
noOpResult.setTranslogLocation(location);
}
@ -1437,9 +1432,6 @@ public class InternalEngine extends Engine {
// pass the new reader reference to the external reader manager.
final long localCheckpointBeforeRefresh = getLocalCheckpoint();
// this will also cause version map ram to be freed hence we always account for it.
final long bytes = indexWriter.ramBytesUsed() + versionMap.ramBytesUsedForRefresh();
writingBytes.addAndGet(bytes);
try (ReleasableLock lock = readLock.acquire()) {
ensureOpen();
if (store.tryIncRef()) {
@ -1465,8 +1457,6 @@ public class InternalEngine extends Engine {
e.addSuppressed(inner);
}
throw new RefreshFailedEngineException(shardId, e);
} finally {
writingBytes.addAndGet(-bytes);
}
assert lastRefreshedCheckpoint() >= localCheckpointBeforeRefresh : "refresh checkpoint was not advanced; " +
"local_checkpoint=" + localCheckpointBeforeRefresh + " refresh_checkpoint=" + lastRefreshedCheckpoint();
@ -1576,11 +1566,6 @@ public class InternalEngine extends Engine {
|| localCheckpointTracker.getCheckpoint() == localCheckpointTracker.getMaxSeqNo();
}
@Override
public CommitId flush() throws EngineException {
return flush(false, false);
}
@Override
public CommitId flush(boolean force, boolean waitIfOngoing) throws EngineException {
ensureOpen();
@ -2340,11 +2325,6 @@ public class InternalEngine extends Engine {
localCheckpointTracker.waitForOpsToComplete(seqNo);
}
@Override
public void resetLocalCheckpoint(long localCheckpoint) {
localCheckpointTracker.resetCheckpoint(localCheckpoint);
}
@Override
public SeqNoStats getSeqNoStats(long globalCheckpoint) {
return localCheckpointTracker.getStats(globalCheckpoint);

8
server/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
View File

@ -434,6 +434,14 @@ final class LiveVersionMap implements ReferenceManager.RefreshListener, Accounta
return maps.current.ramBytesUsed.get();
}
/**
* Returns how much RAM is current being freed up by refreshing. This is {@link #ramBytesUsed()}
* except does not include tombstones because they don't clear on refresh.
*/
long getRefreshingBytes() {
return maps.old.ramBytesUsed.get();
}
@Override
public Collection<Accountable> getChildResources() {
// TODO: useful to break down RAM usage here?

368
server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java Normal file
View File

@ -0,0 +1,368 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.engine;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ReferenceManager;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.seqno.SeqNoStats;
import org.elasticsearch.index.seqno.SequenceNumbers;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.index.translog.TranslogStats;
import java.io.Closeable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.Stream;
/**
* A basic read-only engine that allows switching a shard to be true read-only temporarily or permanently.
* Note: this engine can be opened side-by-side with a read-write engine but will not reflect any changes made to the read-write
* engine.
*
* @see #ReadOnlyEngine(EngineConfig, SeqNoStats, TranslogStats, boolean, Function)
*/
public final class ReadOnlyEngine extends Engine {
private final SegmentInfos lastCommittedSegmentInfos;
private final SeqNoStats seqNoStats;
private final TranslogStats translogStats;
private final SearcherManager searcherManager;
private final IndexCommit indexCommit;
private final Lock indexWriterLock;
/**
* Creates a new ReadOnlyEngine. This ctor can also be used to open a read-only engine on top of an already opened
* read-write engine. It allows to optionally obtain the writer locks for the shard which would time-out if another
* engine is still open.
*
* @param config the engine configuration
* @param seqNoStats sequence number statistics for this engine or null if not provided
* @param translogStats translog stats for this engine or null if not provided
* @param obtainLock if <code>true</code> this engine will try to obtain the {@link IndexWriter#WRITE_LOCK_NAME} lock. Otherwise
* the lock won't be obtained
* @param readerWrapperFunction allows to wrap the index-reader for this engine.
*/
public ReadOnlyEngine(EngineConfig config, SeqNoStats seqNoStats, TranslogStats translogStats, boolean obtainLock,
Function<DirectoryReader, DirectoryReader> readerWrapperFunction) {
super(config);
try {
Store store = config.getStore();
store.incRef();
DirectoryReader reader = null;
Directory directory = store.directory();
Lock indexWriterLock = null;
boolean success = false;
try {
// we obtain the IW lock even though we never modify the index.
// yet this makes sure nobody else does. including some testing tools that try to be messy
indexWriterLock = obtainLock ? directory.obtainLock(IndexWriter.WRITE_LOCK_NAME) : null;
this.lastCommittedSegmentInfos = Lucene.readSegmentInfos(directory);
this.translogStats = translogStats == null ? new TranslogStats(0, 0, 0, 0, 0) : translogStats;
this.seqNoStats = seqNoStats == null ? buildSeqNoStats(lastCommittedSegmentInfos) : seqNoStats;
reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), config.getShardId());
if (config.getIndexSettings().isSoftDeleteEnabled()) {
reader = new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
}
reader = readerWrapperFunction.apply(reader);
this.indexCommit = reader.getIndexCommit();
this.searcherManager = new SearcherManager(reader,
new RamAccountingSearcherFactory(engineConfig.getCircuitBreakerService()));
this.indexWriterLock = indexWriterLock;
success = true;
} finally {
if (success == false) {
IOUtils.close(reader, indexWriterLock, store::decRef);
}
}
} catch (IOException e) {
throw new UncheckedIOException(e); // this is stupid
}
}
@Override
protected void closeNoLock(String reason, CountDownLatch closedLatch) {
if (isClosed.compareAndSet(false, true)) {
try {
IOUtils.close(searcherManager, indexWriterLock, store::decRef);
} catch (Exception ex) {
logger.warn("failed to close searcher", ex);
} finally {
closedLatch.countDown();
}
}
}
public static SeqNoStats buildSeqNoStats(SegmentInfos infos) {
final SequenceNumbers.CommitInfo seqNoStats =
SequenceNumbers.loadSeqNoInfoFromLuceneCommit(infos.userData.entrySet());
long maxSeqNo = seqNoStats.maxSeqNo;
long localCheckpoint = seqNoStats.localCheckpoint;
return new SeqNoStats(maxSeqNo, localCheckpoint, localCheckpoint);
}
@Override
public GetResult get(Get get, BiFunction<String, SearcherScope, Searcher> searcherFactory) throws EngineException {
return getFromSearcher(get, searcherFactory, SearcherScope.EXTERNAL);
}
@Override
protected ReferenceManager<IndexSearcher> getReferenceManager(SearcherScope scope) {
return searcherManager;
}
@Override
protected SegmentInfos getLastCommittedSegmentInfos() {
return lastCommittedSegmentInfos;
}
@Override
public String getHistoryUUID() {
return lastCommittedSegmentInfos.userData.get(Engine.HISTORY_UUID_KEY);
}
@Override
public long getWritingBytes() {
return 0;
}
@Override
public long getIndexThrottleTimeInMillis() {
return 0;
}
@Override
public boolean isThrottled() {
return false;
}
@Override
public IndexResult index(Index index) {
assert false : "this should not be called";
throw new UnsupportedOperationException("indexing is not supported on a read-only engine");
}
@Override
public DeleteResult delete(Delete delete) {
assert false : "this should not be called";
throw new UnsupportedOperationException("deletes are not supported on a read-only engine");
}
@Override
public NoOpResult noOp(NoOp noOp) {
assert false : "this should not be called";
throw new UnsupportedOperationException("no-ops are not supported on a read-only engine");
}
@Override
public boolean isTranslogSyncNeeded() {
return false;
}
@Override
public boolean ensureTranslogSynced(Stream<Translog.Location> locations) {
return false;
}
@Override
public void syncTranslog() {
}
@Override
public Closeable acquireRetentionLockForPeerRecovery() {
return () -> {};
}
@Override
public Translog.Snapshot newChangesSnapshot(String source, MapperService mapperService, long fromSeqNo, long toSeqNo,
boolean requiredFullRange) throws IOException {
return readHistoryOperations(source, mapperService, fromSeqNo);
}
@Override
public Translog.Snapshot readHistoryOperations(String source, MapperService mapperService, long startingSeqNo) throws IOException {
return new Translog.Snapshot() {
@Override
public void close() { }
@Override
public int totalOperations() {
return 0;
}
@Override
public Translog.Operation next() {
return null;
}
};
}
@Override
public int estimateNumberOfHistoryOperations(String source, MapperService mapperService, long startingSeqNo) throws IOException {
return 0;
}
@Override
public boolean hasCompleteOperationHistory(String source, MapperService mapperService, long startingSeqNo) throws IOException {
return false;
}
@Override
public TranslogStats getTranslogStats() {
return translogStats;
}
@Override
public Translog.Location getTranslogLastWriteLocation() {
return new Translog.Location(0,0,0);
}
@Override
public long getLocalCheckpoint() {
return seqNoStats.getLocalCheckpoint();
}
@Override
public void waitForOpsToComplete(long seqNo) {
}
@Override
public SeqNoStats getSeqNoStats(long globalCheckpoint) {
return new SeqNoStats(seqNoStats.getMaxSeqNo(), seqNoStats.getLocalCheckpoint(), globalCheckpoint);
}
@Override
public long getLastSyncedGlobalCheckpoint() {
return seqNoStats.getGlobalCheckpoint();
}
@Override
public long getIndexBufferRAMBytesUsed() {
return 0;
}
@Override
public List<Segment> segments(boolean verbose) {
return Arrays.asList(getSegmentInfo(lastCommittedSegmentInfos, verbose));
}
@Override
public void refresh(String source) {
// we could allow refreshes if we want down the road the searcher manager will then reflect changes to a rw-engine
// opened side-by-side
}
@Override
public void writeIndexingBuffer() throws EngineException {
}
@Override
public boolean shouldPeriodicallyFlush() {
return false;
}
@Override
public SyncedFlushResult syncFlush(String syncId, CommitId expectedCommitId) {
// we can't do synced flushes this would require an indexWriter which we don't have
throw new UnsupportedOperationException("syncedFlush is not supported on a read-only engine");
}
@Override
public CommitId flush(boolean force, boolean waitIfOngoing) throws EngineException {
return new CommitId(lastCommittedSegmentInfos.getId());
}
@Override
public void forceMerge(boolean flush, int maxNumSegments, boolean onlyExpungeDeletes,
boolean upgrade, boolean upgradeOnlyAncientSegments) {
}
@Override
public IndexCommitRef acquireLastIndexCommit(boolean flushFirst) {
store.incRef();
return new IndexCommitRef(indexCommit, store::decRef);
}
@Override
public IndexCommitRef acquireSafeIndexCommit() {
return acquireLastIndexCommit(false);
}
@Override
public void activateThrottling() {
}
@Override
public void deactivateThrottling() {
}
@Override
public void trimUnreferencedTranslogFiles() {
}
@Override
public boolean shouldRollTranslogGeneration() {
return false;
}
@Override
public void rollTranslogGeneration() {
}
@Override
public void restoreLocalCheckpointFromTranslog() {
}
@Override
public int fillSeqNoGaps(long primaryTerm) {
return 0;
}
@Override
public Engine recoverFromTranslog(TranslogRecoveryRunner translogRecoveryRunner, long recoverUpToSeqNo) {
return this;
}
@Override
public void skipTranslogRecovery() {
}
@Override
public void trimOperationsFromTranslog(long belowTerm, long aboveSeqNo) {
}
@Override
public void maybePruneDeletes() {
}
}

14
server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
View File

@ -111,17 +111,6 @@ public abstract class MappedFieldType extends FieldType {
public boolean equals(Object o) {
if (!super.equals(o)) return false;
MappedFieldType fieldType = (MappedFieldType) o;
// check similarity first because we need to check the name, and it might be null
// TODO: SimilarityProvider should have equals?
if (similarity == null || fieldType.similarity == null) {
if (similarity != fieldType.similarity) {
return false;
}
} else {
if (Objects.equals(similarity.name(), fieldType.similarity.name()) == false) {
return false;
}
}
return boost == fieldType.boost &&
docValues == fieldType.docValues &&
@ -131,7 +120,8 @@ public abstract class MappedFieldType extends FieldType {
Objects.equals(searchQuoteAnalyzer(), fieldType.searchQuoteAnalyzer()) &&
Objects.equals(eagerGlobalOrdinals, fieldType.eagerGlobalOrdinals) &&
Objects.equals(nullValue, fieldType.nullValue) &&
Objects.equals(nullValueAsString, fieldType.nullValueAsString);
Objects.equals(nullValueAsString, fieldType.nullValueAsString) &&
Objects.equals(similarity, fieldType.similarity);
}
@Override

12
server/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilder.java
View File

@ -24,7 +24,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.lucene.search.function.ScoreFunction;
import org.elasticsearch.common.lucene.search.function.WeightFactorFunction;
import org.elasticsearch.common.xcontent.ToXContent.Params;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryShardContext;
@ -46,7 +45,7 @@ public abstract class ScoreFunctionBuilder<FB extends ScoreFunctionBuilder<FB>>
* Read from a stream.
*/
public ScoreFunctionBuilder(StreamInput in) throws IOException {
weight = in.readOptionalFloat();
weight = checkWeight(in.readOptionalFloat());
}
@Override
@ -70,10 +69,17 @@ public abstract class ScoreFunctionBuilder<FB extends ScoreFunctionBuilder<FB>>
*/
@SuppressWarnings("unchecked")
public final FB setWeight(float weight) {
this.weight = weight;
this.weight = checkWeight(weight);
return (FB) this;
}
private Float checkWeight(Float weight) {
if (weight != null && Float.compare(weight, 0) < 0) {
throw new IllegalArgumentException("[weight] cannot be negative for a filtering function");
}
return weight;
}
/**
* The weight applied to the function before combining.
*/

1
server/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java
View File

@ -109,6 +109,7 @@ public class LocalCheckpointTracker {
* @param checkpoint the local checkpoint to reset this tracker to
*/
public synchronized void resetCheckpoint(final long checkpoint) {
// TODO: remove this method as after we restore the local history on promotion.
assert checkpoint != SequenceNumbers.UNASSIGNED_SEQ_NO;
assert checkpoint <= this.checkpoint;
processedSeqNo.clear();

1
server/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java
View File

@ -91,5 +91,4 @@ public class SeqNoStats implements ToXContentFragment, Writeable {
", globalCheckpoint=" + globalCheckpoint +
'}';
}
}

2
server/src/main/java/org/elasticsearch/index/shard/AbstractIndexShardComponent.java
View File

@ -51,6 +51,4 @@ public abstract class AbstractIndexShardComponent implements IndexShardComponent
public String nodeName() {
return indexSettings.getNodeName();
}
}

114
server/src/main/java/org/elasticsearch/index/shard/GlobalCheckpointListeners.java
View File

@ -21,13 +21,19 @@ package org.elasticsearch.index.shard;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.FutureUtils;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED;
import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
@ -45,38 +51,43 @@ public class GlobalCheckpointListeners implements Closeable {
public interface GlobalCheckpointListener {
/**
* Callback when the global checkpoint is updated or the shard is closed. If the shard is closed, the value of the global checkpoint
* will be set to {@link org.elasticsearch.index.seqno.SequenceNumbers#UNASSIGNED_SEQ_NO} and the exception will be non-null. If the
* global checkpoint is updated, the exception will be null.
* will be set to {@link org.elasticsearch.index.seqno.SequenceNumbers#UNASSIGNED_SEQ_NO} and the exception will be non-null and an
* instance of {@link IndexShardClosedException }. If the listener timed out waiting for notification then the exception will be
* non-null and an instance of {@link TimeoutException}. If the global checkpoint is updated, the exception will be null.
*
* @param globalCheckpoint the updated global checkpoint
* @param e if non-null, the shard is closed
* @param e if non-null, the shard is closed or the listener timed out
*/
void accept(long globalCheckpoint, IndexShardClosedException e);
void accept(long globalCheckpoint, Exception e);
}
// guarded by this
private boolean closed;
private volatile List<GlobalCheckpointListener> listeners;
private Map<GlobalCheckpointListener, ScheduledFuture<?>> listeners;
private long lastKnownGlobalCheckpoint = UNASSIGNED_SEQ_NO;
private final ShardId shardId;
private final Executor executor;
private final ScheduledExecutorService scheduler;
private final Logger logger;
/**
* Construct a global checkpoint listeners collection.
*
* @param shardId the shard ID on which global checkpoint updates can be listened to
* @param executor the executor for listener notifications
* @param logger a shard-level logger
* @param shardId the shard ID on which global checkpoint updates can be listened to
* @param executor the executor for listener notifications
* @param scheduler the executor used for scheduling timeouts
* @param logger a shard-level logger
*/
GlobalCheckpointListeners(
final ShardId shardId,
final Executor executor,
final ScheduledExecutorService scheduler,
final Logger logger) {
this.shardId = Objects.requireNonNull(shardId);
this.executor = Objects.requireNonNull(executor);
this.logger = Objects.requireNonNull(logger);
this.shardId = Objects.requireNonNull(shardId, "shardId");
this.executor = Objects.requireNonNull(executor, "executor");
this.scheduler = Objects.requireNonNull(scheduler, "scheduler");
this.logger = Objects.requireNonNull(logger, "logger");
}
/**
@ -84,12 +95,15 @@ public class GlobalCheckpointListeners implements Closeable {
* listener will be asynchronously notified on the executor used to construct this collection of global checkpoint listeners. If the
* shard is closed then the listener will be asynchronously notified on the executor used to construct this collection of global
* checkpoint listeners. The listener will only be notified of at most one event, either the global checkpoint is updated or the shard
* is closed. A listener must re-register after one of these events to receive subsequent events.
* is closed. A listener must re-register after one of these events to receive subsequent events. Callers may add a timeout to be
* notified after if the timeout elapses. In this case, the listener will be notified with a {@link TimeoutException}. Passing null for
* the timeout means no timeout will be associated to the listener.
*
* @param currentGlobalCheckpoint the current global checkpoint known to the listener
* @param listener the listener
* @param timeout the listener timeout, or null if no timeout
*/
synchronized void add(final long currentGlobalCheckpoint, final GlobalCheckpointListener listener) {
synchronized void add(final long currentGlobalCheckpoint, final GlobalCheckpointListener listener, final TimeValue timeout) {
if (closed) {
executor.execute(() -> notifyListener(listener, UNASSIGNED_SEQ_NO, new IndexShardClosedException(shardId)));
return;
@ -97,12 +111,43 @@ public class GlobalCheckpointListeners implements Closeable {
if (lastKnownGlobalCheckpoint > currentGlobalCheckpoint) {
// notify directly
executor.execute(() -> notifyListener(listener, lastKnownGlobalCheckpoint, null));
return;
} else {
if (listeners == null) {
listeners = new ArrayList<>();
listeners = new LinkedHashMap<>();
}
if (timeout == null) {
listeners.put(listener, null);
} else {
listeners.put(
listener,
scheduler.schedule(
() -> {
final boolean removed;
synchronized (this) {
/*
* Note that the listeners map can be null if a notification nulled out the map reference when
* notifying listeners, and then our scheduled execution occurred before we could be cancelled by
* the notification. In this case, we would have blocked waiting for access to this critical
* section.
*
* What is more, we know that this listener has a timeout associated with it (otherwise we would
* not be here) so the return value from remove being null is an indication that we are not in the
* map. This can happen if a notification nulled out the listeners, and then our scheduled execution
* occurred before we could be cancelled by the notification, and then another thread added a
* listener causing the listeners map reference to be non-null again. In this case, our listener
* here would not be in the map and we should not fire the timeout logic.
*/
removed = listeners != null && listeners.remove(listener) != null;
}
if (removed) {
final TimeoutException e = new TimeoutException(timeout.getStringRep());
logger.trace("global checkpoint listener timed out", e);
executor.execute(() -> notifyListener(listener, UNASSIGNED_SEQ_NO, e));
}
},
timeout.nanos(),
TimeUnit.NANOSECONDS));
}
listeners.add(listener);
}
}
@ -112,10 +157,25 @@ public class GlobalCheckpointListeners implements Closeable {
notifyListeners(UNASSIGNED_SEQ_NO, new IndexShardClosedException(shardId));
}
/**
* The number of listeners currently pending for notification.
*
* @return the number of listeners pending notification
*/
synchronized int pendingListeners() {
return listeners == null ? 0 : listeners.size();
}
/**
* The scheduled future for a listener that has a timeout associated with it, otherwise null.
*
* @param listener the listener to get the scheduled future for
* @return a scheduled future representing the timeout future for the listener, otherwise null
*/
synchronized ScheduledFuture<?> getTimeoutFuture(final GlobalCheckpointListener listener) {
return listeners.get(listener);
}
/**
* Invoke to notify all registered listeners of an updated global checkpoint.
*
@ -135,19 +195,24 @@ public class GlobalCheckpointListeners implements Closeable {
assert (globalCheckpoint == UNASSIGNED_SEQ_NO && e != null) || (globalCheckpoint >= NO_OPS_PERFORMED && e == null);
if (listeners != null) {
// capture the current listeners
final List<GlobalCheckpointListener> currentListeners = listeners;
final Map<GlobalCheckpointListener, ScheduledFuture<?>> currentListeners = listeners;
listeners = null;
if (currentListeners != null) {
executor.execute(() -> {
for (final GlobalCheckpointListener listener : currentListeners) {
notifyListener(listener, globalCheckpoint, e);
for (final Map.Entry<GlobalCheckpointListener, ScheduledFuture<?>> listener : currentListeners.entrySet()) {
/*
* We do not want to interrupt any timeouts that fired, these will detect that the listener has been notified and
* not trigger the timeout.
*/
FutureUtils.cancel(listener.getValue());
notifyListener(listener.getKey(), globalCheckpoint, e);
}
});
}
}
}
private void notifyListener(final GlobalCheckpointListener listener, final long globalCheckpoint, final IndexShardClosedException e) {
private void notifyListener(final GlobalCheckpointListener listener, final long globalCheckpoint, final Exception e) {
try {
listener.accept(globalCheckpoint, e);
} catch (final Exception caught) {
@ -157,8 +222,11 @@ public class GlobalCheckpointListeners implements Closeable {
"error notifying global checkpoint listener of updated global checkpoint [{}]",
globalCheckpoint),
caught);
} else {
} else if (e instanceof IndexShardClosedException) {
logger.warn("error notifying global checkpoint listener of closed shard", caught);
} else {
assert e instanceof TimeoutException : e;
logger.warn("error notifying global checkpoint listener of timeout", caught);
}
}
}

96
server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
View File

@ -163,7 +163,6 @@ import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import static org.elasticsearch.index.mapper.SourceToParse.source;
import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED;
import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
public class IndexShard extends AbstractIndexShardComponent implements IndicesClusterStateService.Shard {
@ -303,7 +302,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
this.checkIndexOnStartup = indexSettings.getValue(IndexSettings.INDEX_CHECK_ON_STARTUP);
this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, bigArrays);
final String aId = shardRouting.allocationId().getId();
this.globalCheckpointListeners = new GlobalCheckpointListeners(shardId, threadPool.executor(ThreadPool.Names.LISTENER), logger);
this.globalCheckpointListeners =
new GlobalCheckpointListeners(shardId, threadPool.executor(ThreadPool.Names.LISTENER), threadPool.scheduler(), logger);
this.replicationTracker =
new ReplicationTracker(shardId, aId, indexSettings, UNASSIGNED_SEQ_NO, globalCheckpointListeners::globalCheckpointUpdated);
@ -1273,16 +1273,18 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
return result;
}
// package-private for testing
int runTranslogRecovery(Engine engine, Translog.Snapshot snapshot) throws IOException {
recoveryState.getTranslog().totalOperations(snapshot.totalOperations());
recoveryState.getTranslog().totalOperationsOnStart(snapshot.totalOperations());
/**
* Replays translog operations from the provided translog {@code snapshot} to the current engine using the given {@code origin}.
* The callback {@code onOperationRecovered} is notified after each translog operation is replayed successfully.
*/
int runTranslogRecovery(Engine engine, Translog.Snapshot snapshot, Engine.Operation.Origin origin,
Runnable onOperationRecovered) throws IOException {
int opsRecovered = 0;
Translog.Operation operation;
while ((operation = snapshot.next()) != null) {
try {
logger.trace("[translog] recover op {}", operation);
Engine.Result result = applyTranslogOperation(operation, Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY);
Engine.Result result = applyTranslogOperation(operation, origin);
switch (result.getResultType()) {
case FAILURE:
throw result.getFailure();
@ -1295,7 +1297,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
}
opsRecovered++;
recoveryState.getTranslog().incrementRecoveredOperations();
onOperationRecovered.run();
} catch (Exception e) {
if (ExceptionsHelper.status(e) == RestStatus.BAD_REQUEST) {
// mainly for MapperParsingException and Failure to detect xcontent
@ -1313,8 +1315,15 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
* Operations from the translog will be replayed to bring lucene up to date.
**/
public void openEngineAndRecoverFromTranslog() throws IOException {
final RecoveryState.Translog translogRecoveryStats = recoveryState.getTranslog();
final Engine.TranslogRecoveryRunner translogRecoveryRunner = (engine, snapshot) -> {
translogRecoveryStats.totalOperations(snapshot.totalOperations());
translogRecoveryStats.totalOperationsOnStart(snapshot.totalOperations());
return runTranslogRecovery(engine, snapshot, Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY,
translogRecoveryStats::incrementRecoveredOperations);
};
innerOpenEngineAndTranslog();
getEngine().recoverFromTranslog(this::runTranslogRecovery, Long.MAX_VALUE);
getEngine().recoverFromTranslog(translogRecoveryRunner, Long.MAX_VALUE);
}
/**
@ -1352,11 +1361,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
final String translogUUID = store.readLastCommittedSegmentsInfo().getUserData().get(Translog.TRANSLOG_UUID_KEY);
final long globalCheckpoint = Translog.readGlobalCheckpoint(translogConfig.getTranslogPath(), translogUUID);
replicationTracker.updateGlobalCheckpointOnReplica(globalCheckpoint, "read from translog checkpoint");
assertMaxUnsafeAutoIdInCommit();
final long minRetainedTranslogGen = Translog.readMinTranslogGeneration(translogConfig.getTranslogPath(), translogUUID);
store.trimUnsafeCommits(globalCheckpoint, minRetainedTranslogGen, config.getIndexSettings().getIndexVersionCreated());
trimUnsafeCommits();
createNewEngine(config);
verifyNotClosed();
@ -1367,6 +1372,15 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
assert recoveryState.getStage() == RecoveryState.Stage.TRANSLOG : "TRANSLOG stage expected but was: " + recoveryState.getStage();
}
private void trimUnsafeCommits() throws IOException {
assert currentEngineReference.get() == null : "engine is running";
final String translogUUID = store.readLastCommittedSegmentsInfo().getUserData().get(Translog.TRANSLOG_UUID_KEY);
final long globalCheckpoint = Translog.readGlobalCheckpoint(translogConfig.getTranslogPath(), translogUUID);
final long minRetainedTranslogGen = Translog.readMinTranslogGeneration(translogConfig.getTranslogPath(), translogUUID);
assertMaxUnsafeAutoIdInCommit();
store.trimUnsafeCommits(globalCheckpoint, minRetainedTranslogGen, indexSettings.getIndexVersionCreated());
}
private boolean assertSequenceNumbersInCommit() throws IOException {
final Map<String, String> userData = SegmentInfos.readLatestCommit(store.directory()).getUserData();
assert userData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) : "commit point doesn't contains a local checkpoint";
@ -1463,7 +1477,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
if (origin == Engine.Operation.Origin.PRIMARY) {
assert assertPrimaryMode();
} else {
assert origin == Engine.Operation.Origin.REPLICA;
assert origin == Engine.Operation.Origin.REPLICA || origin == Engine.Operation.Origin.LOCAL_RESET;
assert assertReplicationTarget();
}
if (writeAllowedStates.contains(state) == false) {
@ -1768,15 +1782,18 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
/**
* Add a global checkpoint listener. If the global checkpoint is above the current global checkpoint known to the listener then the
* listener will fire immediately on the calling thread.
* listener will fire immediately on the calling thread. If the specified timeout elapses before the listener is notified, the listener
* will be notified with an {@link TimeoutException}. A caller may pass null to specify no timeout.
*
* @param currentGlobalCheckpoint the current global checkpoint known to the listener
* @param listener the listener
* @param timeout the timeout
*/
public void addGlobalCheckpointListener(
final long currentGlobalCheckpoint,
final GlobalCheckpointListeners.GlobalCheckpointListener listener) {
this.globalCheckpointListeners.add(currentGlobalCheckpoint, listener);
final GlobalCheckpointListeners.GlobalCheckpointListener listener,
final TimeValue timeout) {
this.globalCheckpointListeners.add(currentGlobalCheckpoint, listener, timeout);
}
/**
@ -2166,9 +2183,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
private Engine createNewEngine(EngineConfig config) {
synchronized (mutex) {
if (state == IndexShardState.CLOSED) {
throw new AlreadyClosedException(shardId + " can't create engine - shard is closed");
}
verifyNotClosed();
assert this.currentEngineReference.get() == null;
Engine engine = newEngine(config);
onNewEngine(engine); // call this before we pass the memory barrier otherwise actions that happen
@ -2314,19 +2329,14 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
bumpPrimaryTerm(opPrimaryTerm, () -> {
updateGlobalCheckpointOnReplica(globalCheckpoint, "primary term transition");
final long currentGlobalCheckpoint = getGlobalCheckpoint();
final long localCheckpoint;
if (currentGlobalCheckpoint == UNASSIGNED_SEQ_NO) {
localCheckpoint = NO_OPS_PERFORMED;
final long maxSeqNo = seqNoStats().getMaxSeqNo();
logger.info("detected new primary with primary term [{}], global checkpoint [{}], max_seq_no [{}]",
opPrimaryTerm, currentGlobalCheckpoint, maxSeqNo);
if (currentGlobalCheckpoint < maxSeqNo) {
resetEngineToGlobalCheckpoint();
} else {
localCheckpoint = currentGlobalCheckpoint;
getEngine().rollTranslogGeneration();
}
logger.trace(
"detected new primary with primary term [{}], resetting local checkpoint from [{}] to [{}]",
opPrimaryTerm,
getLocalCheckpoint(),
localCheckpoint);
getEngine().resetLocalCheckpoint(localCheckpoint);
getEngine().rollTranslogGeneration();
});
}
}
@ -2687,4 +2697,26 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
}
};
}
/**
* Rollback the current engine to the safe commit, then replay local translog up to the global checkpoint.
*/
void resetEngineToGlobalCheckpoint() throws IOException {
assert getActiveOperationsCount() == 0 : "Ongoing writes [" + getActiveOperations() + "]";
sync(); // persist the global checkpoint to disk
final long globalCheckpoint = getGlobalCheckpoint();
final Engine newEngine;
synchronized (mutex) {
verifyNotClosed();
IOUtils.close(currentEngineReference.getAndSet(null));
trimUnsafeCommits();
newEngine = createNewEngine(newEngineConfig());
active.set(true);
}
final Engine.TranslogRecoveryRunner translogRunner = (engine, snapshot) -> runTranslogRecovery(
engine, snapshot, Engine.Operation.Origin.LOCAL_RESET, () -> {
// TODO: add a dedicate recovery stats for the reset translog
});
newEngine.recoverFromTranslog(translogRunner, globalCheckpoint);
}
}

26
server/src/main/java/org/elasticsearch/index/similarity/SimilarityProvider.java
View File

@ -21,6 +21,8 @@ package org.elasticsearch.index.similarity;
import org.apache.lucene.search.similarities.Similarity;
import java.util.Objects;
/**
* Wrapper around a {@link Similarity} and its name.
*/
@ -48,4 +50,28 @@ public final class SimilarityProvider {
return similarity;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
SimilarityProvider that = (SimilarityProvider) o;
/**
* We check <code>name</code> only because the <code>similarity</code> is
* re-created for each new instance and they don't implement equals.
* This is not entirely correct though but we only use equality checks
* for similarities inside the same index and names are unique in this case.
**/
return Objects.equals(name, that.name);
}
@Override
public int hashCode() {
/**
* We use <code>name</code> only because the <code>similarity</code> is
* re-created for each new instance and they don't implement equals.
* This is not entirely correct though but we only use equality checks
* for similarities a single index and names are unique in this case.
**/
return Objects.hash(name);
}
}

21
server/src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -1439,11 +1439,28 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
*/
public void bootstrapNewHistory() throws IOException {
metadataLock.writeLock().lock();
try (IndexWriter writer = newIndexWriter(IndexWriterConfig.OpenMode.APPEND, directory, null)) {
final Map<String, String> userData = getUserData(writer);
try {
Map<String, String> userData = readLastCommittedSegmentsInfo().getUserData();
final long maxSeqNo = Long.parseLong(userData.get(SequenceNumbers.MAX_SEQ_NO));
bootstrapNewHistory(maxSeqNo);
} finally {
metadataLock.writeLock().unlock();
}
}
/**
* Marks an existing lucene index with a new history uuid and sets the given maxSeqNo as the local checkpoint
* as well as the maximum sequence number.
* This is used to make sure no existing shard will recovery from this index using ops based recovery.
* @see SequenceNumbers#LOCAL_CHECKPOINT_KEY
* @see SequenceNumbers#MAX_SEQ_NO
*/
public void bootstrapNewHistory(long maxSeqNo) throws IOException {
metadataLock.writeLock().lock();
try (IndexWriter writer = newIndexWriter(IndexWriterConfig.OpenMode.APPEND, directory, null)) {
final Map<String, String> map = new HashMap<>();
map.put(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID());
map.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo));
map.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(maxSeqNo));
updateCommitData(writer, map);
} finally {

1
server/src/main/java/org/elasticsearch/indices/IndicesService.java
View File

@ -396,7 +396,6 @@ public class IndicesService extends AbstractLifecycleComponent
public IndexService indexService(Index index) {
return indices.get(index.getUUID());
}
/**
* Returns an IndexService for the specified index if exists otherwise a {@link IndexNotFoundException} is thrown.
*/

167
server/src/main/java/org/elasticsearch/repositories/FilterRepository.java Normal file
View File

@ -0,0 +1,167 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories;
import org.apache.lucene.index.IndexCommit;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.component.Lifecycle;
import org.elasticsearch.common.component.LifecycleListener;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.snapshots.SnapshotId;
import org.elasticsearch.snapshots.SnapshotInfo;
import org.elasticsearch.snapshots.SnapshotShardFailure;
import java.io.IOException;
import java.util.List;
public class FilterRepository implements Repository {
private final Repository in;
public FilterRepository(Repository in) {
this.in = in;
}
@Override
public RepositoryMetaData getMetadata() {
return in.getMetadata();
}
@Override
public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) {
return in.getSnapshotInfo(snapshotId);
}
@Override
public MetaData getSnapshotGlobalMetaData(SnapshotId snapshotId) {
return in.getSnapshotGlobalMetaData(snapshotId);
}
@Override
public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId index) throws IOException {
return in.getSnapshotIndexMetaData(snapshotId, index);
}
@Override
public RepositoryData getRepositoryData() {
return in.getRepositoryData();
}
@Override
public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, MetaData metaData) {
in.initializeSnapshot(snapshotId, indices, metaData);
}
@Override
public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards,
List<SnapshotShardFailure> shardFailures, long repositoryStateId, boolean includeGlobalState) {
return in.finalizeSnapshot(snapshotId, indices, startTime, failure, totalShards, shardFailures, repositoryStateId,
includeGlobalState);
}
@Override
public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId) {
in.deleteSnapshot(snapshotId, repositoryStateId);
}
@Override
public long getSnapshotThrottleTimeInNanos() {
return in.getSnapshotThrottleTimeInNanos();
}
@Override
public long getRestoreThrottleTimeInNanos() {
return in.getRestoreThrottleTimeInNanos();
}
@Override
public String startVerification() {
return in.startVerification();
}
@Override
public void endVerification(String verificationToken) {
in.endVerification(verificationToken);
}
@Override
public void verify(String verificationToken, DiscoveryNode localNode) {
in.verify(verificationToken, localNode);
}
@Override
public boolean isReadOnly() {
return in.isReadOnly();
}
@Override
public void snapshotShard(IndexShard shard, Store store, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit,
IndexShardSnapshotStatus snapshotStatus) {
in.snapshotShard(shard, store, snapshotId, indexId, snapshotIndexCommit, snapshotStatus);
}
@Override
public void restoreShard(IndexShard shard, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId,
RecoveryState recoveryState) {
in.restoreShard(shard, snapshotId, version, indexId, snapshotShardId, recoveryState);
}
@Override
public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) {
return in.getShardSnapshotStatus(snapshotId, version, indexId, shardId);
}
@Override
public Lifecycle.State lifecycleState() {
return in.lifecycleState();
}
@Override
public void addLifecycleListener(LifecycleListener listener) {
in.addLifecycleListener(listener);
}
@Override
public void removeLifecycleListener(LifecycleListener listener) {
in.removeLifecycleListener(listener);
}
@Override
public void start() {
in.start();
}
@Override
public void stop() {
in.stop();
}
@Override
public void close() {
in.close();
}
}

2
server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
View File

@ -398,7 +398,7 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
"repository type [" + repositoryMetaData.type() + "] does not exist");
}
try {
Repository repository = factory.create(repositoryMetaData);
Repository repository = factory.create(repositoryMetaData, typesRegistry::get);
repository.start();
return repository;
} catch (Exception e) {

13
server/src/main/java/org/elasticsearch/repositories/Repository.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.snapshots.SnapshotId;
import org.elasticsearch.snapshots.SnapshotInfo;
@ -35,6 +36,7 @@ import org.elasticsearch.snapshots.SnapshotShardFailure;
import java.io.IOException;
import java.util.List;
import java.util.function.Function;
/**
* An interface for interacting with a repository in snapshot and restore.
@ -46,7 +48,7 @@ import java.util.List;
* <ul>
* <li>Master calls {@link #initializeSnapshot(SnapshotId, List, org.elasticsearch.cluster.metadata.MetaData)}
* with list of indices that will be included into the snapshot</li>
* <li>Data nodes call {@link Repository#snapshotShard(IndexShard, SnapshotId, IndexId, IndexCommit, IndexShardSnapshotStatus)}
* <li>Data nodes call {@link Repository#snapshotShard(IndexShard, Store, SnapshotId, IndexId, IndexCommit, IndexShardSnapshotStatus)}
* for each shard</li>
* <li>When all shard calls return master calls {@link #finalizeSnapshot} with possible list of failures</li>
* </ul>
@ -63,6 +65,10 @@ public interface Repository extends LifecycleComponent {
* @param metadata metadata for the repository including name and settings
*/
Repository create(RepositoryMetaData metadata) throws Exception;
default Repository create(RepositoryMetaData metaData, Function<String, Repository.Factory> typeLookup) throws Exception {
return create(metaData);
}
}
/**
@ -188,14 +194,15 @@ public interface Repository extends LifecycleComponent {
* <p>
* As snapshot process progresses, implementation of this method should update {@link IndexShardSnapshotStatus} object and check
* {@link IndexShardSnapshotStatus#isAborted()} to see if the snapshot process should be aborted.
*
* @param shard shard to be snapshotted
* @param store store to be snapshotted
* @param snapshotId snapshot id
* @param indexId id for the index being snapshotted
* @param snapshotIndexCommit commit point
* @param snapshotStatus snapshot status
*/
void snapshotShard(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus);
void snapshotShard(IndexShard shard, Store store, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit,
IndexShardSnapshotStatus snapshotStatus);
/**
* Restores snapshot of the shard.

15
server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View File

@ -845,8 +845,9 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
}
@Override
public void snapshotShard(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {
SnapshotContext snapshotContext = new SnapshotContext(shard, snapshotId, indexId, snapshotStatus, System.currentTimeMillis());
public void snapshotShard(IndexShard shard, Store store, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit,
IndexShardSnapshotStatus snapshotStatus) {
SnapshotContext snapshotContext = new SnapshotContext(store, snapshotId, indexId, snapshotStatus, System.currentTimeMillis());
try {
snapshotContext.snapshot(snapshotIndexCommit);
} catch (Exception e) {
@ -854,7 +855,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
if (e instanceof IndexShardSnapshotFailedException) {
throw (IndexShardSnapshotFailedException) e;
} else {
throw new IndexShardSnapshotFailedException(shard.shardId(), e);
throw new IndexShardSnapshotFailedException(store.shardId(), e);
}
}
}
@ -1157,15 +1158,15 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
/**
* Constructs new context
*
* @param shard shard to be snapshotted
* @param store store to be snapshotted
* @param snapshotId snapshot id
* @param indexId the id of the index being snapshotted
* @param snapshotStatus snapshot status to report progress
*/
SnapshotContext(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexShardSnapshotStatus snapshotStatus, long startTime) {
super(snapshotId, Version.CURRENT, indexId, shard.shardId());
SnapshotContext(Store store, SnapshotId snapshotId, IndexId indexId, IndexShardSnapshotStatus snapshotStatus, long startTime) {
super(snapshotId, Version.CURRENT, indexId, store.shardId());
this.snapshotStatus = snapshotStatus;
this.store = shard.store();
this.store = store;
this.startTime = startTime;
}

14
server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
View File

@ -46,6 +46,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* Query sub phase which pulls data from doc values
@ -77,6 +78,15 @@ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase {
hits = hits.clone(); // don't modify the incoming hits
Arrays.sort(hits, Comparator.comparingInt(SearchHit::docId));
List<String> noFormatFields = context.docValueFieldsContext().fields().stream().filter(f -> f.format == null).map(f -> f.field)
.collect(Collectors.toList());
if (noFormatFields.isEmpty() == false) {
DEPRECATION_LOGGER.deprecated("There are doc-value fields which are not using a format. The output will "
+ "change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass "
+ "[format={}] with a doc value field in order to opt in for the future behaviour and ease the migration to "
+ "7.0: {}", DocValueFieldsContext.USE_DEFAULT_FORMAT, noFormatFields);
}
for (FieldAndFormat fieldAndFormat : context.docValueFieldsContext().fields()) {
String field = fieldAndFormat.field;
MappedFieldType fieldType = context.mapperService().fullName(field);
@ -84,10 +94,6 @@ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase {
final IndexFieldData<?> indexFieldData = context.getForField(fieldType);
final DocValueFormat format;
if (fieldAndFormat.format == null) {
DEPRECATION_LOGGER.deprecated("Doc-value field [" + fieldAndFormat.field + "] is not using a format. The output will " +
"change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass " +
"[format={}] with the doc value field in order to opt in for the future behaviour and ease the migration to " +
"7.0.", DocValueFieldsContext.USE_DEFAULT_FORMAT);
format = null;
} else {
String formatDesc = fieldAndFormat.format;

3
server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
View File

@ -389,7 +389,8 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
try {
// we flush first to make sure we get the latest writes snapshotted
try (Engine.IndexCommitRef snapshotRef = indexShard.acquireLastIndexCommit(true)) {
repository.snapshotShard(indexShard, snapshot.getSnapshotId(), indexId, snapshotRef.getIndexCommit(), snapshotStatus);
repository.snapshotShard(indexShard, indexShard.store(), snapshot.getSnapshotId(), indexId, snapshotRef.getIndexCommit(),
snapshotStatus);
if (logger.isDebugEnabled()) {
final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy();
logger.debug("snapshot ({}) completed to {} with {}", snapshot, repository, lastSnapshotStatus);

34
server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.SettingUpgrader;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
@ -51,6 +52,11 @@ import java.util.stream.Stream;
*/
public abstract class RemoteClusterAware extends AbstractComponent {
static {
// remove search.remote.* settings in 8.0.0
assert Version.CURRENT.major < 8;
}
public static final Setting.AffixSetting<List<String>> SEARCH_REMOTE_CLUSTERS_SEEDS =
Setting.affixKeySetting(
"search.remote.",
@ -66,6 +72,20 @@ public abstract class RemoteClusterAware extends AbstractComponent {
Setting.Property.Dynamic,
Setting.Property.NodeScope));
public static final SettingUpgrader<List<String>> SEARCH_REMOTE_CLUSTER_SEEDS_UPGRADER = new SettingUpgrader<List<String>>() {
@Override
public Setting<List<String>> getSetting() {
return SEARCH_REMOTE_CLUSTERS_SEEDS;
}
@Override
public String getKey(final String key) {
return key.replaceFirst("^search", "cluster");
}
};
/**
* A list of initial seed nodes to discover eligible nodes from the remote cluster
*/
@ -105,6 +125,20 @@ public abstract class RemoteClusterAware extends AbstractComponent {
Setting.Property.NodeScope),
REMOTE_CLUSTERS_SEEDS);
public static final SettingUpgrader<String> SEARCH_REMOTE_CLUSTERS_PROXY_UPGRADER = new SettingUpgrader<String>() {
@Override
public Setting<String> getSetting() {
return SEARCH_REMOTE_CLUSTERS_PROXY;
}
@Override
public String getKey(final String key) {
return key.replaceFirst("^search", "cluster");
}
};
/**
* A proxy address for the remote cluster.
* NOTE: this settings is undocumented until we have at last one transport that supports passing

24
server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
View File

@ -19,8 +19,6 @@
package org.elasticsearch.transport;
import java.util.Collection;
import java.util.function.Supplier;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.OriginalIndices;
@ -35,6 +33,7 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.SettingUpgrader;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.CountDown;
@ -43,6 +42,7 @@ import org.elasticsearch.threadpool.ThreadPool;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -55,6 +55,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -65,6 +66,11 @@ import static org.elasticsearch.common.settings.Setting.boolSetting;
*/
public final class RemoteClusterService extends RemoteClusterAware implements Closeable {
static {
// remove search.remote.* settings in 8.0.0
assert Version.CURRENT.major < 8;
}
public static final Setting<Integer> SEARCH_REMOTE_CONNECTIONS_PER_CLUSTER =
Setting.intSetting("search.remote.connections_per_cluster", 3, 1, Setting.Property.NodeScope, Setting.Property.Deprecated);
@ -132,6 +138,20 @@ public final class RemoteClusterService extends RemoteClusterAware implements Cl
key -> boolSetting(key, false, Setting.Property.Deprecated, Setting.Property.Dynamic, Setting.Property.NodeScope),
REMOTE_CLUSTERS_SEEDS);
public static final SettingUpgrader<Boolean> SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE_UPGRADER = new SettingUpgrader<Boolean>() {
@Override
public Setting<Boolean> getSetting() {
return SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE;
}
@Override
public String getKey(final String key) {
return key.replaceFirst("^search", "cluster");
}
};
public static final Setting.AffixSetting<Boolean> REMOTE_CLUSTER_SKIP_UNAVAILABLE =
Setting.affixKeySetting(
"cluster.remote.",

11
server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
View File

@ -47,6 +47,7 @@ import java.util.Collections;
import java.util.HashSet;
import static java.util.Collections.emptyMap;
import static org.hamcrest.Matchers.equalTo;
public class TransportResizeActionTests extends ESTestCase {
@ -92,6 +93,16 @@ public class TransportResizeActionTests extends ESTestCase {
).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards "));
IllegalArgumentException softDeletesError = expectThrows(IllegalArgumentException.class, () -> {
ResizeRequest req = new ResizeRequest("target", "source");
req.getTargetIndexRequest().settings(Settings.builder().put("index.soft_deletes.enabled", false));
ClusterState clusterState = createClusterState("source", 8, 1,
Settings.builder().put("index.blocks.write", true).put("index.soft_deletes.enabled", true).build());
TransportResizeAction.prepareCreateIndexRequest(req, clusterState,
(i) -> new DocsStats(between(10, 1000), between(1, 10), between(1, 10000)), "source", "target");
});
assertThat(softDeletesError.getMessage(), equalTo("Can't disable [index.soft_deletes.enabled] setting on resize"));
// create one that won't fail
ClusterState clusterState = ClusterState.builder(createClusterState("source", randomIntBetween(2, 10), 0,
Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1")))

7
server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
View File

@ -53,7 +53,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import static java.util.Collections.emptySet;
@ -113,6 +112,7 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
// manual collection or upon cluster forming.
.put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2)
.put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), "1s")
@ -121,8 +121,7 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(TestPlugin.class,
MockTransportService.TestPlugin.class);
return Arrays.asList(TestPlugin.class, MockTransportService.TestPlugin.class);
}
public void testClusterInfoServiceCollectsInformation() throws Exception {
@ -172,7 +171,7 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
}
}
public void testClusterInfoServiceInformationClearOnError() throws InterruptedException, ExecutionException {
public void testClusterInfoServiceInformationClearOnError() {
internalCluster().startNodes(2,
// manually control publishing
Settings.builder().put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(), "60m").build());

11
server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
View File

@ -261,6 +261,7 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
.put("index.version.upgraded", upgraded)
.put("index.similarity.default.type", "BM25")
.put("index.analysis.analyzer.default.tokenizer", "keyword")
.put("index.soft_deletes.enabled", "true")
.build();
runPrepareResizeIndexSettingsTest(
indexSettings,
@ -277,6 +278,7 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
assertThat(settings.get("index.allocation.max_retries"), equalTo("1"));
assertThat(settings.getAsVersion("index.version.created", null), equalTo(version));
assertThat(settings.getAsVersion("index.version.upgraded", null), equalTo(upgraded));
assertThat(settings.get("index.soft_deletes.enabled"), equalTo("true"));
});
}
@ -337,6 +339,15 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
}
public void testDoNotOverrideSoftDeletesSettingOnResize() {
runPrepareResizeIndexSettingsTest(
Settings.builder().put("index.soft_deletes.enabled", "false").build(),
Settings.builder().put("index.soft_deletes.enabled", "true").build(),
Collections.emptyList(),
randomBoolean(),
settings -> assertThat(settings.get("index.soft_deletes.enabled"), equalTo("true")));
}
private void runPrepareResizeIndexSettingsTest(
final Settings sourceSettings,
final Settings requestSettings,

44
server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
View File

@ -47,6 +47,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
@ -1171,4 +1172,47 @@ public class ScopedSettingsTests extends ESTestCase {
}
}
public void testUpgradeListSetting() {
final Setting<List<String>> oldSetting =
Setting.listSetting("foo.old", Collections.emptyList(), Function.identity(), Property.NodeScope);
final Setting<List<String>> newSetting =
Setting.listSetting("foo.new", Collections.emptyList(), Function.identity(), Property.NodeScope);
final AbstractScopedSettings service =
new ClusterSettings(
Settings.EMPTY,
new HashSet<>(Arrays.asList(oldSetting, newSetting)),
Collections.singleton(new SettingUpgrader<List<String>>() {
@Override
public Setting<List<String>> getSetting() {
return oldSetting;
}
@Override
public String getKey(final String key) {
return "foo.new";
}
@Override
public List<String> getListValue(final List<String> value) {
return value.stream().map(s -> "new." + s).collect(Collectors.toList());
}
}));
final int length = randomIntBetween(0, 16);
final List<String> values = length == 0 ? Collections.emptyList() : new ArrayList<>(length);
for (int i = 0; i < length; i++) {
values.add(randomAlphaOfLength(8));
}
final Settings settings = Settings.builder().putList("foo.old", values).build();
final Settings upgradedSettings = service.upgradeSettings(settings);
assertFalse(oldSetting.exists(upgradedSettings));
assertTrue(newSetting.exists(upgradedSettings));
assertThat(
newSetting.get(upgradedSettings),
equalTo(oldSetting.get(settings).stream().map(s -> "new." + s).collect(Collectors.toList())));
}
}

7
server/src/test/java/org/elasticsearch/common/settings/SettingTests.java
View File

@ -180,6 +180,13 @@ public class SettingTests extends ESTestCase {
}
}
public void testValidateStringSetting() {
Settings settings = Settings.builder().putList("foo.bar", Arrays.asList("bla-a", "bla-b")).build();
Setting<String> stringSetting = Setting.simpleString("foo.bar", Property.NodeScope);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> stringSetting.get(settings));
assertEquals("Found list type value for setting [foo.bar] but but did not expect a list for it.", e.getMessage());
}
private static final Setting<String> FOO_BAR_SETTING = new Setting<>(
"foo.bar",
"foobar",

Some files were not shown because too many files have changed in this diff Show More