honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into index-lifecycle

This commit is contained in:
Lee Hinman 2018-09-19 09:43:26 -06:00
parent 27dd25857b 6ec12bef0d
commit 81e9150c7a
102 changed files with 5151 additions and 2227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/src/test/java/org/elasticsearch/gradle/precommit/JarHellTaskIT.java
View File

@ -34,7 +34,7 @@ public class JarHellTaskIT extends GradleIntegrationTestCase {
assertTaskFailed(result, ":jarHell");
assertOutputContains(
result.getOutput(),
"Exception in thread \"main\" java.lang.IllegalStateException: jar hell!",
"java.lang.IllegalStateException: jar hell!",
"class: org.apache.logging.log4j.Logger"
);
}

12
client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
View File

@ -28,6 +28,7 @@ import org.apache.http.entity.ByteArrayEntity;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.client.RequestConverters.EndpointBuilder;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.DeleteCalendarRequest;
import org.elasticsearch.client.ml.DeleteDatafeedRequest;
import org.elasticsearch.client.ml.DeleteForecastRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
@ -372,4 +373,15 @@ final class MLRequestConverters {
request.setEntity(createEntity(getCalendarsRequest, REQUEST_BODY_CONTENT_TYPE));
return request;
}
static Request deleteCalendar(DeleteCalendarRequest deleteCalendarRequest) {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_xpack")
.addPathPartAsIs("ml")
.addPathPartAsIs("calendars")
.addPathPart(deleteCalendarRequest.getCalendarId())
.build();
Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
return request;
}
}

41
client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
View File

@ -22,6 +22,7 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.CloseJobResponse;
import org.elasticsearch.client.ml.DeleteCalendarRequest;
import org.elasticsearch.client.ml.DeleteDatafeedRequest;
import org.elasticsearch.client.ml.DeleteForecastRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
@ -910,4 +911,44 @@ public final class MachineLearningClient {
listener,
Collections.emptySet());
}
/**
* Deletes the given Machine Learning Calendar
* <p>
* For additional info see
* <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-calendar.html">
* ML Delete calendar documentation</a>
*
* @param request The request to delete the calendar
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return action acknowledgement
* @throws IOException when there is a serialization issue sending the request or receiving the response
*/
public AcknowledgedResponse deleteCalendar(DeleteCalendarRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(request,
MLRequestConverters::deleteCalendar,
options,
AcknowledgedResponse::fromXContent,
Collections.emptySet());
}
/**
* Deletes the given Machine Learning Job asynchronously and notifies the listener on completion
* <p>
* For additional info see
* <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-calendar.html">
* ML Delete calendar documentation</a>
*
* @param request The request to delete the calendar
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener Listener to be notified upon request completion
*/
public void deleteCalendarAsync(DeleteCalendarRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
restHighLevelClient.performRequestAsyncAndParseEntity(request,
MLRequestConverters::deleteCalendar,
options,
AcknowledgedResponse::fromXContent,
listener,
Collections.emptySet());
}
}

65
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteCalendarRequest.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import java.util.Objects;
/**
* Request to delete a Machine Learning Calendar
*/
public class DeleteCalendarRequest extends ActionRequest {
private final String calendarId;
/**
* The constructor requires a single calendar id.
* @param calendarId The calendar to delete. Must be {@code non-null}
*/
public DeleteCalendarRequest(String calendarId) {
this.calendarId = Objects.requireNonNull(calendarId, "[calendar_id] must not be null");
}
public String getCalendarId() {
return calendarId;
}
@Override
public ActionRequestValidationException validate() {
return null;
}
@Override
public int hashCode() {
return Objects.hash(calendarId);
}
@Override
public boolean equals(Object obj) {
if (obj == null || getClass() != obj.getClass()) {
return false;
}
DeleteCalendarRequest other = (DeleteCalendarRequest) obj;
return Objects.equals(calendarId, other.calendarId);
}
}

8
client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
View File

@ -24,6 +24,7 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.DeleteCalendarRequest;
import org.elasticsearch.client.ml.DeleteDatafeedRequest;
import org.elasticsearch.client.ml.DeleteForecastRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
@ -438,6 +439,13 @@ public class MLRequestConvertersTests extends ESTestCase {
}
}
public void testDeleteCalendar() {
DeleteCalendarRequest deleteCalendarRequest = new DeleteCalendarRequest(randomAlphaOfLength(10));
Request request = MLRequestConverters.deleteCalendar(deleteCalendarRequest);
assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
assertEquals("/_xpack/ml/calendars/" + deleteCalendarRequest.getCalendarId(), request.getEndpoint());
}
private static Job createValidJob(String jobId) {
AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList(
Detector.builder().setFunction("count").build()));

19
client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.CloseJobResponse;
import org.elasticsearch.client.ml.DeleteCalendarRequest;
import org.elasticsearch.client.ml.DeleteDatafeedRequest;
import org.elasticsearch.client.ml.DeleteForecastRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
@ -517,6 +518,24 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
assertEquals(calendar1, getCalendarsResponse.calendars().get(0));
}
public void testDeleteCalendar() throws IOException {
Calendar calendar = CalendarTests.testInstance();
MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
execute(new PutCalendarRequest(calendar), machineLearningClient::putCalendar,
machineLearningClient::putCalendarAsync);
AcknowledgedResponse response = execute(new DeleteCalendarRequest(calendar.getId()),
machineLearningClient::deleteCalendar,
machineLearningClient::deleteCalendarAsync);
assertTrue(response.isAcknowledged());
// calendar is missing
ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class,
() -> execute(new DeleteCalendarRequest(calendar.getId()), machineLearningClient::deleteCalendar,
machineLearningClient::deleteCalendarAsync));
assertThat(exception.status().getStatus(), equalTo(404));
}
public static String randomValidJobId() {
CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz0123456789".toCharArray());
return generator.ofCodePointsLength(random(), 10, 10);

47
client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.CloseJobResponse;
import org.elasticsearch.client.ml.DeleteCalendarRequest;
import org.elasticsearch.client.ml.DeleteDatafeedRequest;
import org.elasticsearch.client.ml.DeleteForecastRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
@ -1591,4 +1592,50 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}
public void testDeleteCalendar() throws IOException, InterruptedException {
RestHighLevelClient client = highLevelClient();
Calendar calendar = new Calendar("holidays", Collections.singletonList("job_1"), "A calendar for public holidays");
PutCalendarRequest putCalendarRequest = new PutCalendarRequest(calendar);
client.machineLearning().putCalendar(putCalendarRequest, RequestOptions.DEFAULT);
//tag::x-pack-ml-delete-calendar-request
DeleteCalendarRequest request = new DeleteCalendarRequest("holidays"); // <1>
//end::x-pack-ml-delete-calendar-request
//tag::x-pack-ml-delete-calendar-execute
AcknowledgedResponse response = client.machineLearning().deleteCalendar(request, RequestOptions.DEFAULT);
//end::x-pack-ml-delete-calendar-execute
//tag::x-pack-ml-delete-calendar-response
boolean isAcknowledged = response.isAcknowledged(); // <1>
//end::x-pack-ml-delete-calendar-response
assertTrue(isAcknowledged);
// tag::x-pack-ml-delete-calendar-listener
ActionListener<AcknowledgedResponse> listener = new ActionListener<AcknowledgedResponse>() {
@Override
public void onResponse(AcknowledgedResponse response) {
// <1>
}
@Override
public void onFailure(Exception e) {
// <2>
}
};
// end::x-pack-ml-delete-calendar-listener
// Replace the empty listener by a blocking listener in test
final CountDownLatch latch = new CountDownLatch(1);
listener = new LatchedActionListener<>(listener, latch);
// tag::x-pack-ml-delete-calendar-execute-async
client.machineLearning().deleteCalendarAsync(request, RequestOptions.DEFAULT, listener); // <1>
// end::x-pack-ml-delete-calendar-execute-async
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}

43
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteCalendarRequestTests.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.test.ESTestCase;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
public class DeleteCalendarRequestTests extends ESTestCase {
public void testWithNullId() {
NullPointerException ex = expectThrows(NullPointerException.class, () -> new DeleteCalendarRequest(null));
assertEquals("[calendar_id] must not be null", ex.getMessage());
}
public void testEqualsAndHash() {
String id1 = randomAlphaOfLength(8);
String id2 = id1 + "_a";
assertThat(new DeleteCalendarRequest(id1), equalTo(new DeleteCalendarRequest(id1)));
assertThat(new DeleteCalendarRequest(id1).hashCode(), equalTo(new DeleteCalendarRequest(id1).hashCode()));
assertThat(new DeleteCalendarRequest(id1), not(equalTo(new DeleteCalendarRequest(id2))));
assertThat(new DeleteCalendarRequest(id1).hashCode(), not(equalTo(new DeleteCalendarRequest(id2).hashCode())));
}
}

5
distribution/src/bin/elasticsearch-shard Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
ES_MAIN_CLASS=org.elasticsearch.index.shard.ShardToolCli \
"`dirname "$0"`"/elasticsearch-cli \
"$@"

12
distribution/src/bin/elasticsearch-shard.bat Normal file
View File

@ -0,0 +1,12 @@
@echo off
setlocal enabledelayedexpansion
setlocal enableextensions
set ES_MAIN_CLASS=org.elasticsearch.index.shard.ShardToolCli
call "%~dp0elasticsearch-cli.bat" ^
%%* ^
|| exit /b 1
endlocal
endlocal

59
docs/java-rest/high-level/ml/delete-calendar.asciidoc Normal file
View File

@ -0,0 +1,59 @@
[[java-rest-high-x-pack-ml-delete-calendar]]
=== Delete Calendar API
Delete a {ml} calendar.
The API accepts a `DeleteCalendarRequest` and responds
with a `AcknowledgedResponse` object.
[[java-rest-high-x-pack-ml-delete-calendar-request]]
==== Delete Calendar Request
A `DeleteCalendar` object requires a non-null `calendarId`.
["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-calendar-request]
---------------------------------------------------
<1> Constructing a new request referencing an existing Calendar
[[java-rest-high-x-pack-ml-delete-calendar-response]]
==== Delete Calendar Response
The returned `AcknowledgedResponse` object indicates the acknowledgement of the request:
["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-calendar-response]
---------------------------------------------------
<1> `isAcknowledged` was the deletion request acknowledged or not
[[java-rest-high-x-pack-ml-delete-calendar-execution]]
==== Execution
The request can be executed through the `MachineLearningClient` contained
in the `RestHighLevelClient` object, accessed via the `machineLearningClient()` method.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-calendar-execute]
--------------------------------------------------
[[java-rest-high-x-pack-ml-delete-calendar-async]]
==== Delete Calendar Asynchronously
This request can also be made asynchronously.
["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-calendar-execute-async]
---------------------------------------------------
<1> The `DeleteCalendarRequest` to execute and the `ActionListener` to alert on completion or error.
The deletion request returns immediately. Once the request is completed, the `ActionListener` is
called back using the `onResponse` or `onFailure`. The latter indicates some failure occurred when
making the request.
A typical listener for a `DeleteCalendarRequest` could be defined as follows:
["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-calendar-listener]
---------------------------------------------------
<1> The action to be taken when it is completed
<2> What to do when a failure occurs

2
docs/java-rest/high-level/supported-apis.asciidoc
View File

@ -233,6 +233,7 @@ The Java High Level REST Client supports the following Machine Learning APIs:
* <<java-rest-high-x-pack-ml-get-categories>>
* <<java-rest-high-x-pack-ml-get-calendars>>
* <<java-rest-high-x-pack-ml-put-calendar>>
* <<java-rest-high-x-pack-ml-delete-calendar>>
include::ml/put-job.asciidoc[]
include::ml/get-job.asciidoc[]
@ -255,6 +256,7 @@ include::ml/get-influencers.asciidoc[]
include::ml/get-categories.asciidoc[]
include::ml/get-calendars.asciidoc[]
include::ml/put-calendar.asciidoc[]
include::ml/delete-calendar.asciidoc[]
== Migration APIs

12
docs/painless/painless-getting-started.asciidoc
View File

@ -198,13 +198,11 @@ POST hockey/player/1/_update
==== Dates
Date fields are exposed as
`ReadableDateTime` or
so they support methods like
`getYear`,
and `getDayOfWeek`.
To get milliseconds since epoch call
`getMillis`.
For example, the following returns every hockey player's birth year:
`ReadableDateTime`, so they support methods like `getYear`, `getDayOfWeek`
or e.g. getting milliseconds since epoch with `getMillis`. To use these
in a script, leave out the `get` prefix and continue with lowercasing the
rest of the method name. For example, the following returns every hockey
player's birth year:
[source,js]
----------------------------------------------------------------

8
docs/reference/analysis/tokenfilters/multiplexer-tokenfilter.asciidoc
View File

@ -114,3 +114,11 @@ And it'd respond:
<1> The stemmer has also emitted a token `home` at position 1, but because it is a
duplicate of this token it has been removed from the token stream
NOTE: The synonym and synonym_graph filters use their preceding analysis chain to
parse and analyse their synonym lists, and ignore any token filters in the chain
that produce multiple tokens at the same position. This means that any filters
within the multiplexer will be ignored for the purpose of synonyms. If you want to
use filters contained within the multiplexer for parsing synonyms (for example, to
apply stemming to the synonym lists), then you should append the synonym filter
to the relevant multiplexer filter list.

2
docs/reference/commands/index.asciidoc
View File

@ -12,6 +12,7 @@ tasks from the command line:
* <<migrate-tool>>
* <<saml-metadata>>
* <<setup-passwords>>
* <<shard-tool>>
* <<syskeygen>>
* <<users-command>>
@ -22,5 +23,6 @@ include::certutil.asciidoc[]
include::migrate-tool.asciidoc[]
include::saml-metadata.asciidoc[]
include::setup-passwords.asciidoc[]
include::shard-tool.asciidoc[]
include::syskeygen.asciidoc[]
include::users-command.asciidoc[]

107
docs/reference/commands/shard-tool.asciidoc Normal file
View File

@ -0,0 +1,107 @@
[[shard-tool]]
== elasticsearch-shard
In some cases the Lucene index or translog of a shard copy can become
corrupted. The `elasticsearch-shard` command enables you to remove corrupted
parts of the shard if a good copy of the shard cannot be recovered
automatically or restored from backup.
[WARNING]
You will lose the corrupted data when you run `elasticsearch-shard`. This tool
should only be used as a last resort if there is no way to recover from another
copy of the shard or restore a snapshot.
When Elasticsearch detects that a shard's data is corrupted, it fails that
shard copy and refuses to use it. Under normal conditions, the shard is
automatically recovered from another copy. If no good copy of the shard is
available and you cannot restore from backup, you can use `elasticsearch-shard`
to remove the corrupted data and restore access to any remaining data in
unaffected segments.
[WARNING]
Stop Elasticsearch before running `elasticsearch-shard`.
To remove corrupted shard data use the `remove-corrupted-data` subcommand.
There are two ways to specify the path:
* Specify the index name and shard name with the `--index` and `--shard-id`
options.
* Use the `--dir` option to specify the full path to the corrupted index or
translog files.
[float]
=== Removing corrupted data
`elasticsearch-shard` analyses the shard copy and provides an overview of the
corruption found. To proceed you must then confirm that you want to remove the
corrupted data.
[WARNING]
Back up your data before running `elasticsearch-shard`. This is a destructive
operation that removes corrupted data from the shard.
[source,txt]
--------------------------------------------------
$ bin/elasticsearch-shard remove-corrupted-data --index twitter --shard-id 0
WARNING: Elasticsearch MUST be stopped before running this tool.
Please make a complete backup of your index before using this tool.
Opening Lucene index at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
>> Lucene index is corrupted at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
Opening translog at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
>> Translog is clean at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
Corrupted Lucene index segments found - 32 documents will be lost.
WARNING: YOU WILL LOSE DATA.
Continue and remove docs from the index ? Y
WARNING: 1 broken segments (containing 32 documents) detected
Took 0.056 sec total.
Writing...
OK
Wrote new segments file "segments_c"
Marking index with the new history uuid : 0pIBd9VTSOeMfzYT6p0AsA
Changing allocation id V8QXk-QXSZinZMT-NvEq4w to tjm9Ve6uTBewVFAlfUMWjA
You should run the following command to allocate this shard:
POST /_cluster/reroute
{
"commands" : [
{
"allocate_stale_primary" : {
"index" : "index42",
"shard" : 0,
"node" : "II47uXW2QvqzHBnMcl2o_Q",
"accept_data_loss" : false
}
}
]
}
You must accept the possibility of data loss by changing parameter `accept_data_loss` to `true`.
Deleted corrupt marker corrupted_FzTSBSuxT7i3Tls_TgwEag from /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
--------------------------------------------------
When you use `elasticsearch-shard` to drop the corrupted data, the shard's
allocation ID changes. After restarting the node, you must use the
<<cluster-reroute,cluster reroute API>> to tell Elasticsearch to use the new
ID. The `elasticsearch-shard` command shows the request that
you need to submit.
You can also use the `-h` option to get a list of all options and parameters
that the `elasticsearch-shard` tool supports.

4
docs/reference/index-modules/translog.asciidoc
View File

@ -92,6 +92,10 @@ The maximum duration for which translog files will be kept. Defaults to `12h`.
[[corrupt-translog-truncation]]
=== What to do if the translog becomes corrupted?
[WARNING]
This tool is deprecated and will be completely removed in 7.0.
Use the <<shard-tool,elasticsearch-shard tool>> instead of this one.
In some cases (a bad drive, user error) the translog on a shard copy can become
corrupted. When this corruption is detected by Elasticsearch due to mismatching
checksums, Elasticsearch will fail that shard copy and refuse to use that copy

49
docs/reference/modules/snapshots.asciidoc
View File

@ -1,22 +1,47 @@
[[modules-snapshots]]
== Snapshot And Restore
You can store snapshots of individual indices or an entire cluster in
a remote repository like a shared file system, S3, or HDFS. These snapshots
are great for backups because they can be restored relatively quickly. However,
snapshots can only be restored to versions of Elasticsearch that can read the
indices:
A snapshot is a backup taken from a running Elasticsearch cluster. You can take
a snapshot of individual indices or of the entire cluster and store it in a
repository on a shared filesystem, and there are plugins that support remote
repositories on S3, HDFS, Azure, Google Cloud Storage and more.
Snapshots are taken incrementally. This means that when creating a snapshot of
an index Elasticsearch will avoid copying any data that is already stored in
the repository as part of an earlier snapshot of the same index. Therefore it
can be efficient to take snapshots of your cluster quite frequently.
Snapshots can be restored into a running cluster via the restore API. When
restoring an index it is possible to alter the name of the restored index as
well as some of its settings, allowing a great deal of flexibility in how the
snapshot and restore functionality can be used.
WARNING: It is not possible to back up an Elasticsearch cluster simply by
taking a copy of the data directories of all of its nodes. Elasticsearch may be
making changes to the contents of its data directories while it is running, and
this means that copying its data directories cannot be expected to capture a
consistent picture of their contents. Attempts to restore a cluster from such a
backup may fail, reporting corruption and/or missing files, or may appear to
have succeeded having silently lost some of its data. The only reliable way to
back up a cluster is by using the snapshot and restore functionality.
[float]
=== Version compatibility
A snapshot contains a copy of the on-disk data structures that make up an
index. This means that snapshots can only be restored to versions of
Elasticsearch that can read the indices:
* A snapshot of an index created in 5.x can be restored to 6.x.
* A snapshot of an index created in 2.x can be restored to 5.x.
* A snapshot of an index created in 1.x can be restored to 2.x.
Conversely, snapshots of indices created in 1.x **cannot** be restored to
5.x or 6.x, and snapshots of indices created in 2.x **cannot** be restored
to 6.x.
Conversely, snapshots of indices created in 1.x **cannot** be restored to 5.x
or 6.x, and snapshots of indices created in 2.x **cannot** be restored to 6.x.
Snapshots are incremental and can contain indices created in various
versions of Elasticsearch. If any indices in a snapshot were created in an
Each snapshot can contain indices created in various versions of Elasticsearch,
and when restoring a snapshot it must be possible to restore all of the indices
into the target cluster. If any indices in a snapshot were created in an
incompatible version, you will not be able restore the snapshot.
IMPORTANT: When backing up your data prior to an upgrade, keep in mind that you
@ -28,8 +53,8 @@ that is incompatible with the version of the cluster you are currently running,
you can restore it on the latest compatible version and use
<<reindex-from-remote,reindex-from-remote>> to rebuild the index on the current
version. Reindexing from remote is only possible if the original index has
source enabled. Retrieving and reindexing the data can take significantly longer
than simply restoring a snapshot. If you have a large amount of data, we
source enabled. Retrieving and reindexing the data can take significantly
longer than simply restoring a snapshot. If you have a large amount of data, we
recommend testing the reindex from remote process with a subset of your data to
understand the time requirements before proceeding.

7
libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java
View File

@ -85,12 +85,17 @@ public abstract class Terminal {
/** Prints message to the terminal at {@code verbosity} level, without a newline. */
public final void print(Verbosity verbosity, String msg) {
if (this.verbosity.ordinal() >= verbosity.ordinal()) {
if (isPrintable(verbosity)) {
getWriter().print(msg);
getWriter().flush();
}
}
/** Checks if is enough {@code verbosity} level to be printed */
public final boolean isPrintable(Verbosity verbosity) {
return this.verbosity.ordinal() >= verbosity.ordinal();
}
/**
* Prompt for a yes or no answer from the user. This method will loop until 'y' or 'n'
* (or the default empty value) is entered.

83
modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java
View File

@ -29,39 +29,64 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
import org.elasticsearch.index.analysis.ReferringFilterFactory;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory implements ReferringFilterFactory {
public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory {
private List<TokenFilterFactory> filters;
private List<String> filterNames;
private final boolean preserveOriginal;
private static final TokenFilterFactory IDENTITY_FACTORY = new TokenFilterFactory() {
@Override
public String name() {
return "identity";
}
@Override
public TokenStream create(TokenStream tokenStream) {
return tokenStream;
}
};
public MultiplexerTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) throws IOException {
super(indexSettings, name, settings);
this.filterNames = settings.getAsList("filters");
this.preserveOriginal = settings.getAsBoolean("preserve_original", true);
}
@Override
public TokenStream create(TokenStream tokenStream) {
throw new UnsupportedOperationException("TokenFilterFactory.getChainAwareTokenFilterFactory() must be called first");
}
@Override
public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
List<TokenFilterFactory> previousTokenFilters,
Function<String, TokenFilterFactory> allFilters) {
List<TokenFilterFactory> filters = new ArrayList<>();
if (preserveOriginal) {
filters.add(IDENTITY_FILTER);
}
for (String filter : filterNames) {
String[] parts = Strings.tokenizeToStringArray(filter, ",");
if (parts.length == 1) {
TokenFilterFactory factory = resolveFilterFactory(allFilters, parts[0]);
factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, previousTokenFilters, allFilters);
filters.add(factory);
} else {
List<TokenFilterFactory> existingChain = new ArrayList<>(previousTokenFilters);
List<TokenFilterFactory> chain = new ArrayList<>();
for (String subfilter : parts) {
TokenFilterFactory factory = resolveFilterFactory(allFilters, subfilter);
factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, existingChain, allFilters);
chain.add(factory);
existingChain.add(factory);
}
filters.add(chainFilters(filter, chain));
}
}
return new TokenFilterFactory() {
@Override
public String name() {
return MultiplexerTokenFilterFactory.this.name();
}
@Override
public TokenStream create(TokenStream tokenStream) {
List<Function<TokenStream, TokenStream>> functions = new ArrayList<>();
@ -72,23 +97,10 @@ public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory im
}
@Override
public void setReferences(Map<String, TokenFilterFactory> factories) {
filters = new ArrayList<>();
if (preserveOriginal) {
filters.add(IDENTITY_FACTORY);
}
for (String filter : filterNames) {
String[] parts = Strings.tokenizeToStringArray(filter, ",");
if (parts.length == 1) {
filters.add(resolveFilterFactory(factories, parts[0]));
} else {
List<TokenFilterFactory> chain = new ArrayList<>();
for (String subfilter : parts) {
chain.add(resolveFilterFactory(factories, subfilter));
}
filters.add(chainFilters(filter, chain));
}
public TokenFilterFactory getSynonymFilter() {
return IDENTITY_FILTER;
}
};
}
private TokenFilterFactory chainFilters(String name, List<TokenFilterFactory> filters) {
@ -108,11 +120,12 @@ public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory im
};
}
private TokenFilterFactory resolveFilterFactory(Map<String, TokenFilterFactory> factories, String name) {
if (factories.containsKey(name) == false) {
private TokenFilterFactory resolveFilterFactory(Function<String, TokenFilterFactory> factories, String name) {
TokenFilterFactory factory = factories.apply(name);
if (factory == null) {
throw new IllegalArgumentException("Multiplexing filter [" + name() + "] refers to undefined tokenfilter [" + name + "]");
} else {
return factories.get(name);
return factory;
}
}

52
modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java
View File

@ -24,26 +24,24 @@ import org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
import org.elasticsearch.index.analysis.ReferringFilterFactory;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
/**
* A factory for a conditional token filter that only applies child filters if the underlying token
* matches an {@link AnalysisPredicateScript}
*/
public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFactory implements ReferringFilterFactory {
public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFactory {
private final AnalysisPredicateScript.Factory factory;
private final List<TokenFilterFactory> filters = new ArrayList<>();
private final List<String> filterNames;
ScriptedConditionTokenFilterFactory(IndexSettings indexSettings, String name,
@ -63,6 +61,34 @@ public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFact
}
}
@Override
public TokenStream create(TokenStream tokenStream) {
throw new UnsupportedOperationException("getChainAwareTokenFilterFactory should be called first");
}
@Override
public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
List<TokenFilterFactory> previousTokenFilters,
Function<String, TokenFilterFactory> allFilters) {
List<TokenFilterFactory> filters = new ArrayList<>();
List<TokenFilterFactory> existingChain = new ArrayList<>(previousTokenFilters);
for (String filter : filterNames) {
TokenFilterFactory tff = allFilters.apply(filter);
if (tff == null) {
throw new IllegalArgumentException("ScriptedConditionTokenFilter [" + name() +
"] refers to undefined token filter [" + filter + "]");
}
tff = tff.getChainAwareTokenFilterFactory(tokenizer, charFilters, existingChain, allFilters);
filters.add(tff);
existingChain.add(tff);
}
return new TokenFilterFactory() {
@Override
public String name() {
return ScriptedConditionTokenFilterFactory.this.name();
}
@Override
public TokenStream create(TokenStream tokenStream) {
Function<TokenStream, TokenStream> filter = in -> {
@ -73,6 +99,8 @@ public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFact
};
return new ScriptedConditionTokenFilter(tokenStream, filter, factory.newInstance());
}
};
}
private static class ScriptedConditionTokenFilter extends ConditionalTokenFilter {
@ -87,22 +115,10 @@ public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFact
}
@Override
protected boolean shouldFilter() throws IOException {
protected boolean shouldFilter() {
token.updatePosition();
return script.execute(token);
}
}
@Override
public void setReferences(Map<String, TokenFilterFactory> factories) {
for (String filter : filterNames) {
TokenFilterFactory tff = factories.get(filter);
if (tff == null) {
throw new IllegalArgumentException("ScriptedConditionTokenFilter [" + name() +
"] refers to undefined token filter [" + filter + "]");
}
filters.add(tff);
}
}
}

21
modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.analysis.common;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.elasticsearch.Version;
@ -117,6 +118,26 @@ public class SynonymsAnalysisTests extends ESTestCase {
}
}
public void testSynonymsWithMultiplexer() throws IOException {
Settings settings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put("path.home", createTempDir().toString())
.put("index.analysis.filter.synonyms.type", "synonym")
.putList("index.analysis.filter.synonyms.synonyms", "programmer, developer")
.put("index.analysis.filter.my_english.type", "stemmer")
.put("index.analysis.filter.my_english.language", "porter2")
.put("index.analysis.filter.stem_repeat.type", "multiplexer")
.putList("index.analysis.filter.stem_repeat.filters", "my_english, synonyms")
.put("index.analysis.analyzer.synonymAnalyzer.tokenizer", "standard")
.putList("index.analysis.analyzer.synonymAnalyzer.filter", "lowercase", "stem_repeat")
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
BaseTokenStreamTestCase.assertAnalyzesTo(indexAnalyzers.get("synonymAnalyzer"), "Some developers are odd",
new String[]{ "some", "developers", "develop", "programm", "are", "odd" },
new int[]{ 1, 1, 0, 0, 1, 1 });
}
private void match(String analyzerName, String source, String target) throws IOException {
Analyzer analyzer = indexAnalyzers.get(analyzerName).analyzer();

17
qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
View File

@ -325,4 +325,21 @@ public abstract class ArchiveTestCase extends PackagingTestCase {
}
}
public void test100RepairIndexCliPackaging() {
assumeThat(installation, is(notNullValue()));
final Installation.Executables bin = installation.executables();
final Shell sh = new Shell();
Platforms.PlatformAction action = () -> {
final Result result = sh.run(bin.elasticsearchShard + " help");
assertThat(result.stdout, containsString("A CLI tool to remove corrupted parts of unrecoverable shards"));
};
if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
Platforms.onLinux(action);
Platforms.onWindows(action);
}
}
}

1
qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java
View File

@ -186,6 +186,7 @@ public class Archives {
"elasticsearch-env",
"elasticsearch-keystore",
"elasticsearch-plugin",
"elasticsearch-shard",
"elasticsearch-translog"
).forEach(executable -> {

3
qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java
View File

@ -100,8 +100,9 @@ public class Installation {
public final Path elasticsearch = platformExecutable("elasticsearch");
public final Path elasticsearchPlugin = platformExecutable("elasticsearch-plugin");
public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore");
public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog");
public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil");
public final Path elasticsearchShard = platformExecutable("elasticsearch-shard");
public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog");
private Path platformExecutable(String name) {
final String platformExecutableName = Platforms.WINDOWS

1
qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java
View File

@ -187,6 +187,7 @@ public class Packages {
"elasticsearch",
"elasticsearch-plugin",
"elasticsearch-keystore",
"elasticsearch-shard",
"elasticsearch-translog"
).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));

1
qa/vagrant/src/test/resources/packaging/utils/packages.bash
View File

@ -95,6 +95,7 @@ verify_package_installation() {
assert_file "$ESHOME/bin" d root root 755
assert_file "$ESHOME/bin/elasticsearch" f root root 755
assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755
assert_file "$ESHOME/bin/elasticsearch-shard" f root root 755
assert_file "$ESHOME/bin/elasticsearch-translog" f root root 755
assert_file "$ESHOME/lib" d root root 755
assert_file "$ESCONFIG" d root elasticsearch 2750

1
qa/vagrant/src/test/resources/packaging/utils/tar.bash
View File

@ -94,6 +94,7 @@ verify_archive_installation() {
assert_file "$ESHOME/bin/elasticsearch-env" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-keystore" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-shard" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-translog" f elasticsearch elasticsearch 755
assert_file "$ESCONFIG" d elasticsearch elasticsearch 755
assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660

74
server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
View File

@ -48,11 +48,9 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.CustomAnalyzer;
import org.elasticsearch.index.analysis.CustomAnalyzerProvider;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.MultiTermAwareComponent;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.analysis.ReferringFilterFactory;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
@ -66,6 +64,7 @@ import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@ -73,6 +72,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Function;
/**
* Transport action used to execute analyze requests
@ -571,11 +571,48 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
return charFilterFactoryList;
}
public static class DeferredTokenFilterRegistry implements Function<String, TokenFilterFactory> {
private final AnalysisRegistry analysisRegistry;
private final IndexSettings indexSettings;
Map<String, TokenFilterFactory> prebuiltFilters;
public DeferredTokenFilterRegistry(AnalysisRegistry analysisRegistry, IndexSettings indexSettings) {
this.analysisRegistry = analysisRegistry;
if (indexSettings == null) {
// Settings are null when _analyze is called with no index name, so
// we create dummy settings which will make prebuilt analysis components
// available
Settings settings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
.build();
IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
indexSettings = new IndexSettings(metaData, Settings.EMPTY);
}
this.indexSettings = indexSettings;
}
@Override
public TokenFilterFactory apply(String s) {
if (prebuiltFilters == null) {
try {
prebuiltFilters = analysisRegistry.buildTokenFilterFactories(indexSettings);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
return prebuiltFilters.get(s);
}
}
private static List<TokenFilterFactory> parseTokenFilterFactories(AnalyzeRequest request, IndexSettings indexSettings, AnalysisRegistry analysisRegistry,
Environment environment, Tuple<String, TokenizerFactory> tokenizerFactory,
List<CharFilterFactory> charFilterFactoryList, boolean normalizer) throws IOException {
List<TokenFilterFactory> tokenFilterFactoryList = new ArrayList<>();
List<ReferringFilterFactory> referringFilters = new ArrayList<>();
DeferredTokenFilterRegistry deferredRegistry = new DeferredTokenFilterRegistry(analysisRegistry, indexSettings);
if (request.tokenFilters() != null && request.tokenFilters().size() > 0) {
List<AnalyzeRequest.NameOrDefinition> tokenFilters = request.tokenFilters();
for (AnalyzeRequest.NameOrDefinition tokenFilter : tokenFilters) {
@ -594,11 +631,8 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
}
// Need to set anonymous "name" of tokenfilter
tokenFilterFactory = tokenFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenfilter", settings);
tokenFilterFactory = CustomAnalyzerProvider.checkAndApplySynonymFilter(tokenFilterFactory, tokenizerFactory.v1(), tokenizerFactory.v2(), tokenFilterFactoryList,
charFilterFactoryList, environment);
if (tokenFilterFactory instanceof ReferringFilterFactory) {
referringFilters.add((ReferringFilterFactory)tokenFilterFactory);
}
tokenFilterFactory = tokenFilterFactory.getChainAwareTokenFilterFactory(tokenizerFactory.v2(), charFilterFactoryList,
tokenFilterFactoryList, deferredRegistry);
} else {
AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory;
@ -616,8 +650,8 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
Settings settings = AnalysisRegistry.getSettingsFromIndexSettings(indexSettings,
AnalysisRegistry.INDEX_ANALYSIS_FILTER + "." + tokenFilter.name);
tokenFilterFactory = tokenFilterFactoryFactory.get(indexSettings, environment, tokenFilter.name, settings);
tokenFilterFactory = CustomAnalyzerProvider.checkAndApplySynonymFilter(tokenFilterFactory, tokenizerFactory.v1(), tokenizerFactory.v2(), tokenFilterFactoryList,
charFilterFactoryList, environment);
tokenFilterFactory = tokenFilterFactory.getChainAwareTokenFilterFactory(tokenizerFactory.v2(), charFilterFactoryList,
tokenFilterFactoryList, deferredRegistry);
}
}
if (tokenFilterFactory == null) {
@ -633,26 +667,6 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
tokenFilterFactoryList.add(tokenFilterFactory);
}
}
if (referringFilters.isEmpty() == false) {
// The request included at least one custom referring tokenfilter that has not already been built by the
// analysis registry, so we need to set its references. Note that this will only apply pre-built
// tokenfilters
if (indexSettings == null) {
Settings settings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
.build();
IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
indexSettings = new IndexSettings(metaData, Settings.EMPTY);
}
Map<String, TokenFilterFactory> prebuiltFilters = analysisRegistry.buildTokenFilterFactories(indexSettings);
for (ReferringFilterFactory rff : referringFilters) {
rff.setReferences(prebuiltFilters);
}
}
return tokenFilterFactoryList;
}

4
server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.document.LatLonDocValuesField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FilterCodecReader;
import org.apache.lucene.index.FilterDirectoryReader;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.IndexCommit;
@ -726,6 +727,9 @@ public class Lucene {
} else if (reader instanceof FilterLeafReader) {
final FilterLeafReader fReader = (FilterLeafReader) reader;
return segmentReader(FilterLeafReader.unwrap(fReader));
} else if (reader instanceof FilterCodecReader) {
final FilterCodecReader fReader = (FilterCodecReader) reader;
return segmentReader(FilterCodecReader.unwrap(fReader));
}
// hard fail - we can't get a SegmentReader
throw new IllegalStateException("Can not extract segment reader from given index reader [" + reader + "]");

2
server/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java
View File

@ -70,7 +70,7 @@ public class RandomScoreFunction extends ScoreFunction {
public double score(int docId, float subQueryScore) throws IOException {
int hash;
if (values == null) {
hash = BitMixer.mix(docId, saltedSeed);
hash = BitMixer.mix(ctx.docBase + docId, saltedSeed);
} else if (values.advanceExact(docId)) {
hash = StringHelper.murmurhash3_x86_32(values.nextValue(), saltedSeed);
} else {

160
server/src/main/java/org/elasticsearch/env/NodeEnvironment.java vendored
View File

@ -30,6 +30,8 @@ import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.NativeFSLockFactory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -75,6 +77,7 @@ import java.util.Set;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import static java.util.Collections.unmodifiableSet;
@ -171,6 +174,63 @@ public final class NodeEnvironment implements Closeable {
public static final String INDICES_FOLDER = "indices";
public static final String NODE_LOCK_FILENAME = "node.lock";
public static class NodeLock implements Releasable {
private final int nodeId;
private final Lock[] locks;
private final NodePath[] nodePaths;
/**
* Tries to acquire a node lock for a node id, throws {@code IOException} if it is unable to acquire it
* @param pathFunction function to check node path before attempt of acquiring a node lock
*/
public NodeLock(final int nodeId, final Logger logger,
final Environment environment,
final CheckedFunction<Path, Boolean, IOException> pathFunction) throws IOException {
this.nodeId = nodeId;
nodePaths = new NodePath[environment.dataWithClusterFiles().length];
locks = new Lock[nodePaths.length];
try {
final Path[] dataPaths = environment.dataFiles();
for (int dirIndex = 0; dirIndex < dataPaths.length; dirIndex++) {
Path dataDir = dataPaths[dirIndex];
Path dir = resolveNodePath(dataDir, nodeId);
if (pathFunction.apply(dir) == false) {
continue;
}
try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
logger.trace("obtaining node lock on {} ...", dir.toAbsolutePath());
locks[dirIndex] = luceneDir.obtainLock(NODE_LOCK_FILENAME);
nodePaths[dirIndex] = new NodePath(dir);
} catch (IOException e) {
logger.trace(() -> new ParameterizedMessage(
"failed to obtain node lock on {}", dir.toAbsolutePath()), e);
// release all the ones that were obtained up until now
throw (e instanceof LockObtainFailedException ? e
: new IOException("failed to obtain lock on " + dir.toAbsolutePath(), e));
}
}
} catch (IOException e) {
close();
throw e;
}
}
public NodePath[] getNodePaths() {
return nodePaths;
}
@Override
public void close() {
for (int i = 0; i < locks.length; i++) {
if (locks[i] != null) {
IOUtils.closeWhileHandlingException(locks[i]);
}
locks[i] = null;
}
}
}
/**
* Setup the environment.
* @param settings settings from elasticsearch.yml
@ -188,51 +248,39 @@ public final class NodeEnvironment implements Closeable {
nodeIdConsumer.accept(nodeMetaData.nodeId());
return;
}
final NodePath[] nodePaths = new NodePath[environment.dataWithClusterFiles().length];
final Lock[] locks = new Lock[nodePaths.length];
boolean success = false;
NodeLock nodeLock = null;
try {
sharedDataPath = environment.sharedDataFile();
int nodeLockId = -1;
IOException lastException = null;
int maxLocalStorageNodes = MAX_LOCAL_STORAGE_NODES_SETTING.get(settings);
final AtomicReference<IOException> onCreateDirectoriesException = new AtomicReference<>();
for (int possibleLockId = 0; possibleLockId < maxLocalStorageNodes; possibleLockId++) {
for (int dirIndex = 0; dirIndex < environment.dataFiles().length; dirIndex++) {
Path dataDir = environment.dataFiles()[dirIndex];
Path dir = resolveNodePath(dataDir, possibleLockId);
Files.createDirectories(dir);
try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
logger.trace("obtaining node lock on {} ...", dir.toAbsolutePath());
try {
locks[dirIndex] = luceneDir.obtainLock(NODE_LOCK_FILENAME);
nodePaths[dirIndex] = new NodePath(dir);
nodeLockId = possibleLockId;
} catch (LockObtainFailedException ex) {
logger.trace(
new ParameterizedMessage("failed to obtain node lock on {}", dir.toAbsolutePath()), ex);
// release all the ones that were obtained up until now
releaseAndNullLocks(locks);
break;
}
nodeLock = new NodeLock(possibleLockId, logger, environment,
dir -> {
try {
Files.createDirectories(dir);
} catch (IOException e) {
logger.trace(() -> new ParameterizedMessage(
"failed to obtain node lock on {}", dir.toAbsolutePath()), e);
lastException = new IOException("failed to obtain lock on " + dir.toAbsolutePath(), e);
// release all the ones that were obtained up until now
releaseAndNullLocks(locks);
break;
onCreateDirectoriesException.set(e);
throw e;
}
}
if (locks[0] != null) {
// we found a lock, break
return true;
});
break;
} catch (LockObtainFailedException e) {
// ignore any LockObtainFailedException
} catch (IOException e) {
if (onCreateDirectoriesException.get() != null) {
throw onCreateDirectoriesException.get();
}
lastException = e;
}
}
if (locks[0] == null) {
if (nodeLock == null) {
final String message = String.format(
Locale.ROOT,
"failed to obtain node locks, tried [%s] with lock id%s;" +
@ -243,13 +291,12 @@ public final class NodeEnvironment implements Closeable {
maxLocalStorageNodes);
throw new IllegalStateException(message, lastException);
}
this.locks = nodeLock.locks;
this.nodePaths = nodeLock.nodePaths;
this.nodeLockId = nodeLock.nodeId;
this.nodeMetaData = loadOrCreateNodeMetaData(settings, logger, nodePaths);
nodeIdConsumer.accept(nodeMetaData.nodeId());
this.nodeLockId = nodeLockId;
this.locks = locks;
this.nodePaths = nodePaths;
if (logger.isDebugEnabled()) {
logger.debug("using node location [{}], local_lock_id [{}]", nodePaths, nodeLockId);
}
@ -262,7 +309,7 @@ public final class NodeEnvironment implements Closeable {
success = true;
} finally {
if (success == false) {
IOUtils.closeWhileHandlingException(locks);
close();
}
}
}
@ -278,15 +325,6 @@ public final class NodeEnvironment implements Closeable {
return path.resolve(NODES_FOLDER).resolve(Integer.toString(nodeLockId));
}
private static void releaseAndNullLocks(Lock[] locks) {
for (int i = 0; i < locks.length; i++) {
if (locks[i] != null) {
IOUtils.closeWhileHandlingException(locks[i]);
}
locks[i] = null;
}
}
private void maybeLogPathDetails() throws IOException {
// We do some I/O in here, so skip this if DEBUG/INFO are not enabled:
@ -696,6 +734,13 @@ public final class NodeEnvironment implements Closeable {
return paths;
}
/**
* Returns shared data path for this node environment
*/
public Path sharedDataPath() {
return sharedDataPath;
}
/**
* returns the unique uuid describing this node. The uuid is persistent in the data folder of this node
* and remains across restarts.
@ -956,11 +1001,22 @@ public final class NodeEnvironment implements Closeable {
* @param indexSettings settings for the index
*/
public Path resolveBaseCustomLocation(IndexSettings indexSettings) {
return resolveBaseCustomLocation(indexSettings, sharedDataPath, nodeLockId);
}
/**
* Resolve the custom path for a index's shard.
* Uses the {@code IndexMetaData.SETTING_DATA_PATH} setting to determine
* the root path for the index.
*
* @param indexSettings settings for the index
*/
public static Path resolveBaseCustomLocation(IndexSettings indexSettings, Path sharedDataPath, int nodeLockId) {
String customDataDir = indexSettings.customDataPath();
if (customDataDir != null) {
// This assert is because this should be caught by MetaDataCreateIndexService
assert sharedDataPath != null;
return sharedDataPath.resolve(customDataDir).resolve(Integer.toString(this.nodeLockId));
return sharedDataPath.resolve(customDataDir).resolve(Integer.toString(nodeLockId));
} else {
throw new IllegalArgumentException("no custom " + IndexMetaData.SETTING_DATA_PATH + " setting available");
}
@ -974,7 +1030,11 @@ public final class NodeEnvironment implements Closeable {
* @param indexSettings settings for the index
*/
private Path resolveIndexCustomLocation(IndexSettings indexSettings) {
return resolveBaseCustomLocation(indexSettings).resolve(indexSettings.getUUID());
return resolveIndexCustomLocation(indexSettings, sharedDataPath, nodeLockId);
}
private static Path resolveIndexCustomLocation(IndexSettings indexSettings, Path sharedDataPath, int nodeLockId) {
return resolveBaseCustomLocation(indexSettings, sharedDataPath, nodeLockId).resolve(indexSettings.getUUID());
}
/**
@ -986,7 +1046,11 @@ public final class NodeEnvironment implements Closeable {
* @param shardId shard to resolve the path to
*/
public Path resolveCustomLocation(IndexSettings indexSettings, final ShardId shardId) {
return resolveIndexCustomLocation(indexSettings).resolve(Integer.toString(shardId.id()));
return resolveCustomLocation(indexSettings, shardId, sharedDataPath, nodeLockId);
}
public static Path resolveCustomLocation(IndexSettings indexSettings, final ShardId shardId, Path sharedDataPath, int nodeLockId) {
return resolveIndexCustomLocation(indexSettings, sharedDataPath, nodeLockId).resolve(Integer.toString(shardId.id()));
}
/**

12
server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
View File

@ -167,17 +167,7 @@ public final class AnalysisRegistry implements Closeable {
tokenFilters.put("synonym", requiresAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings)));
tokenFilters.put("synonym_graph", requiresAnalysisSettings((is, env, name, settings) -> new SynonymGraphTokenFilterFactory(is, env, this, name, settings)));
Map<String, TokenFilterFactory> mappings
= buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.preConfiguredTokenFilters);
// ReferringTokenFilters require references to other tokenfilters, so we pass these in
// after all factories have been registered
for (TokenFilterFactory tff : mappings.values()) {
if (tff instanceof ReferringFilterFactory) {
((ReferringFilterFactory)tff).setReferences(mappings);
}
}
return mappings;
return buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.preConfiguredTokenFilters);
}
public Map<String, TokenizerFactory> buildTokenizerFactories(IndexSettings indexSettings) throws IOException {

31
server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java
View File

@ -81,9 +81,7 @@ public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider<Custom
if (tokenFilter == null) {
throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find filter under name [" + tokenFilterName + "]");
}
// no need offsetGap for tokenize synonyms
tokenFilter = checkAndApplySynonymFilter(tokenFilter, tokenizerName, tokenizer, tokenFilterList, charFiltersList,
this.environment);
tokenFilter = tokenFilter.getChainAwareTokenFilterFactory(tokenizer, charFiltersList, tokenFilterList, tokenFilters::get);
tokenFilterList.add(tokenFilter);
}
@ -95,33 +93,6 @@ public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider<Custom
);
}
public static TokenFilterFactory checkAndApplySynonymFilter(TokenFilterFactory tokenFilter, String tokenizerName, TokenizerFactory tokenizer,
List<TokenFilterFactory> tokenFilterList,
List<CharFilterFactory> charFiltersList, Environment env) {
if (tokenFilter instanceof SynonymGraphTokenFilterFactory) {
List<TokenFilterFactory> tokenFiltersListForSynonym = new ArrayList<>(tokenFilterList);
try (CustomAnalyzer analyzer = new CustomAnalyzer(tokenizerName, tokenizer,
charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]),
tokenFiltersListForSynonym.toArray(new TokenFilterFactory[tokenFiltersListForSynonym.size()]),
TextFieldMapper.Defaults.POSITION_INCREMENT_GAP,
-1)){
tokenFilter = ((SynonymGraphTokenFilterFactory) tokenFilter).createPerAnalyzerSynonymGraphFactory(analyzer, env);
}
} else if (tokenFilter instanceof SynonymTokenFilterFactory) {
List<TokenFilterFactory> tokenFiltersListForSynonym = new ArrayList<>(tokenFilterList);
try (CustomAnalyzer analyzer = new CustomAnalyzer(tokenizerName, tokenizer,
charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]),
tokenFiltersListForSynonym.toArray(new TokenFilterFactory[tokenFiltersListForSynonym.size()]),
TextFieldMapper.Defaults.POSITION_INCREMENT_GAP,
-1)) {
tokenFilter = ((SynonymTokenFilterFactory) tokenFilter).createPerAnalyzerSynonymFactory(analyzer, env);
}
}
return tokenFilter;
}
@Override
public CustomAnalyzer get() {
return this.customAnalyzer;

46
server/src/main/java/org/elasticsearch/index/analysis/SynonymGraphTokenFilterFactory.java
View File

@ -28,9 +28,11 @@ import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import java.io.IOException;
import java.io.Reader;
import java.util.List;
import java.util.function.Function;
public class SynonymGraphTokenFilterFactory extends SynonymTokenFilterFactory {
public SynonymGraphTokenFilterFactory(IndexSettings indexSettings, Environment env, AnalysisRegistry analysisRegistry,
String name, Settings settings) throws IOException {
super(indexSettings, env, analysisRegistry, name, settings);
@ -41,42 +43,24 @@ public class SynonymGraphTokenFilterFactory extends SynonymTokenFilterFactory {
throw new IllegalStateException("Call createPerAnalyzerSynonymGraphFactory to specialize this factory for an analysis chain first");
}
Factory createPerAnalyzerSynonymGraphFactory(Analyzer analyzerForParseSynonym, Environment env){
return new Factory("synonymgraph", analyzerForParseSynonym, getRulesFromSettings(env));
}
public class Factory implements TokenFilterFactory{
private final String name;
private final SynonymMap synonymMap;
public Factory(String name, final Analyzer analyzerForParseSynonym, Reader rulesReader) {
this.name = name;
try {
SynonymMap.Builder parser;
if ("wordnet".equalsIgnoreCase(format)) {
parser = new ESWordnetSynonymParser(true, expand, lenient, analyzerForParseSynonym);
((ESWordnetSynonymParser) parser).parse(rulesReader);
} else {
parser = new ESSolrSynonymParser(true, expand, lenient, analyzerForParseSynonym);
((ESSolrSynonymParser) parser).parse(rulesReader);
}
synonymMap = parser.build();
} catch (Exception e) {
throw new IllegalArgumentException("failed to build synonyms", e);
}
}
@Override
public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
List<TokenFilterFactory> previousTokenFilters,
Function<String, TokenFilterFactory> allFilters) {
final Analyzer analyzer = buildSynonymAnalyzer(tokenizer, charFilters, previousTokenFilters);
final SynonymMap synonyms = buildSynonyms(analyzer, getRulesFromSettings(environment));
final String name = name();
return new TokenFilterFactory() {
@Override
public String name() {
return this.name;
return name;
}
@Override
public TokenStream create(TokenStream tokenStream) {
// fst is null means no synonyms
return synonymMap.fst == null ? tokenStream : new SynonymGraphFilter(tokenStream, synonymMap, false);
return synonyms.fst == null ? tokenStream : new SynonymGraphFilter(tokenStream, synonyms, false);
}
};
}
}

87
server/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java
View File

@ -31,6 +31,7 @@ import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.List;
import java.util.function.Function;
public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
@ -38,6 +39,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
protected final boolean expand;
protected final boolean lenient;
protected final Settings settings;
protected final Environment environment;
public SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env, AnalysisRegistry analysisRegistry,
String name, Settings settings) throws IOException {
@ -53,6 +55,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
this.expand = settings.getAsBoolean("expand", true);
this.lenient = settings.getAsBoolean("lenient", false);
this.format = settings.get("format", "");
this.environment = env;
}
@Override
@ -60,6 +63,50 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
throw new IllegalStateException("Call createPerAnalyzerSynonymFactory to specialize this factory for an analysis chain first");
}
@Override
public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
List<TokenFilterFactory> previousTokenFilters,
Function<String, TokenFilterFactory> allFilters) {
final Analyzer analyzer = buildSynonymAnalyzer(tokenizer, charFilters, previousTokenFilters);
final SynonymMap synonyms = buildSynonyms(analyzer, getRulesFromSettings(environment));
final String name = name();
return new TokenFilterFactory() {
@Override
public String name() {
return name;
}
@Override
public TokenStream create(TokenStream tokenStream) {
return synonyms.fst == null ? tokenStream : new SynonymFilter(tokenStream, synonyms, false);
}
};
}
protected Analyzer buildSynonymAnalyzer(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
List<TokenFilterFactory> tokenFilters) {
return new CustomAnalyzer("synonyms", tokenizer, charFilters.toArray(new CharFilterFactory[0]),
tokenFilters.stream()
.map(TokenFilterFactory::getSynonymFilter)
.toArray(TokenFilterFactory[]::new));
}
protected SynonymMap buildSynonyms(Analyzer analyzer, Reader rules) {
try {
SynonymMap.Builder parser;
if ("wordnet".equalsIgnoreCase(format)) {
parser = new ESWordnetSynonymParser(true, expand, lenient, analyzer);
((ESWordnetSynonymParser) parser).parse(rules);
} else {
parser = new ESSolrSynonymParser(true, expand, lenient, analyzer);
((ESSolrSynonymParser) parser).parse(rules);
}
return parser.build();
} catch (Exception e) {
throw new IllegalArgumentException("failed to build synonyms", e);
}
}
protected Reader getRulesFromSettings(Environment env) {
Reader rulesReader;
if (settings.getAsList("synonyms", null) != null) {
@ -77,44 +124,4 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
return rulesReader;
}
Factory createPerAnalyzerSynonymFactory(Analyzer analyzerForParseSynonym, Environment env){
return new Factory("synonym", analyzerForParseSynonym, getRulesFromSettings(env));
}
public class Factory implements TokenFilterFactory{
private final String name;
private final SynonymMap synonymMap;
public Factory(String name, Analyzer analyzerForParseSynonym, Reader rulesReader) {
this.name = name;
try {
SynonymMap.Builder parser;
if ("wordnet".equalsIgnoreCase(format)) {
parser = new ESWordnetSynonymParser(true, expand, lenient, analyzerForParseSynonym);
((ESWordnetSynonymParser) parser).parse(rulesReader);
} else {
parser = new ESSolrSynonymParser(true, expand, lenient, analyzerForParseSynonym);
((ESSolrSynonymParser) parser).parse(rulesReader);
}
synonymMap = parser.build();
} catch (Exception e) {
throw new IllegalArgumentException("failed to build synonyms", e);
}
}
@Override
public String name() {
return this.name;
}
@Override
public TokenStream create(TokenStream tokenStream) {
// fst is null means no synonyms
return synonymMap.fst == null ? tokenStream : new SynonymFilter(tokenStream, synonymMap, false);
}
}
}

42
server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java
View File

@ -23,6 +23,9 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter;
import java.util.List;
import java.util.function.Function;
public interface TokenFilterFactory {
String name();
@ -36,4 +39,43 @@ public interface TokenFilterFactory {
default boolean breaksFastVectorHighlighter() {
return false;
}
/**
* Rewrite the TokenFilterFactory to take into account the preceding analysis chain, or refer
* to other TokenFilterFactories
* @param tokenizer the TokenizerFactory for the preceding chain
* @param charFilters any CharFilterFactories for the preceding chain
* @param previousTokenFilters a list of TokenFilterFactories in the preceding chain
* @param allFilters access to previously defined TokenFilterFactories
*/
default TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
List<TokenFilterFactory> previousTokenFilters,
Function<String, TokenFilterFactory> allFilters) {
return this;
}
/**
* Return a version of this TokenFilterFactory appropriate for synonym parsing
*
* Filters that should not be applied to synonyms (for example, those that produce
* multiple tokens) can return {@link #IDENTITY_FILTER}
*/
default TokenFilterFactory getSynonymFilter() {
return this;
}
/**
* A TokenFilterFactory that does no filtering to its TokenStream
*/
TokenFilterFactory IDENTITY_FILTER = new TokenFilterFactory() {
@Override
public String name() {
return "identity";
}
@Override
public TokenStream create(TokenStream tokenStream) {
return tokenStream;
}
};
}

36
server/src/main/java/org/elasticsearch/index/engine/Engine.java
View File

@ -66,6 +66,7 @@ import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.seqno.SeqNoStats;
import org.elasticsearch.index.seqno.SequenceNumbers;
import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.Translog;
@ -175,6 +176,41 @@ public abstract class Engine implements Closeable {
/** Returns how many bytes we are currently moving from heap to disk */
public abstract long getWritingBytes();
/**
* Returns the {@link DocsStats} for this engine
*/
public DocsStats docStats() {
// we calculate the doc stats based on the internal reader that is more up-to-date and not subject
// to external refreshes. For instance we don't refresh an external reader if we flush and indices with
// index.refresh_interval=-1 won't see any doc stats updates at all. This change will give more accurate statistics
// when indexing but not refreshing in general. Yet, if a refresh happens the internal reader is refresh as well so we are
// safe here.
try (Engine.Searcher searcher = acquireSearcher("docStats", Engine.SearcherScope.INTERNAL)) {
return docsStats(searcher.reader());
}
}
protected final DocsStats docsStats(IndexReader indexReader) {
long numDocs = 0;
long numDeletedDocs = 0;
long sizeInBytes = 0;
// we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
// the next scheduled refresh to go through and refresh the stats as well
for (LeafReaderContext readerContext : indexReader.leaves()) {
// we go on the segment level here to get accurate numbers
final SegmentReader segmentReader = Lucene.segmentReader(readerContext.reader());
SegmentCommitInfo info = segmentReader.getSegmentInfo();
numDocs += readerContext.reader().numDocs();
numDeletedDocs += readerContext.reader().numDeletedDocs();
try {
sizeInBytes += info.sizeInBytes();
} catch (IOException e) {
logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e);
}
}
return new DocsStats(numDocs, numDeletedDocs, sizeInBytes);
}
/**
* A throttling class that can be activated, causing the
* {@code acquireThrottle} method to block on a lock when throttling

8
server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.seqno.SeqNoStats;
import org.elasticsearch.index.seqno.SequenceNumbers;
import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.index.translog.TranslogStats;
@ -63,6 +64,7 @@ public final class ReadOnlyEngine extends Engine {
private final SearcherManager searcherManager;
private final IndexCommit indexCommit;
private final Lock indexWriterLock;
private final DocsStats docsStats;
/**
* Creates a new ReadOnlyEngine. This ctor can also be used to open a read-only engine on top of an already opened
@ -101,6 +103,7 @@ public final class ReadOnlyEngine extends Engine {
this.indexCommit = reader.getIndexCommit();
this.searcherManager = new SearcherManager(reader,
new RamAccountingSearcherFactory(engineConfig.getCircuitBreakerService()));
this.docsStats = docsStats(reader);
this.indexWriterLock = indexWriterLock;
success = true;
} finally {
@ -365,4 +368,9 @@ public final class ReadOnlyEngine extends Engine {
@Override
public void maybePruneDeletes() {
}
@Override
public DocsStats docStats() {
return docsStats;
}
}

32
server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
View File

@ -21,13 +21,9 @@ package org.elasticsearch.index.shard;
import com.carrotsearch.hppc.ObjectLongMap;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SegmentCommitInfo;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryCachingPolicy;
@ -879,32 +875,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
}
public DocsStats docStats() {
// we calculate the doc stats based on the internal reader that is more up-to-date and not subject
// to external refreshes. For instance we don't refresh an external reader if we flush and indices with
// index.refresh_interval=-1 won't see any doc stats updates at all. This change will give more accurate statistics
// when indexing but not refreshing in general. Yet, if a refresh happens the internal reader is refresh as well so we are
// safe here.
long numDocs = 0;
long numDeletedDocs = 0;
long sizeInBytes = 0;
try (Engine.Searcher searcher = acquireSearcher("docStats", Engine.SearcherScope.INTERNAL)) {
// we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
// the next scheduled refresh to go through and refresh the stats as well
readAllowed();
DocsStats docsStats = getEngine().docStats();
markSearcherAccessed();
for (LeafReaderContext reader : searcher.reader().leaves()) {
// we go on the segment level here to get accurate numbers
final SegmentReader segmentReader = Lucene.segmentReader(reader.reader());
SegmentCommitInfo info = segmentReader.getSegmentInfo();
numDocs += reader.reader().numDocs();
numDeletedDocs += reader.reader().numDeletedDocs();
try {
sizeInBytes += info.sizeInBytes();
} catch (IOException e) {
logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e);
}
}
}
return new DocsStats(numDocs, numDeletedDocs, sizeInBytes);
return docsStats;
}
/**

100
server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedLuceneSegmentsAction.java Normal file
View File

@ -0,0 +1,100 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.shard;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.common.collect.Tuple;
import java.io.IOException;
import java.io.PrintStream;
/**
* Removes corrupted Lucene index segments
*/
public class RemoveCorruptedLuceneSegmentsAction {
public Tuple<RemoveCorruptedShardDataCommand.CleanStatus, String> getCleanStatus(ShardPath shardPath,
Directory indexDirectory,
Lock writeLock,
PrintStream printStream,
boolean verbose) throws IOException {
if (RemoveCorruptedShardDataCommand.isCorruptMarkerFileIsPresent(indexDirectory) == false) {
return Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.CLEAN, null);
}
final CheckIndex.Status status;
try (CheckIndex checker = new CheckIndex(indexDirectory, writeLock)) {
checker.setChecksumsOnly(true);
checker.setInfoStream(printStream, verbose);
status = checker.checkIndex(null);
if (status.missingSegments) {
return Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.UNRECOVERABLE,
"Index is unrecoverable - there are missing segments");
}
return status.clean
? Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.CLEAN_WITH_CORRUPTED_MARKER, null)
: Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.CORRUPTED,
"Corrupted Lucene index segments found - " + status.totLoseDocCount + " documents will be lost.");
}
}
public void execute(Terminal terminal,
ShardPath shardPath,
Directory indexDirectory,
Lock writeLock,
PrintStream printStream,
boolean verbose) throws IOException {
checkCorruptMarkerFileIsPresent(indexDirectory);
final CheckIndex.Status status;
try (CheckIndex checker = new CheckIndex(indexDirectory, writeLock)) {
checker.setChecksumsOnly(true);
checker.setInfoStream(printStream, verbose);
status = checker.checkIndex(null);
if (status.missingSegments == false) {
if (status.clean == false) {
terminal.println("Writing...");
checker.exorciseIndex(status);
terminal.println("OK");
terminal.println("Wrote new segments file \"" + status.segmentsFileName + "\"");
}
} else {
throw new ElasticsearchException("Index is unrecoverable - there are missing segments");
}
}
}
protected void checkCorruptMarkerFileIsPresent(Directory directory) throws IOException {
if (RemoveCorruptedShardDataCommand.isCorruptMarkerFileIsPresent(directory) == false) {
throw new ElasticsearchException("There is no corruption file marker");
}
}
}

545
server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java Normal file
View File

@ -0,0 +1,545 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.shard;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.NativeFSLockFactory;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cli.EnvironmentAwareCommand;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.AllocationId;
import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand;
import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.NodeMetaData;
import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.seqno.SequenceNumbers;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.TruncateTranslogAction;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
public class RemoveCorruptedShardDataCommand extends EnvironmentAwareCommand {
private static final Logger logger = Loggers.getLogger(RemoveCorruptedShardDataCommand.class);
private final OptionSpec<String> folderOption;
private final OptionSpec<String> indexNameOption;
private final OptionSpec<Integer> shardIdOption;
private final RemoveCorruptedLuceneSegmentsAction removeCorruptedLuceneSegmentsAction;
private final TruncateTranslogAction truncateTranslogAction;
private final NamedXContentRegistry namedXContentRegistry;
public RemoveCorruptedShardDataCommand() {
this(false);
}
public RemoveCorruptedShardDataCommand(boolean translogOnly) {
super("Removes corrupted shard files");
folderOption = parser.acceptsAll(Arrays.asList("d", "dir"),
"Index directory location on disk")
.withRequiredArg();
indexNameOption = parser.accepts("index", "Index name")
.withRequiredArg();
shardIdOption = parser.accepts("shard-id", "Shard id")
.withRequiredArg()
.ofType(Integer.class);
namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
removeCorruptedLuceneSegmentsAction = translogOnly ? null : new RemoveCorruptedLuceneSegmentsAction();
truncateTranslogAction = new TruncateTranslogAction(namedXContentRegistry);
}
@Override
protected void printAdditionalHelp(Terminal terminal) {
if (removeCorruptedLuceneSegmentsAction == null) {
// that's only for 6.x branch for bwc with elasticsearch-translog
terminal.println("This tool truncates the translog and translog checkpoint files to create a new translog");
} else {
terminal.println("This tool attempts to detect and remove unrecoverable corrupted data in a shard.");
}
}
// Visible for testing
public OptionParser getParser() {
return this.parser;
}
@SuppressForbidden(reason = "Necessary to use the path passed in")
protected Path getPath(String dirValue) {
return PathUtils.get(dirValue, "", "");
}
protected void findAndProcessShardPath(OptionSet options, Environment environment, CheckedConsumer<ShardPath, IOException> consumer)
throws IOException {
final Settings settings = environment.settings();
final String indexName;
final int shardId;
final int fromNodeId;
final int toNodeId;
if (options.has(folderOption)) {
final Path path = getPath(folderOption.value(options)).getParent();
final Path shardParent = path.getParent();
final Path shardParentParent = shardParent.getParent();
final Path indexPath = path.resolve(ShardPath.INDEX_FOLDER_NAME);
if (Files.exists(indexPath) == false || Files.isDirectory(indexPath) == false) {
throw new ElasticsearchException("index directory [" + indexPath + "], must exist and be a directory");
}
final IndexMetaData indexMetaData =
IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, shardParent);
final String shardIdFileName = path.getFileName().toString();
final String nodeIdFileName = shardParentParent.getParent().getFileName().toString();
if (Files.isDirectory(path) && shardIdFileName.chars().allMatch(Character::isDigit) // SHARD-ID path element check
&& NodeEnvironment.INDICES_FOLDER.equals(shardParentParent.getFileName().toString()) // `indices` check
&& nodeIdFileName.chars().allMatch(Character::isDigit) // NODE-ID check
&& NodeEnvironment.NODES_FOLDER.equals(shardParentParent.getParent().getParent().getFileName().toString()) // `nodes` check
) {
shardId = Integer.parseInt(shardIdFileName);
indexName = indexMetaData.getIndex().getName();
fromNodeId = Integer.parseInt(nodeIdFileName);
toNodeId = fromNodeId + 1;
} else {
throw new ElasticsearchException("Unable to resolve shard id. Wrong folder structure at [ " + path.toString()
+ " ], expected .../nodes/[NODE-ID]/indices/[INDEX-UUID]/[SHARD-ID]");
}
} else {
// otherwise resolve shardPath based on the index name and shard id
indexName = Objects.requireNonNull(indexNameOption.value(options), "Index name is required");
shardId = Objects.requireNonNull(shardIdOption.value(options), "Shard ID is required");
// resolve shard path in case of multi-node layout per environment
fromNodeId = 0;
toNodeId = NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.get(settings);
}
// have to iterate over possibleLockId as NodeEnvironment; on a contrast to it - we have to fail if node is busy
for (int possibleLockId = fromNodeId; possibleLockId < toNodeId; possibleLockId++) {
try {
try (NodeEnvironment.NodeLock nodeLock = new NodeEnvironment.NodeLock(possibleLockId, logger, environment, Files::exists)) {
final NodeEnvironment.NodePath[] nodePaths = nodeLock.getNodePaths();
for (NodeEnvironment.NodePath nodePath : nodePaths) {
if (Files.exists(nodePath.indicesPath)) {
// have to scan all index uuid folders to resolve from index name
try (DirectoryStream<Path> stream = Files.newDirectoryStream(nodePath.indicesPath)) {
for (Path file : stream) {
if (Files.exists(file.resolve(MetaDataStateFormat.STATE_DIR_NAME)) == false) {
continue;
}
final IndexMetaData indexMetaData =
IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, file);
if (indexMetaData == null) {
continue;
}
final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings);
final Index index = indexMetaData.getIndex();
if (indexName.equals(index.getName()) == false) {
continue;
}
final ShardId shId = new ShardId(index, shardId);
final Path shardPathLocation = nodePath.resolve(shId);
if (Files.exists(shardPathLocation) == false) {
continue;
}
final ShardPath shardPath = ShardPath.loadShardPath(logger, shId, indexSettings,
new Path[]{shardPathLocation}, possibleLockId, nodePath.path);
if (shardPath != null) {
consumer.accept(shardPath);
return;
}
}
}
}
}
}
} catch (LockObtainFailedException lofe) {
throw new ElasticsearchException("Failed to lock node's directory [" + lofe.getMessage()
+ "], is Elasticsearch still running ?");
}
}
throw new ElasticsearchException("Unable to resolve shard path for index [" + indexName + "] and shard id [" + shardId + "]");
}
public static boolean isCorruptMarkerFileIsPresent(final Directory directory) throws IOException {
boolean found = false;
final String[] files = directory.listAll();
for (String file : files) {
if (file.startsWith(Store.CORRUPTED)) {
found = true;
break;
}
}
return found;
}
protected void dropCorruptMarkerFiles(Terminal terminal, Path path, Directory directory, boolean clean) throws IOException {
if (clean) {
confirm("This shard has been marked as corrupted but no corruption can now be detected.\n"
+ "This may indicate an intermittent hardware problem. The corruption marker can be \n"
+ "removed, but there is a risk that data has been undetectably lost.\n\n"
+ "Are you taking a risk of losing documents and proceed with removing a corrupted marker ?",
terminal);
}
String[] files = directory.listAll();
boolean found = false;
for (String file : files) {
if (file.startsWith(Store.CORRUPTED)) {
directory.deleteFile(file);
terminal.println("Deleted corrupt marker " + file + " from " + path);
}
}
}
private static void loseDataDetailsBanner(Terminal terminal, Tuple<CleanStatus, String> cleanStatus) {
if (cleanStatus.v2() != null) {
terminal.println("");
terminal.println(" " + cleanStatus.v2());
terminal.println("");
}
}
private static void confirm(String msg, Terminal terminal) {
terminal.println(msg);
String text = terminal.readText("Confirm [y/N] ");
if (text.equalsIgnoreCase("y") == false) {
throw new ElasticsearchException("aborted by user");
}
}
private void warnAboutESShouldBeStopped(Terminal terminal) {
terminal.println("-----------------------------------------------------------------------");
terminal.println("");
terminal.println(" WARNING: Elasticsearch MUST be stopped before running this tool.");
terminal.println("");
// that's only for 6.x branch for bwc with elasticsearch-translog
if (removeCorruptedLuceneSegmentsAction == null) {
terminal.println(" This tool is deprecated and will be completely removed in 7.0.");
terminal.println(" It is replaced by the elasticsearch-shard tool. ");
terminal.println("");
}
terminal.println(" Please make a complete backup of your index before using this tool.");
terminal.println("");
terminal.println("-----------------------------------------------------------------------");
}
// Visible for testing
@Override
public void execute(Terminal terminal, OptionSet options, Environment environment) throws Exception {
warnAboutESShouldBeStopped(terminal);
findAndProcessShardPath(options, environment, shardPath -> {
final Path indexPath = shardPath.resolveIndex();
final Path translogPath = shardPath.resolveTranslog();
final Path nodePath = getNodePath(shardPath);
if (Files.exists(translogPath) == false || Files.isDirectory(translogPath) == false) {
throw new ElasticsearchException("translog directory [" + translogPath + "], must exist and be a directory");
}
final PrintWriter writer = terminal.getWriter();
final PrintStream printStream = new PrintStream(new OutputStream() {
@Override
public void write(int b) {
writer.write(b);
}
}, false, "UTF-8");
final boolean verbose = terminal.isPrintable(Terminal.Verbosity.VERBOSE);
final Directory indexDirectory = getDirectory(indexPath);
final Tuple<CleanStatus, String> indexCleanStatus;
final Tuple<CleanStatus, String> translogCleanStatus;
try (Directory indexDir = indexDirectory) {
// keep the index lock to block any runs of older versions of this tool
try (Lock writeIndexLock = indexDir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
////////// Index
// that's only for 6.x branch for bwc with elasticsearch-translog
if (removeCorruptedLuceneSegmentsAction != null) {
terminal.println("");
terminal.println("Opening Lucene index at " + indexPath);
terminal.println("");
try {
indexCleanStatus = removeCorruptedLuceneSegmentsAction.getCleanStatus(shardPath, indexDir,
writeIndexLock, printStream, verbose);
} catch (Exception e) {
terminal.println(e.getMessage());
throw e;
}
terminal.println("");
terminal.println(" >> Lucene index is " + indexCleanStatus.v1().getMessage() + " at " + indexPath);
terminal.println("");
} else {
indexCleanStatus = Tuple.tuple(CleanStatus.CLEAN, null);
}
////////// Translog
// as translog relies on data stored in an index commit - we have to have non unrecoverable index to truncate translog
if (indexCleanStatus.v1() != CleanStatus.UNRECOVERABLE) {
terminal.println("");
terminal.println("Opening translog at " + translogPath);
terminal.println("");
try {
translogCleanStatus = truncateTranslogAction.getCleanStatus(shardPath, indexDir);
} catch (Exception e) {
terminal.println(e.getMessage());
throw e;
}
terminal.println("");
terminal.println(" >> Translog is " + translogCleanStatus.v1().getMessage() + " at " + translogPath);
terminal.println("");
} else {
translogCleanStatus = Tuple.tuple(CleanStatus.UNRECOVERABLE, null);
}
////////// Drop corrupted data
final CleanStatus indexStatus = indexCleanStatus.v1();
final CleanStatus translogStatus = translogCleanStatus.v1();
if (indexStatus == CleanStatus.CLEAN && translogStatus == CleanStatus.CLEAN) {
throw new ElasticsearchException("Shard does not seem to be corrupted at " + shardPath.getDataPath());
}
if (indexStatus == CleanStatus.UNRECOVERABLE) {
if (indexCleanStatus.v2() != null) {
terminal.println("Details: " + indexCleanStatus.v2());
}
terminal.println("You can allocate a new, empty, primary shard with the following command:");
printRerouteCommand(shardPath, terminal, false);
throw new ElasticsearchException("Index is unrecoverable");
}
terminal.println("-----------------------------------------------------------------------");
if (indexStatus != CleanStatus.CLEAN) {
loseDataDetailsBanner(terminal, indexCleanStatus);
}
if (translogStatus != CleanStatus.CLEAN) {
loseDataDetailsBanner(terminal, translogCleanStatus);
}
terminal.println(" WARNING: YOU MAY LOSE DATA.");
terminal.println("-----------------------------------------------------------------------");
confirm("Continue and remove corrupted data from the shard ?", terminal);
if (indexStatus != CleanStatus.CLEAN) {
removeCorruptedLuceneSegmentsAction.execute(terminal, shardPath, indexDir,
writeIndexLock, printStream, verbose);
}
if (translogStatus != CleanStatus.CLEAN) {
truncateTranslogAction.execute(terminal, shardPath, indexDir);
}
} catch (LockObtainFailedException lofe) {
final String msg = "Failed to lock shard's directory at [" + indexPath + "], is Elasticsearch still running?";
terminal.println(msg);
throw new ElasticsearchException(msg);
}
final CleanStatus indexStatus = indexCleanStatus.v1();
final CleanStatus translogStatus = translogCleanStatus.v1();
// newHistoryCommit obtains its own lock
addNewHistoryCommit(indexDir, terminal, translogStatus != CleanStatus.CLEAN);
newAllocationId(environment, shardPath, terminal);
if (indexStatus != CleanStatus.CLEAN) {
dropCorruptMarkerFiles(terminal, indexPath, indexDir, indexStatus == CleanStatus.CLEAN_WITH_CORRUPTED_MARKER);
}
}
});
}
private Directory getDirectory(Path indexPath) {
Directory directory;
try {
directory = FSDirectory.open(indexPath, NativeFSLockFactory.INSTANCE);
} catch (Throwable t) {
throw new ElasticsearchException("ERROR: could not open directory \"" + indexPath + "\"; exiting");
}
return directory;
}
protected void addNewHistoryCommit(Directory indexDirectory, Terminal terminal, boolean updateLocalCheckpoint) throws IOException {
final String historyUUID = UUIDs.randomBase64UUID();
terminal.println("Marking index with the new history uuid : " + historyUUID);
// commit the new history id
final IndexWriterConfig iwc = new IndexWriterConfig(null)
// we don't want merges to happen here - we call maybe merge on the engine
// later once we stared it up otherwise we would need to wait for it here
// we also don't specify a codec here and merges should use the engines for this index
.setCommitOnClose(false)
.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
.setMergePolicy(NoMergePolicy.INSTANCE)
.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
// IndexWriter acquires directory lock by its own
try (IndexWriter indexWriter = new IndexWriter(indexDirectory, iwc)) {
final Map<String, String> userData = new HashMap<>();
indexWriter.getLiveCommitData().forEach(e -> userData.put(e.getKey(), e.getValue()));
if (updateLocalCheckpoint) {
// In order to have a safe commit invariant, we have to assign the global checkpoint to the max_seqno of the last commit.
// We can only safely do it because we will generate a new history uuid this shard.
final SequenceNumbers.CommitInfo commitInfo = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userData.entrySet());
// Also advances the local checkpoint of the last commit to its max_seqno.
userData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(commitInfo.maxSeqNo));
}
// commit the new history id
userData.put(Engine.HISTORY_UUID_KEY, historyUUID);
indexWriter.setLiveCommitData(userData.entrySet());
indexWriter.commit();
}
}
protected void newAllocationId(Environment environment, ShardPath shardPath, Terminal terminal) throws IOException {
final Path shardStatePath = shardPath.getShardStatePath();
final ShardStateMetaData shardStateMetaData =
ShardStateMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, shardStatePath);
if (shardStateMetaData == null) {
throw new ElasticsearchException("No shard state meta data at " + shardStatePath);
}
final AllocationId newAllocationId = AllocationId.newInitializing();
terminal.println("Changing allocation id " + shardStateMetaData.allocationId.getId()
+ " to " + newAllocationId.getId());
final ShardStateMetaData newShardStateMetaData =
new ShardStateMetaData(shardStateMetaData.primary, shardStateMetaData.indexUUID, newAllocationId);
ShardStateMetaData.FORMAT.write(newShardStateMetaData, shardStatePath);
terminal.println("");
terminal.println("You should run the following command to allocate this shard:");
printRerouteCommand(shardPath, terminal, true);
}
private void printRerouteCommand(ShardPath shardPath, Terminal terminal, boolean allocateStale) throws IOException {
final IndexMetaData indexMetaData =
IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry,
shardPath.getDataPath().getParent());
final Path nodePath = getNodePath(shardPath);
final NodeMetaData nodeMetaData =
NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodePath);
if (nodeMetaData == null) {
throw new ElasticsearchException("No node meta data at " + nodePath);
}
final String nodeId = nodeMetaData.nodeId();
final String index = indexMetaData.getIndex().getName();
final int id = shardPath.getShardId().id();
final AllocationCommands commands = new AllocationCommands(
allocateStale
? new AllocateStalePrimaryAllocationCommand(index, id, nodeId, false)
: new AllocateEmptyPrimaryAllocationCommand(index, id, nodeId, false));
terminal.println("");
terminal.println("POST /_cluster/reroute'\n"
+ Strings.toString(commands, true, true) + "'");
terminal.println("");
terminal.println("You must accept the possibility of data loss by changing parameter `accept_data_loss` to `true`.");
terminal.println("");
}
private Path getNodePath(ShardPath shardPath) {
final Path nodePath = shardPath.getDataPath().getParent().getParent().getParent();
if (Files.exists(nodePath) == false || Files.exists(nodePath.resolve(MetaDataStateFormat.STATE_DIR_NAME)) == false) {
throw new ElasticsearchException("Unable to resolve node path for " + shardPath);
}
return nodePath;
}
public enum CleanStatus {
CLEAN("clean"),
CLEAN_WITH_CORRUPTED_MARKER("marked corrupted, but no corruption detected"),
CORRUPTED("corrupted"),
UNRECOVERABLE("corrupted and unrecoverable");
private final String msg;
CleanStatus(String msg) {
this.msg = msg;
}
public String getMessage() {
return msg;
}
}
}

25
server/src/main/java/org/elasticsearch/index/shard/ShardPath.java
View File

@ -112,16 +112,31 @@ public final class ShardPath {
* <b>Note:</b> this method resolves custom data locations for the shard.
*/
public static ShardPath loadShardPath(Logger logger, NodeEnvironment env, ShardId shardId, IndexSettings indexSettings) throws IOException {
final String indexUUID = indexSettings.getUUID();
final Path[] paths = env.availableShardPaths(shardId);
final int nodeLockId = env.getNodeLockId();
final Path sharedDataPath = env.sharedDataPath();
return loadShardPath(logger, shardId, indexSettings, paths, nodeLockId, sharedDataPath);
}
/**
* This method walks through the nodes shard paths to find the data and state path for the given shard. If multiple
* directories with a valid shard state exist the one with the highest version will be used.
* <b>Note:</b> this method resolves custom data locations for the shard.
*/
public static ShardPath loadShardPath(Logger logger, ShardId shardId, IndexSettings indexSettings, Path[] availableShardPaths,
int nodeLockId, Path sharedDataPath) throws IOException {
final String indexUUID = indexSettings.getUUID();
Path loadedPath = null;
for (Path path : paths) {
for (Path path : availableShardPaths) {
// EMPTY is safe here because we never call namedObject
ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
if (load != null) {
if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
logger.warn("{} found shard on path: [{}] with a different index UUID - this shard seems to be leftover from a different index with the same name. Remove the leftover shard in order to reuse the path with the current index", shardId, path);
throw new IllegalStateException(shardId + " index UUID in shard state was: " + load.indexUUID + " expected: " + indexUUID + " on shard path: " + path);
logger.warn("{} found shard on path: [{}] with a different index UUID - this "
+ "shard seems to be leftover from a different index with the same name. "
+ "Remove the leftover shard in order to reuse the path with the current index", shardId, path);
throw new IllegalStateException(shardId + " index UUID in shard state was: " + load.indexUUID
+ " expected: " + indexUUID + " on shard path: " + path);
}
if (loadedPath == null) {
loadedPath = path;
@ -137,7 +152,7 @@ public final class ShardPath {
final Path dataPath;
final Path statePath = loadedPath;
if (indexSettings.hasCustomDataPath()) {
dataPath = env.resolveCustomLocation(indexSettings, shardId);
dataPath = NodeEnvironment.resolveCustomLocation(indexSettings, shardId, sharedDataPath, nodeLockId);
} else {
dataPath = statePath;
}

26
server/src/main/java/org/elasticsearch/index/analysis/ReferringFilterFactory.java → server/src/main/java/org/elasticsearch/index/shard/ShardToolCli.java
View File

@ -16,22 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.shard;
package org.elasticsearch.index.analysis;
import java.util.Map;
import org.elasticsearch.cli.LoggingAwareMultiCommand;
import org.elasticsearch.cli.Terminal;
/**
* Marks a {@link TokenFilterFactory} that refers to other filter factories.
*
* The analysis registry will call {@link #setReferences(Map)} with a map of all
* available TokenFilterFactories after all factories have been registered
* Class encapsulating and dispatching commands from the {@code elasticsearch-shard} command line tool
*/
public interface ReferringFilterFactory {
public class ShardToolCli extends LoggingAwareMultiCommand {
/**
* Called with a map of all registered filter factories
*/
void setReferences(Map<String, TokenFilterFactory> factories);
private ShardToolCli() {
super("A CLI tool to remove corrupted parts of unrecoverable shards");
subcommands.put("remove-corrupted-data", new RemoveCorruptedShardDataCommand());
}
public static void main(String[] args) throws Exception {
exit(new ShardToolCli().main(args, Terminal.DEFAULT));
}
}

96
server/src/main/java/org/elasticsearch/index/similarity/NonNegativeScoresSimilarity.java Normal file
View File

@ -0,0 +1,96 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.similarity;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.Similarity;
/**
* A {@link Similarity} that rejects negative scores. This class exists so that users get
* an error instead of silently corrupt top hits. It should be applied to any custom or
* scripted similarity.
*/
// public for testing
public final class NonNegativeScoresSimilarity extends Similarity {
// Escape hatch
private static final String ES_ENFORCE_POSITIVE_SCORES = "es.enforce.positive.scores";
private static final boolean ENFORCE_POSITIVE_SCORES;
static {
String enforcePositiveScores = System.getProperty(ES_ENFORCE_POSITIVE_SCORES);
if (enforcePositiveScores == null) {
ENFORCE_POSITIVE_SCORES = true;
} else if ("false".equals(enforcePositiveScores)) {
ENFORCE_POSITIVE_SCORES = false;
} else {
throw new IllegalArgumentException(ES_ENFORCE_POSITIVE_SCORES + " may only be unset or set to [false], but got [" +
enforcePositiveScores + "]");
}
}
private final Similarity in;
public NonNegativeScoresSimilarity(Similarity in) {
this.in = in;
}
public Similarity getDelegate() {
return in;
}
@Override
public long computeNorm(FieldInvertState state) {
return in.computeNorm(state);
}
@Override
public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
final SimScorer inScorer = in.scorer(boost, collectionStats, termStats);
return new SimScorer() {
@Override
public float score(float freq, long norm) {
float score = inScorer.score(freq, norm);
if (score < 0f) {
if (ENFORCE_POSITIVE_SCORES) {
throw new IllegalArgumentException("Similarities must not produce negative scores, but got:\n" +
inScorer.explain(Explanation.match(freq, "term frequency"), norm));
} else {
return 0f;
}
}
return score;
}
@Override
public Explanation explain(Explanation freq, long norm) {
Explanation expl = inScorer.explain(freq, norm);
if (expl.isMatch() && expl.getValue().floatValue() < 0) {
expl = Explanation.match(0f, "max of:",
expl, Explanation.match(0f, "Minimum allowed score"));
}
return expl;
}
};
}
}

97
server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
View File

@ -19,15 +19,22 @@
package org.elasticsearch.index.similarity;
import org.apache.logging.log4j.LogManager;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.BooleanSimilarity;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.common.TriFunction;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.IndexModule;
@ -44,7 +51,7 @@ import java.util.function.Supplier;
public final class SimilarityService extends AbstractIndexComponent {
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(SimilarityService.class));
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(SimilarityService.class));
public static final String DEFAULT_SIMILARITY = "BM25";
private static final String CLASSIC_SIMILARITY = "classic";
private static final Map<String, Function<Version, Supplier<Similarity>>> DEFAULTS;
@ -131,8 +138,14 @@ public final class SimilarityService extends AbstractIndexComponent {
}
TriFunction<Settings, Version, ScriptService, Similarity> defaultFactory = BUILT_IN.get(typeName);
TriFunction<Settings, Version, ScriptService, Similarity> factory = similarities.getOrDefault(typeName, defaultFactory);
final Similarity similarity = factory.apply(providerSettings, indexSettings.getIndexVersionCreated(), scriptService);
providers.put(name, () -> similarity);
Similarity similarity = factory.apply(providerSettings, indexSettings.getIndexVersionCreated(), scriptService);
validateSimilarity(indexSettings.getIndexVersionCreated(), similarity);
if (BUILT_IN.containsKey(typeName) == false || "scripted".equals(typeName)) {
// We don't trust custom similarities
similarity = new NonNegativeScoresSimilarity(similarity);
}
final Similarity similarityF = similarity; // like similarity but final
providers.put(name, () -> similarityF);
}
for (Map.Entry<String, Function<Version, Supplier<Similarity>>> entry : DEFAULTS.entrySet()) {
providers.put(entry.getKey(), entry.getValue().apply(indexSettings.getIndexVersionCreated()));
@ -182,4 +195,80 @@ public final class SimilarityService extends AbstractIndexComponent {
return (fieldType != null && fieldType.similarity() != null) ? fieldType.similarity().get() : defaultSimilarity;
}
}
static void validateSimilarity(Version indexCreatedVersion, Similarity similarity) {
validateScoresArePositive(indexCreatedVersion, similarity);
validateScoresDoNotDecreaseWithFreq(indexCreatedVersion, similarity);
validateScoresDoNotIncreaseWithNorm(indexCreatedVersion, similarity);
}
private static void validateScoresArePositive(Version indexCreatedVersion, Similarity similarity) {
CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000);
TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130);
SimScorer scorer = similarity.scorer(2f, collectionStats, termStats);
FieldInvertState state = new FieldInvertState(indexCreatedVersion.major, "some_field",
IndexOptions.DOCS_AND_FREQS, 20, 20, 0, 50, 10, 3); // length = 20, no overlap
final long norm = similarity.computeNorm(state);
for (int freq = 1; freq <= 10; ++freq) {
float score = scorer.score(freq, norm);
if (score < 0) {
fail(indexCreatedVersion, "Similarities should not return negative scores:\n" +
scorer.explain(Explanation.match(freq, "term freq"), norm));
}
}
}
private static void validateScoresDoNotDecreaseWithFreq(Version indexCreatedVersion, Similarity similarity) {
CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000);
TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130);
SimScorer scorer = similarity.scorer(2f, collectionStats, termStats);
FieldInvertState state = new FieldInvertState(indexCreatedVersion.major, "some_field",
IndexOptions.DOCS_AND_FREQS, 20, 20, 0, 50, 10, 3); // length = 20, no overlap
final long norm = similarity.computeNorm(state);
float previousScore = 0;
for (int freq = 1; freq <= 10; ++freq) {
float score = scorer.score(freq, norm);
if (score < previousScore) {
fail(indexCreatedVersion, "Similarity scores should not decrease when term frequency increases:\n" +
scorer.explain(Explanation.match(freq - 1, "term freq"), norm) + "\n" +
scorer.explain(Explanation.match(freq, "term freq"), norm));
}
previousScore = score;
}
}
private static void validateScoresDoNotIncreaseWithNorm(Version indexCreatedVersion, Similarity similarity) {
CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000);
TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130);
SimScorer scorer = similarity.scorer(2f, collectionStats, termStats);
long previousNorm = 0;
float previousScore = Float.MAX_VALUE;
for (int length = 1; length <= 10; ++length) {
FieldInvertState state = new FieldInvertState(indexCreatedVersion.major, "some_field",
IndexOptions.DOCS_AND_FREQS, length, length, 0, 50, 10, 3); // length = 20, no overlap
final long norm = similarity.computeNorm(state);
if (Long.compareUnsigned(previousNorm, norm) > 0) {
// esoteric similarity, skip this check
break;
}
float score = scorer.score(1, norm);
if (score > previousScore) {
fail(indexCreatedVersion, "Similarity scores should not increase when norm increases:\n" +
scorer.explain(Explanation.match(1, "term freq"), norm - 1) + "\n" +
scorer.explain(Explanation.match(1, "term freq"), norm));
}
previousScore = score;
previousNorm = norm;
}
}
private static void fail(Version indexCreatedVersion, String message) {
if (indexCreatedVersion.onOrAfter(Version.V_7_0_0_alpha1)) {
throw new IllegalArgumentException(message);
} else if (indexCreatedVersion.onOrAfter(Version.V_6_5_0)) {
DEPRECATION_LOGGER.deprecated(message);
}
}
}

3
server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java
View File

@ -144,7 +144,6 @@ final class TranslogHeader {
final long primaryTerm;
if (version == VERSION_PRIMARY_TERM) {
primaryTerm = in.readLong();
assert primaryTerm >= 0 : "Primary term must be non-negative [" + primaryTerm + "]; translog path [" + path + "]";
} else {
assert version == VERSION_CHECKPOINTS : "Unknown header version [" + version + "]";
primaryTerm = UNKNOWN_PRIMARY_TERM;
@ -153,6 +152,8 @@ final class TranslogHeader {
if (version >= VERSION_PRIMARY_TERM) {
Translog.verifyChecksum(in);
}
assert primaryTerm >= 0 : "Primary term must be non-negative [" + primaryTerm + "]; translog path [" + path + "]";
final int headerSizeInBytes = headerSizeInBytes(version, uuid.length);
assert channel.position() == headerSizeInBytes :
"Header is not fully read; header size [" + headerSizeInBytes + "], position [" + channel.position() + "]";

5
server/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java
View File

@ -21,15 +21,18 @@ package org.elasticsearch.index.translog;
import org.elasticsearch.cli.LoggingAwareMultiCommand;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.index.shard.RemoveCorruptedShardDataCommand;
/**
* Class encapsulating and dispatching commands from the {@code elasticsearch-translog} command line tool
*/
@Deprecated
public class TranslogToolCli extends LoggingAwareMultiCommand {
private TranslogToolCli() {
// that's only for 6.x branch for bwc with elasticsearch-translog
super("A CLI tool for various Elasticsearch translog actions");
subcommands.put("truncate", new TruncateTranslogCommand());
subcommands.put("truncate", new RemoveCorruptedShardDataCommand(true));
}
public static void main(String[] args) throws Exception {

245
server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java Normal file
View File

@ -0,0 +1,245 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.translog;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.seqno.SequenceNumbers;
import org.elasticsearch.index.shard.RemoveCorruptedShardDataCommand;
import org.elasticsearch.index.shard.ShardPath;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
public class TruncateTranslogAction {
protected static final Logger logger = Loggers.getLogger(TruncateTranslogAction.class);
private final NamedXContentRegistry namedXContentRegistry;
public TruncateTranslogAction(NamedXContentRegistry namedXContentRegistry) {
this.namedXContentRegistry = namedXContentRegistry;
}
public Tuple<RemoveCorruptedShardDataCommand.CleanStatus, String> getCleanStatus(ShardPath shardPath,
Directory indexDirectory) throws IOException {
final Path indexPath = shardPath.resolveIndex();
final Path translogPath = shardPath.resolveTranslog();
final List<IndexCommit> commits;
try {
commits = DirectoryReader.listCommits(indexDirectory);
} catch (IndexNotFoundException infe) {
throw new ElasticsearchException("unable to find a valid shard at [" + indexPath + "]", infe);
}
// Retrieve the generation and UUID from the existing data
final Map<String, String> commitData = new HashMap<>(commits.get(commits.size() - 1).getUserData());
final String translogUUID = commitData.get(Translog.TRANSLOG_UUID_KEY);
if (translogUUID == null) {
throw new ElasticsearchException("shard must have a valid translog UUID but got: [null]");
}
final boolean clean = isTranslogClean(shardPath, translogUUID);
if (clean) {
return Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.CLEAN, null);
}
// Hold the lock open for the duration of the tool running
Set<Path> translogFiles;
try {
translogFiles = filesInDirectory(translogPath);
} catch (IOException e) {
throw new ElasticsearchException("failed to find existing translog files", e);
}
final String details = deletingFilesDetails(translogPath, translogFiles);
return Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.CORRUPTED, details);
}
public void execute(Terminal terminal, ShardPath shardPath, Directory indexDirectory) throws IOException {
final Path indexPath = shardPath.resolveIndex();
final Path translogPath = shardPath.resolveTranslog();
final String historyUUID = UUIDs.randomBase64UUID();
final Map<String, String> commitData;
// Hold the lock open for the duration of the tool running
Set<Path> translogFiles;
try {
terminal.println("Checking existing translog files");
translogFiles = filesInDirectory(translogPath);
} catch (IOException e) {
terminal.println("encountered IOException while listing directory, aborting...");
throw new ElasticsearchException("failed to find existing translog files", e);
}
List<IndexCommit> commits;
try {
terminal.println("Reading translog UUID information from Lucene commit from shard at [" + indexPath + "]");
commits = DirectoryReader.listCommits(indexDirectory);
} catch (IndexNotFoundException infe) {
throw new ElasticsearchException("unable to find a valid shard at [" + indexPath + "]", infe);
}
// Retrieve the generation and UUID from the existing data
commitData = commits.get(commits.size() - 1).getUserData();
final String translogGeneration = commitData.get(Translog.TRANSLOG_GENERATION_KEY);
final String translogUUID = commitData.get(Translog.TRANSLOG_UUID_KEY);
if (translogGeneration == null || translogUUID == null) {
throw new ElasticsearchException("shard must have a valid translog generation and UUID but got: [{}] and: [{}]",
translogGeneration, translogUUID);
}
final long globalCheckpoint = commitData.containsKey(SequenceNumbers.MAX_SEQ_NO)
? Long.parseLong(commitData.get(SequenceNumbers.MAX_SEQ_NO))
: SequenceNumbers.UNASSIGNED_SEQ_NO;
terminal.println("Translog Generation: " + translogGeneration);
terminal.println("Translog UUID : " + translogUUID);
terminal.println("History UUID : " + historyUUID);
Path tempEmptyCheckpoint = translogPath.resolve("temp-" + Translog.CHECKPOINT_FILE_NAME);
Path realEmptyCheckpoint = translogPath.resolve(Translog.CHECKPOINT_FILE_NAME);
Path tempEmptyTranslog = translogPath.resolve("temp-" + Translog.TRANSLOG_FILE_PREFIX +
translogGeneration + Translog.TRANSLOG_FILE_SUFFIX);
Path realEmptyTranslog = translogPath.resolve(Translog.TRANSLOG_FILE_PREFIX +
translogGeneration + Translog.TRANSLOG_FILE_SUFFIX);
// Write empty checkpoint and translog to empty files
long gen = Long.parseLong(translogGeneration);
int translogLen = writeEmptyTranslog(tempEmptyTranslog, translogUUID);
writeEmptyCheckpoint(tempEmptyCheckpoint, translogLen, gen, globalCheckpoint);
terminal.println("Removing existing translog files");
IOUtils.rm(translogFiles.toArray(new Path[]{}));
terminal.println("Creating new empty checkpoint at [" + realEmptyCheckpoint + "]");
Files.move(tempEmptyCheckpoint, realEmptyCheckpoint, StandardCopyOption.ATOMIC_MOVE);
terminal.println("Creating new empty translog at [" + realEmptyTranslog + "]");
Files.move(tempEmptyTranslog, realEmptyTranslog, StandardCopyOption.ATOMIC_MOVE);
// Fsync the translog directory after rename
IOUtils.fsync(translogPath, true);
}
private boolean isTranslogClean(ShardPath shardPath, String translogUUID) throws IOException {
// perform clean check of translog instead of corrupted marker file
boolean clean = true;
try {
final Path translogPath = shardPath.resolveTranslog();
final long translogGlobalCheckpoint = Translog.readGlobalCheckpoint(translogPath, translogUUID);
final IndexMetaData indexMetaData =
IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, shardPath.getDataPath().getParent());
final IndexSettings indexSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
final TranslogConfig translogConfig = new TranslogConfig(shardPath.getShardId(), translogPath,
indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
long primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardPath.getShardId().id());
final TranslogDeletionPolicy translogDeletionPolicy =
new TranslogDeletionPolicy(indexSettings.getTranslogRetentionSize().getBytes(),
indexSettings.getTranslogRetentionAge().getMillis());
try (Translog translog = new Translog(translogConfig, translogUUID,
translogDeletionPolicy, () -> translogGlobalCheckpoint, () -> primaryTerm);
Translog.Snapshot snapshot = translog.newSnapshot()) {
while (snapshot.next() != null) {
// just iterate over snapshot
}
}
} catch (TranslogCorruptedException e) {
clean = false;
}
return clean;
}
/** Write a checkpoint file to the given location with the given generation */
static void writeEmptyCheckpoint(Path filename, int translogLength, long translogGeneration, long globalCheckpoint) throws IOException {
Checkpoint emptyCheckpoint = Checkpoint.emptyTranslogCheckpoint(translogLength, translogGeneration,
globalCheckpoint, translogGeneration);
Checkpoint.write(FileChannel::open, filename, emptyCheckpoint,
StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW);
// fsync with metadata here to make sure.
IOUtils.fsync(filename, false);
}
/**
* Write a translog containing the given translog UUID to the given location. Returns the number of bytes written.
*/
private static int writeEmptyTranslog(Path filename, String translogUUID) throws IOException {
try (FileChannel fc = FileChannel.open(filename, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)) {
TranslogHeader header = new TranslogHeader(translogUUID, TranslogHeader.UNKNOWN_PRIMARY_TERM);
header.write(fc);
return header.sizeInBytes();
}
}
/** Show a warning about deleting files, asking for a confirmation if {@code batchMode} is false */
private String deletingFilesDetails(Path translogPath, Set<Path> files) {
StringBuilder builder = new StringBuilder();
builder
.append("Documents inside of translog files will be lost.\n")
.append(" The following files will be DELETED at ")
.append(translogPath)
.append("\n\n");
for(Iterator<Path> it = files.iterator();it.hasNext();) {
builder.append(" --> ").append(it.next().getFileName());
if (it.hasNext()) {
builder.append("\n");
}
}
return builder.toString();
}
/** Return a Set of all files in a given directory */
public static Set<Path> filesInDirectory(Path directory) throws IOException {
Set<Path> files = new TreeSet<>();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory)) {
for (Path file : stream) {
files.add(file);
}
}
return files;
}
}

254
server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java
View File

@ -1,254 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.translog;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.NativeFSLockFactory;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cli.EnvironmentAwareCommand;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.seqno.SequenceNumbers;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class TruncateTranslogCommand extends EnvironmentAwareCommand {
private final OptionSpec<String> translogFolder;
private final OptionSpec<Void> batchMode;
public TruncateTranslogCommand() {
super("Truncates a translog to create a new, empty translog");
this.translogFolder = parser.acceptsAll(Arrays.asList("d", "dir"),
"Translog Directory location on disk")
.withRequiredArg()
.required();
this.batchMode = parser.acceptsAll(Arrays.asList("b", "batch"),
"Enable batch mode explicitly, automatic confirmation of warnings");
}
// Visible for testing
public OptionParser getParser() {
return this.parser;
}
@Override
protected void printAdditionalHelp(Terminal terminal) {
terminal.println("This tool truncates the translog and translog");
terminal.println("checkpoint files to create a new translog");
}
@SuppressForbidden(reason = "Necessary to use the path passed in")
private Path getTranslogPath(OptionSet options) {
return PathUtils.get(translogFolder.value(options), "", "");
}
@Override
protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
boolean batch = options.has(batchMode);
Path translogPath = getTranslogPath(options);
Path idxLocation = translogPath.getParent().resolve("index");
if (Files.exists(translogPath) == false || Files.isDirectory(translogPath) == false) {
throw new ElasticsearchException("translog directory [" + translogPath + "], must exist and be a directory");
}
if (Files.exists(idxLocation) == false || Files.isDirectory(idxLocation) == false) {
throw new ElasticsearchException("unable to find a shard at [" + idxLocation + "], which must exist and be a directory");
}
try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE)) {
final String historyUUID = UUIDs.randomBase64UUID();
final Map<String, String> commitData;
// Hold the lock open for the duration of the tool running
try (Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
Set<Path> translogFiles;
try {
terminal.println("Checking existing translog files");
translogFiles = filesInDirectory(translogPath);
} catch (IOException e) {
terminal.println("encountered IOException while listing directory, aborting...");
throw new ElasticsearchException("failed to find existing translog files", e);
}
// Warn about ES being stopped and files being deleted
warnAboutDeletingFiles(terminal, translogFiles, batch);
List<IndexCommit> commits;
try {
terminal.println("Reading translog UUID information from Lucene commit from shard at [" + idxLocation + "]");
commits = DirectoryReader.listCommits(dir);
} catch (IndexNotFoundException infe) {
throw new ElasticsearchException("unable to find a valid shard at [" + idxLocation + "]", infe);
}
// Retrieve the generation and UUID from the existing data
commitData = new HashMap<>(commits.get(commits.size() - 1).getUserData());
String translogGeneration = commitData.get(Translog.TRANSLOG_GENERATION_KEY);
String translogUUID = commitData.get(Translog.TRANSLOG_UUID_KEY);
final long globalCheckpoint;
// In order to have a safe commit invariant, we have to assign the global checkpoint to the max_seqno of the last commit.
// We can only safely do it because we will generate a new history uuid this shard.
if (commitData.containsKey(SequenceNumbers.MAX_SEQ_NO)) {
globalCheckpoint = Long.parseLong(commitData.get(SequenceNumbers.MAX_SEQ_NO));
// Also advances the local checkpoint of the last commit to its max_seqno.
commitData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(globalCheckpoint));
} else {
globalCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
}
if (translogGeneration == null || translogUUID == null) {
throw new ElasticsearchException("shard must have a valid translog generation and UUID but got: [{}] and: [{}]",
translogGeneration, translogUUID);
}
terminal.println("Translog Generation: " + translogGeneration);
terminal.println("Translog UUID : " + translogUUID);
terminal.println("History UUID : " + historyUUID);
Path tempEmptyCheckpoint = translogPath.resolve("temp-" + Translog.CHECKPOINT_FILE_NAME);
Path realEmptyCheckpoint = translogPath.resolve(Translog.CHECKPOINT_FILE_NAME);
Path tempEmptyTranslog = translogPath.resolve("temp-" + Translog.TRANSLOG_FILE_PREFIX +
translogGeneration + Translog.TRANSLOG_FILE_SUFFIX);
Path realEmptyTranslog = translogPath.resolve(Translog.TRANSLOG_FILE_PREFIX +
translogGeneration + Translog.TRANSLOG_FILE_SUFFIX);
// Write empty checkpoint and translog to empty files
long gen = Long.parseLong(translogGeneration);
int translogLen = writeEmptyTranslog(tempEmptyTranslog, translogUUID);
writeEmptyCheckpoint(tempEmptyCheckpoint, translogLen, gen, globalCheckpoint);
terminal.println("Removing existing translog files");
IOUtils.rm(translogFiles.toArray(new Path[]{}));
terminal.println("Creating new empty checkpoint at [" + realEmptyCheckpoint + "]");
Files.move(tempEmptyCheckpoint, realEmptyCheckpoint, StandardCopyOption.ATOMIC_MOVE);
terminal.println("Creating new empty translog at [" + realEmptyTranslog + "]");
Files.move(tempEmptyTranslog, realEmptyTranslog, StandardCopyOption.ATOMIC_MOVE);
// Fsync the translog directory after rename
IOUtils.fsync(translogPath, true);
}
terminal.println("Marking index with the new history uuid");
// commit the new histroy id
IndexWriterConfig iwc = new IndexWriterConfig(null)
.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
.setCommitOnClose(false)
// we don't want merges to happen here - we call maybe merge on the engine
// later once we stared it up otherwise we would need to wait for it here
// we also don't specify a codec here and merges should use the engines for this index
.setMergePolicy(NoMergePolicy.INSTANCE)
.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
try (IndexWriter writer = new IndexWriter(dir, iwc)) {
Map<String, String> newCommitData = new HashMap<>(commitData);
newCommitData.put(Engine.HISTORY_UUID_KEY, historyUUID);
writer.setLiveCommitData(newCommitData.entrySet());
writer.commit();
}
} catch (LockObtainFailedException lofe) {
throw new ElasticsearchException("Failed to lock shard's directory at [" + idxLocation + "], is Elasticsearch still running?");
}
terminal.println("Done.");
}
/** Write a checkpoint file to the given location with the given generation */
static void writeEmptyCheckpoint(Path filename, int translogLength, long translogGeneration, long globalCheckpoint) throws IOException {
Checkpoint emptyCheckpoint = Checkpoint.emptyTranslogCheckpoint(translogLength, translogGeneration,
globalCheckpoint, translogGeneration);
Checkpoint.write(FileChannel::open, filename, emptyCheckpoint,
StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW);
// fsync with metadata here to make sure.
IOUtils.fsync(filename, false);
}
/**
* Write a translog containing the given translog UUID to the given location. Returns the number of bytes written.
*/
public static int writeEmptyTranslog(Path filename, String translogUUID) throws IOException {
try (FileChannel fc = FileChannel.open(filename, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)) {
TranslogHeader header = new TranslogHeader(translogUUID, TranslogHeader.UNKNOWN_PRIMARY_TERM);
header.write(fc);
return header.sizeInBytes();
}
}
/** Show a warning about deleting files, asking for a confirmation if {@code batchMode} is false */
public static void warnAboutDeletingFiles(Terminal terminal, Set<Path> files, boolean batchMode) {
terminal.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
terminal.println("! WARNING: Elasticsearch MUST be stopped before running this tool !");
terminal.println("! !");
terminal.println("! WARNING: Documents inside of translog files will be lost !");
terminal.println("! !");
terminal.println("! WARNING: The following files will be DELETED! !");
terminal.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
for (Path file : files) {
terminal.println("--> " + file);
}
terminal.println("");
if (batchMode == false) {
String text = terminal.readText("Continue and DELETE files? [y/N] ");
if (!text.equalsIgnoreCase("y")) {
throw new ElasticsearchException("aborted by user");
}
}
}
/** Return a Set of all files in a given directory */
public static Set<Path> filesInDirectory(Path directory) throws IOException {
Set<Path> files = new HashSet<>();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory)) {
for (Path file : stream) {
files.add(file);
}
}
return files;
}
}

4
server/src/main/java/org/elasticsearch/search/profile/AbstractProfileBreakdown.java
View File

@ -49,6 +49,10 @@ public abstract class AbstractProfileBreakdown<T extends Enum<T>> {
return timings[timing.ordinal()];
}
public void setTimer(T timing, Timer timer) {
timings[timing.ordinal()] = timer;
}
/** Convert this record to a map from timingType to times. */
public Map<String, Long> toTimingMap() {
Map<String, Long> map = new HashMap<>();

31
server/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
View File

@ -19,7 +19,9 @@
package org.elasticsearch.search.profile.query;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;
@ -36,7 +38,10 @@ final class ProfileScorer extends Scorer {
private final Scorer scorer;
private ProfileWeight profileWeight;
private final Timer scoreTimer, nextDocTimer, advanceTimer, matchTimer, shallowAdvanceTimer, computeMaxScoreTimer;
private final boolean isConstantScoreQuery;
ProfileScorer(ProfileWeight w, Scorer scorer, QueryProfileBreakdown profile) throws IOException {
super(w);
@ -48,6 +53,26 @@ final class ProfileScorer extends Scorer {
matchTimer = profile.getTimer(QueryTimingType.MATCH);
shallowAdvanceTimer = profile.getTimer(QueryTimingType.SHALLOW_ADVANCE);
computeMaxScoreTimer = profile.getTimer(QueryTimingType.COMPUTE_MAX_SCORE);
ProfileScorer profileScorer = null;
if (w.getQuery() instanceof ConstantScoreQuery && scorer instanceof ProfileScorer) {
//Case when we have a totalHits query and it is not cached
profileScorer = (ProfileScorer) scorer;
} else if (w.getQuery() instanceof ConstantScoreQuery && scorer.getChildren().size() == 1) {
//Case when we have a top N query. If the scorer has no children, it is because it is cached
//and in that case we do not do any special treatment
Scorable childScorer = scorer.getChildren().iterator().next().child;
if (childScorer instanceof ProfileScorer) {
profileScorer = (ProfileScorer) childScorer;
}
}
if (profileScorer != null) {
isConstantScoreQuery = true;
profile.setTimer(QueryTimingType.NEXT_DOC, profileScorer.nextDocTimer);
profile.setTimer(QueryTimingType.ADVANCE, profileScorer.advanceTimer);
profile.setTimer(QueryTimingType.MATCH, profileScorer.matchTimer);
} else {
isConstantScoreQuery = false;
}
}
@Override
@ -77,6 +102,9 @@ final class ProfileScorer extends Scorer {
@Override
public DocIdSetIterator iterator() {
if (isConstantScoreQuery) {
return scorer.iterator();
}
final DocIdSetIterator in = scorer.iterator();
return new DocIdSetIterator() {
@ -114,6 +142,9 @@ final class ProfileScorer extends Scorer {
@Override
public TwoPhaseIterator twoPhaseIterator() {
if (isConstantScoreQuery) {
return scorer.twoPhaseIterator();
}
final TwoPhaseIterator in = scorer.twoPhaseIterator();
if (in == null) {
return null;

11
server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View File

@ -59,6 +59,7 @@ import org.elasticsearch.index.shard.IndexSearcherWrapper;
import org.elasticsearch.index.shard.IndexingOperationListener;
import org.elasticsearch.index.shard.SearchOperationListener;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.similarity.NonNegativeScoresSimilarity;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.indices.IndicesModule;
@ -77,6 +78,7 @@ import org.elasticsearch.test.TestSearchContext;
import org.elasticsearch.test.engine.MockEngineFactory;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.hamcrest.Matchers;
import java.io.IOException;
import java.util.Collections;
@ -295,10 +297,13 @@ public class IndexModuleTests extends ESTestCase {
IndexService indexService = newIndexService(module);
SimilarityService similarityService = indexService.similarityService();
assertNotNull(similarityService.getSimilarity("my_similarity"));
assertTrue(similarityService.getSimilarity("my_similarity").get() instanceof TestSimilarity);
Similarity similarity = similarityService.getSimilarity("my_similarity").get();
assertNotNull(similarity);
assertThat(similarity, Matchers.instanceOf(NonNegativeScoresSimilarity.class));
similarity = ((NonNegativeScoresSimilarity) similarity).getDelegate();
assertThat(similarity, Matchers.instanceOf(TestSimilarity.class));
assertEquals("my_similarity", similarityService.getSimilarity("my_similarity").name());
assertEquals("there is a key", ((TestSimilarity) similarityService.getSimilarity("my_similarity").get()).key);
assertEquals("there is a key", ((TestSimilarity) similarity).key);
indexService.close("simon says", false);
}

34
server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View File

@ -23,7 +23,6 @@ import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
@ -2439,7 +2438,7 @@ public class IndexShardTests extends IndexShardTestCase {
closeShards(sourceShard, targetShard);
}
public void testDocStats() throws IOException, InterruptedException {
public void testDocStats() throws Exception {
IndexShard indexShard = null;
try {
indexShard = newStartedShard(
@ -2456,7 +2455,14 @@ public class IndexShardTests extends IndexShardTestCase {
indexShard.flush(new FlushRequest());
}
{
IndexShard shard = indexShard;
assertBusy(() -> {
ThreadPool threadPool = shard.getThreadPool();
assertThat(threadPool.relativeTimeInMillis(), greaterThan(shard.getLastSearcherAccess()));
});
long prevAccessTime = shard.getLastSearcherAccess();
final DocsStats docsStats = indexShard.docStats();
assertThat("searcher was not marked as accessed", shard.getLastSearcherAccess(), greaterThan(prevAccessTime));
assertThat(docsStats.getCount(), equalTo(numDocs));
try (Engine.Searcher searcher = indexShard.acquireSearcher("test")) {
assertTrue(searcher.reader().numDocs() <= docsStats.getCount());
@ -2641,7 +2647,8 @@ public class IndexShardTests extends IndexShardTestCase {
final ShardPath shardPath = indexShard.shardPath();
final Path indexPath = corruptIndexFile(shardPath);
final Path indexPath = shardPath.getDataPath().resolve(ShardPath.INDEX_FOLDER_NAME);
CorruptionUtils.corruptIndex(random(), indexPath, false);
final AtomicInteger corruptedMarkerCount = new AtomicInteger();
final SimpleFileVisitor<Path> corruptedVisitor = new SimpleFileVisitor<Path>() {
@ -2750,22 +2757,6 @@ public class IndexShardTests extends IndexShardTestCase {
assertThat("store still has a single corrupt marker", corruptedMarkerCount.get(), equalTo(1));
}
private Path corruptIndexFile(ShardPath shardPath) throws IOException {
final Path indexPath = shardPath.getDataPath().resolve(ShardPath.INDEX_FOLDER_NAME);
final Path[] filesToCorrupt =
Files.walk(indexPath)
.filter(p -> {
final String name = p.getFileName().toString();
return Files.isRegularFile(p)
&& name.startsWith("extra") == false // Skip files added by Lucene's ExtrasFS
&& IndexWriter.WRITE_LOCK_NAME.equals(name) == false
&& name.startsWith("segments_") == false && name.endsWith(".si") == false;
})
.toArray(Path[]::new);
CorruptionUtils.corruptFile(random(), filesToCorrupt);
return indexPath;
}
/**
* Simulates a scenario that happens when we are async fetching snapshot metadata from GatewayService
* and checking index concurrently. This should always be possible without any exception.
@ -3428,4 +3419,9 @@ public class IndexShardTests extends IndexShardTestCase {
assertThat(shard.translogStats().estimatedNumberOfOperations(), equalTo(translogStats.estimatedNumberOfOperations()));
closeShard(shard, false);
}
@Override
public Settings threadPoolSettings() {
return Settings.builder().put(super.threadPoolSettings()).put("thread_pool.estimated_time_interval", "5ms").build();
}
}

652
server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java Normal file
View File

@ -0,0 +1,652 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.shard;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.NativeFSLockFactory;
import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.cli.MockTerminal;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.ShardIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.MergePolicyConfig;
import org.elasticsearch.index.MockEngineFactoryPlugin;
import org.elasticsearch.index.seqno.SeqNoStats;
import org.elasticsearch.index.translog.TestTranslog;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.CorruptionUtils;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
import org.elasticsearch.test.InternalTestCluster;
import org.elasticsearch.test.engine.MockEngineSupport;
import org.elasticsearch.test.transport.MockTransportService;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.startsWith;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0)
public class RemoveCorruptedShardDataCommandIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(MockTransportService.TestPlugin.class, MockEngineFactoryPlugin.class, InternalSettingsPlugin.class);
}
public void testCorruptIndex() throws Exception {
final String node = internalCluster().startNode();
final String indexName = "index42";
assertAcked(prepareCreate(indexName).setSettings(Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")
.put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true)
.put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), "checksum")
));
// index some docs in several segments
int numDocs = 0;
for (int k = 0, attempts = randomIntBetween(5, 10); k < attempts; k++) {
final int numExtraDocs = between(10, 100);
IndexRequestBuilder[] builders = new IndexRequestBuilder[numExtraDocs];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex(indexName, "type").setSource("foo", "bar");
}
numDocs += numExtraDocs;
indexRandom(false, false, false, Arrays.asList(builders));
flush(indexName);
}
logger.info("--> indexed {} docs", numDocs);
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
final MockTerminal terminal = new MockTerminal();
final OptionParser parser = command.getParser();
final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
final OptionSet options = parser.parse("-index", indexName, "-shard-id", "0");
// Try running it before the node is stopped (and shard is closed)
try {
command.execute(terminal, options, environment);
fail("expected the command to fail as node is locked");
} catch (Exception e) {
assertThat(e.getMessage(),
allOf(containsString("Failed to lock node's directory"),
containsString("is Elasticsearch still running ?")));
}
final Set<Path> indexDirs = getDirs(indexName, ShardPath.INDEX_FOLDER_NAME);
assertThat(indexDirs, hasSize(1));
internalCluster().restartNode(node, new InternalTestCluster.RestartCallback() {
@Override
public Settings onNodeStopped(String nodeName) throws Exception {
// Try running it before the shard is corrupted, it should flip out because there is no corruption file marker
try {
command.execute(terminal, options, environment);
fail("expected the command to fail as there is no corruption file marker");
} catch (Exception e) {
assertThat(e.getMessage(), startsWith("Shard does not seem to be corrupted at"));
}
CorruptionUtils.corruptIndex(random(), indexDirs.iterator().next(), false);
return super.onNodeStopped(nodeName);
}
});
// shard should be failed due to a corrupted index
assertBusy(() -> {
final ClusterAllocationExplanation explanation =
client().admin().cluster().prepareAllocationExplain()
.setIndex(indexName).setShard(0).setPrimary(true)
.get().getExplanation();
final ShardAllocationDecision shardAllocationDecision = explanation.getShardAllocationDecision();
assertThat(shardAllocationDecision.isDecisionTaken(), equalTo(true));
assertThat(shardAllocationDecision.getAllocateDecision().getAllocationDecision(),
equalTo(AllocationDecision.NO_VALID_SHARD_COPY));
});
internalCluster().restartNode(node, new InternalTestCluster.RestartCallback() {
@Override
public Settings onNodeStopped(String nodeName) throws Exception {
terminal.addTextInput("y");
command.execute(terminal, options, environment);
return super.onNodeStopped(nodeName);
}
});
waitNoPendingTasksOnAll();
String nodeId = null;
final ClusterState state = client().admin().cluster().prepareState().get().getState();
final DiscoveryNodes nodes = state.nodes();
for (ObjectObjectCursor<String, DiscoveryNode> cursor : nodes.getNodes()) {
final String name = cursor.value.getName();
if (name.equals(node)) {
nodeId = cursor.key;
break;
}
}
assertThat(nodeId, notNullValue());
logger.info("--> output:\n{}", terminal.getOutput());
assertThat(terminal.getOutput(), containsString("allocate_stale_primary"));
assertThat(terminal.getOutput(), containsString("\"node\" : \"" + nodeId + "\""));
// there is only _stale_ primary (due to new allocation id)
assertBusy(() -> {
final ClusterAllocationExplanation explanation =
client().admin().cluster().prepareAllocationExplain()
.setIndex(indexName).setShard(0).setPrimary(true)
.get().getExplanation();
final ShardAllocationDecision shardAllocationDecision = explanation.getShardAllocationDecision();
assertThat(shardAllocationDecision.isDecisionTaken(), equalTo(true));
assertThat(shardAllocationDecision.getAllocateDecision().getAllocationDecision(),
equalTo(AllocationDecision.NO_VALID_SHARD_COPY));
});
client().admin().cluster().prepareReroute()
.add(new AllocateStalePrimaryAllocationCommand(indexName, 0, nodeId, true))
.get();
assertBusy(() -> {
final ClusterAllocationExplanation explanation =
client().admin().cluster().prepareAllocationExplain()
.setIndex(indexName).setShard(0).setPrimary(true)
.get().getExplanation();
assertThat(explanation.getCurrentNode(), notNullValue());
assertThat(explanation.getShardState(), equalTo(ShardRoutingState.STARTED));
});
final Pattern pattern = Pattern.compile("Corrupted Lucene index segments found -\\s+(?<docs>\\d+) documents will be lost.");
final Matcher matcher = pattern.matcher(terminal.getOutput());
assertThat(matcher.find(), equalTo(true));
final int expectedNumDocs = numDocs - Integer.parseInt(matcher.group("docs"));
ensureGreen(indexName);
assertHitCount(client().prepareSearch(indexName).setQuery(matchAllQuery()).get(), expectedNumDocs);
}
public void testCorruptTranslogTruncation() throws Exception {
internalCluster().startNodes(2, Settings.EMPTY);
final String node1 = internalCluster().getNodeNames()[0];
final String node2 = internalCluster().getNodeNames()[1];
final String indexName = "test";
assertAcked(prepareCreate(indexName).setSettings(Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")
.put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true) // never flush - always recover from translog
.put("index.routing.allocation.exclude._name", node2)
));
ensureYellow();
assertAcked(client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder()
.put("index.routing.allocation.exclude._name", (String)null)
));
ensureGreen();
// Index some documents
int numDocsToKeep = randomIntBetween(10, 100);
logger.info("--> indexing [{}] docs to be kept", numDocsToKeep);
IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocsToKeep];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex(indexName, "type").setSource("foo", "bar");
}
indexRandom(false, false, false, Arrays.asList(builders));
flush(indexName);
disableTranslogFlush(indexName);
// having no extra docs is an interesting case for seq no based recoveries - test it more often
int numDocsToTruncate = randomBoolean() ? 0 : randomIntBetween(0, 100);
logger.info("--> indexing [{}] more doc to be truncated", numDocsToTruncate);
builders = new IndexRequestBuilder[numDocsToTruncate];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex(indexName, "type").setSource("foo", "bar");
}
indexRandom(false, false, false, Arrays.asList(builders));
Set<Path> translogDirs = getDirs(indexName, ShardPath.TRANSLOG_FOLDER_NAME);
// that's only for 6.x branch for bwc with elasticsearch-translog
final boolean translogOnly = randomBoolean();
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand(translogOnly);
final MockTerminal terminal = new MockTerminal();
final OptionParser parser = command.getParser();
if (randomBoolean() && numDocsToTruncate > 0) {
// flush the replica, so it will have more docs than what the primary will have
Index index = resolveIndex(indexName);
IndexShard replica = internalCluster().getInstance(IndicesService.class, node2).getShardOrNull(new ShardId(index, 0));
replica.flush(new FlushRequest());
logger.info("--> performed extra flushing on replica");
}
// shut down the replica node to be tested later
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node2));
// Corrupt the translog file(s)
logger.info("--> corrupting translog");
corruptRandomTranslogFiles(indexName);
// Restart the single node
logger.info("--> restarting node");
internalCluster().restartRandomDataNode();
// all shards should be failed due to a corrupted translog
assertBusy(() -> {
final ClusterAllocationExplanation explanation =
client().admin().cluster().prepareAllocationExplain()
.setIndex(indexName).setShard(0).setPrimary(true)
.get().getExplanation();
final UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
assertThat(unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
});
// have to shut down primary node - otherwise node lock is present
final InternalTestCluster.RestartCallback callback =
new InternalTestCluster.RestartCallback() {
@Override
public Settings onNodeStopped(String nodeName) throws Exception {
// and we can actually truncate the translog
for (Path translogDir : translogDirs) {
final Path idxLocation = translogDir.getParent().resolve(ShardPath.INDEX_FOLDER_NAME);
assertBusy(() -> {
logger.info("--> checking that lock has been released for {}", idxLocation);
try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE);
Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
// Great, do nothing, we just wanted to obtain the lock
} catch (LockObtainFailedException lofe) {
logger.info("--> failed acquiring lock for {}", idxLocation);
fail("still waiting for lock release at [" + idxLocation + "]");
} catch (IOException ioe) {
fail("Got an IOException: " + ioe);
}
});
final Settings defaultSettings = internalCluster().getDefaultSettings();
final Environment environment = TestEnvironment.newEnvironment(defaultSettings);
terminal.addTextInput("y");
OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString());
logger.info("--> running command for [{}]", translogDir.toAbsolutePath());
command.execute(terminal, options, environment);
logger.info("--> output:\n{}", terminal.getOutput());
}
return super.onNodeStopped(nodeName);
}
};
internalCluster().restartNode(node1, callback);
String primaryNodeId = null;
final ClusterState state = client().admin().cluster().prepareState().get().getState();
final DiscoveryNodes nodes = state.nodes();
for (ObjectObjectCursor<String, DiscoveryNode> cursor : nodes.getNodes()) {
final String name = cursor.value.getName();
if (name.equals(node1)) {
primaryNodeId = cursor.key;
break;
}
}
assertThat(primaryNodeId, notNullValue());
assertThat(terminal.getOutput(), containsString("allocate_stale_primary"));
assertThat(terminal.getOutput(), containsString("\"node\" : \"" + primaryNodeId + "\""));
// there is only _stale_ primary (due to new allocation id)
assertBusy(() -> {
final ClusterAllocationExplanation explanation =
client().admin().cluster().prepareAllocationExplain()
.setIndex(indexName).setShard(0).setPrimary(true)
.get().getExplanation();
final ShardAllocationDecision shardAllocationDecision = explanation.getShardAllocationDecision();
assertThat(shardAllocationDecision.isDecisionTaken(), equalTo(true));
assertThat(shardAllocationDecision.getAllocateDecision().getAllocationDecision(),
equalTo(AllocationDecision.NO_VALID_SHARD_COPY));
});
client().admin().cluster().prepareReroute()
.add(new AllocateStalePrimaryAllocationCommand(indexName, 0, primaryNodeId, true))
.get();
assertBusy(() -> {
final ClusterAllocationExplanation explanation =
client().admin().cluster().prepareAllocationExplain()
.setIndex(indexName).setShard(0).setPrimary(true)
.get().getExplanation();
assertThat(explanation.getCurrentNode(), notNullValue());
assertThat(explanation.getShardState(), equalTo(ShardRoutingState.STARTED));
});
ensureYellow(indexName);
// Run a search and make sure it succeeds
assertHitCount(client().prepareSearch(indexName).setQuery(matchAllQuery()).get(), numDocsToKeep);
logger.info("--> starting the replica node to test recovery");
internalCluster().startNode();
ensureGreen(indexName);
for (String node : internalCluster().nodesInclude(indexName)) {
SearchRequestBuilder q = client().prepareSearch(indexName).setPreference("_only_nodes:" + node).setQuery(matchAllQuery());
assertHitCount(q.get(), numDocsToKeep);
}
final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).setActiveOnly(false).get();
final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get(indexName).stream()
.filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get();
assertThat(replicaRecoveryState.getIndex().toString(), replicaRecoveryState.getIndex().recoveredFileCount(), greaterThan(0));
// Ensure that the global checkpoint and local checkpoint are restored from the max seqno of the last commit.
final SeqNoStats seqNoStats = getSeqNoStats(indexName, 0);
assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
assertThat(seqNoStats.getLocalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
}
public void testCorruptTranslogTruncationOfReplica() throws Exception {
internalCluster().startNodes(2, Settings.EMPTY);
final String node1 = internalCluster().getNodeNames()[0];
final String node2 = internalCluster().getNodeNames()[1];
logger.info("--> nodes name: {}, {}", node1, node2);
final String indexName = "test";
assertAcked(prepareCreate(indexName).setSettings(Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")
.put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true) // never flush - always recover from translog
.put("index.routing.allocation.exclude._name", node2)
));
ensureYellow();
assertAcked(client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder()
.put("index.routing.allocation.exclude._name", (String)null)
));
ensureGreen();
// Index some documents
int numDocsToKeep = randomIntBetween(0, 100);
logger.info("--> indexing [{}] docs to be kept", numDocsToKeep);
IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocsToKeep];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex(indexName, "type").setSource("foo", "bar");
}
indexRandom(false, false, false, Arrays.asList(builders));
flush(indexName);
disableTranslogFlush(indexName);
// having no extra docs is an interesting case for seq no based recoveries - test it more often
int numDocsToTruncate = randomBoolean() ? 0 : randomIntBetween(0, 100);
logger.info("--> indexing [{}] more docs to be truncated", numDocsToTruncate);
builders = new IndexRequestBuilder[numDocsToTruncate];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex(indexName, "type").setSource("foo", "bar");
}
indexRandom(false, false, false, Arrays.asList(builders));
final int totalDocs = numDocsToKeep + numDocsToTruncate;
// sample the replica node translog dirs
final ShardId shardId = new ShardId(resolveIndex(indexName), 0);
final Set<Path> translogDirs = getDirs(node2, shardId, ShardPath.TRANSLOG_FOLDER_NAME);
// stop the cluster nodes. we don't use full restart so the node start up order will be the same
// and shard roles will be maintained
internalCluster().stopRandomDataNode();
internalCluster().stopRandomDataNode();
// Corrupt the translog file(s)
logger.info("--> corrupting translog");
TestTranslog.corruptRandomTranslogFile(logger, random(), translogDirs);
// Restart the single node
logger.info("--> starting node");
internalCluster().startNode();
ensureYellow();
// Run a search and make sure it succeeds
assertHitCount(client().prepareSearch(indexName).setQuery(matchAllQuery()).get(), totalDocs);
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
final MockTerminal terminal = new MockTerminal();
final OptionParser parser = command.getParser();
final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {
@Override
public Settings onNodeStopped(String nodeName) throws Exception {
logger.info("--> node {} stopped", nodeName);
for (Path translogDir : translogDirs) {
final Path idxLocation = translogDir.getParent().resolve(ShardPath.INDEX_FOLDER_NAME);
assertBusy(() -> {
logger.info("--> checking that lock has been released for {}", idxLocation);
try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE);
Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
// Great, do nothing, we just wanted to obtain the lock
} catch (LockObtainFailedException lofe) {
logger.info("--> failed acquiring lock for {}", idxLocation);
fail("still waiting for lock release at [" + idxLocation + "]");
} catch (IOException ioe) {
fail("Got an IOException: " + ioe);
}
});
terminal.addTextInput("y");
OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString());
logger.info("--> running command for [{}]", translogDir.toAbsolutePath());
command.execute(terminal, options, environment);
logger.info("--> output:\n{}", terminal.getOutput());
}
return super.onNodeStopped(nodeName);
}
});
logger.info("--> starting the replica node to test recovery");
internalCluster().startNode();
ensureGreen(indexName);
for (String node : internalCluster().nodesInclude(indexName)) {
assertHitCount(client().prepareSearch(indexName)
.setPreference("_only_nodes:" + node).setQuery(matchAllQuery()).get(), totalDocs);
}
final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).setActiveOnly(false).get();
final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get(indexName).stream()
.filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get();
// the replica translog was disabled so it doesn't know what hte global checkpoint is and thus can't do ops based recovery
assertThat(replicaRecoveryState.getIndex().toString(), replicaRecoveryState.getIndex().recoveredFileCount(), greaterThan(0));
// Ensure that the global checkpoint and local checkpoint are restored from the max seqno of the last commit.
final SeqNoStats seqNoStats = getSeqNoStats(indexName, 0);
assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
assertThat(seqNoStats.getLocalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
}
public void testResolvePath() throws Exception {
final int numOfNodes = randomIntBetween(1, 5);
final List<String> nodeNames = internalCluster().startNodes(numOfNodes, Settings.EMPTY);
final String indexName = "test" + randomInt(100);
assertAcked(prepareCreate(indexName).setSettings(Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numOfNodes - 1)
));
flush(indexName);
ensureGreen(indexName);
final Map<String, String> nodeNameToNodeId = new HashMap<>();
final ClusterState state = client().admin().cluster().prepareState().get().getState();
final DiscoveryNodes nodes = state.nodes();
for (ObjectObjectCursor<String, DiscoveryNode> cursor : nodes.getNodes()) {
nodeNameToNodeId.put(cursor.value.getName(), cursor.key);
}
final GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{indexName}, false);
final List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
final ShardRouting shardRouting = iterators.iterator().next().nextOrNull();
assertThat(shardRouting, notNullValue());
final ShardId shardId = shardRouting.shardId();
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
final OptionParser parser = command.getParser();
final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
final Map<String, Path> indexPathByNodeName = new HashMap<>();
for (String nodeName : nodeNames) {
final String nodeId = nodeNameToNodeId.get(nodeName);
final Set<Path> indexDirs = getDirs(nodeId, shardId, ShardPath.INDEX_FOLDER_NAME);
assertThat(indexDirs, hasSize(1));
indexPathByNodeName.put(nodeName, indexDirs.iterator().next());
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeName));
logger.info(" -- stopped {}", nodeName);
}
for (String nodeName : nodeNames) {
final Path indexPath = indexPathByNodeName.get(nodeName);
final OptionSet options = parser.parse("--dir", indexPath.toAbsolutePath().toString());
command.findAndProcessShardPath(options, environment,
shardPath -> assertThat(shardPath.resolveIndex(), equalTo(indexPath)));
}
}
private Set<Path> getDirs(String indexName, String dirSuffix) {
ClusterState state = client().admin().cluster().prepareState().get().getState();
GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{indexName}, false);
List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
ShardRouting shardRouting = shardIterator.nextOrNull();
assertNotNull(shardRouting);
assertTrue(shardRouting.primary());
assertTrue(shardRouting.assignedToNode());
String nodeId = shardRouting.currentNodeId();
ShardId shardId = shardRouting.shardId();
return getDirs(nodeId, shardId, dirSuffix);
}
private Set<Path> getDirs(String nodeId, ShardId shardId, String dirSuffix) {
final NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
final Set<Path> translogDirs = new TreeSet<>();
final NodeStats nodeStats = nodeStatses.getNodes().get(0);
for (FsInfo.Path fsPath : nodeStats.getFs()) {
final String path = fsPath.getPath();
final Path p = PathUtils.get(path)
.resolve(NodeEnvironment.INDICES_FOLDER)
.resolve(shardId.getIndex().getUUID())
.resolve(Integer.toString(shardId.getId()))
.resolve(dirSuffix);
if (Files.isDirectory(p)) {
translogDirs.add(p);
}
}
return translogDirs;
}
private void corruptRandomTranslogFiles(String indexName) throws IOException {
Set<Path> translogDirs = getDirs(indexName, ShardPath.TRANSLOG_FOLDER_NAME);
TestTranslog.corruptRandomTranslogFile(logger, random(), translogDirs);
}
/** Disables translog flushing for the specified index */
private static void disableTranslogFlush(String index) {
Settings settings = Settings.builder()
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB))
.build();
client().admin().indices().prepareUpdateSettings(index).setSettings(settings).get();
}
private SeqNoStats getSeqNoStats(String index, int shardId) {
final ShardStats[] shardStats = client().admin().indices()
.prepareStats(index).get()
.getIndices().get(index).getShards();
return shardStats[shardId].getSeqNoStats();
}
}

409
server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java Normal file
View File

@ -0,0 +1,409 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.shard;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.cli.MockTerminal;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.RecoverySource;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingHelper;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.TestShardRouting;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.MergePolicyConfig;
import org.elasticsearch.index.engine.EngineException;
import org.elasticsearch.index.engine.InternalEngineFactory;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.TestTranslog;
import org.elasticsearch.index.translog.TranslogCorruptedException;
import org.elasticsearch.test.CorruptionUtils;
import org.elasticsearch.test.DummyShardLock;
import org.junit.Before;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.startsWith;
public class RemoveCorruptedShardDataCommandTests extends IndexShardTestCase {
private ShardId shardId;
private ShardRouting routing;
private Path dataDir;
private Environment environment;
private Settings settings;
private ShardPath shardPath;
private IndexMetaData indexMetaData;
private IndexShard indexShard;
private Path translogPath;
private Path indexPath;
@Before
public void setup() throws IOException {
shardId = new ShardId("index0", "_na_", 0);
final String nodeId = randomAlphaOfLength(10);
routing = TestShardRouting.newShardRouting(shardId, nodeId, true, ShardRoutingState.INITIALIZING,
RecoverySource.EmptyStoreRecoverySource.INSTANCE);
dataDir = createTempDir();
environment =
TestEnvironment.newEnvironment(Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), dataDir)
.putList(Environment.PATH_DATA_SETTING.getKey(), dataDir.toAbsolutePath().toString()).build());
// create same directory structure as prod does
final Path path = NodeEnvironment.resolveNodePath(dataDir, 0);
Files.createDirectories(path);
settings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.build();
final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(path);
shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
final IndexMetaData.Builder metaData = IndexMetaData.builder(routing.getIndexName())
.settings(settings)
.primaryTerm(0, randomIntBetween(1, 100))
.putMapping("_doc", "{ \"properties\": {} }");
indexMetaData = metaData.build();
indexShard = newStartedShard(p ->
newShard(routing, shardPath, indexMetaData, null, null,
new InternalEngineFactory(), () -> {
}, EMPTY_EVENT_LISTENER),
true);
translogPath = shardPath.resolveTranslog();
indexPath = shardPath.resolveIndex();
}
public void testShardLock() throws Exception {
indexDocs(indexShard, true);
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
final MockTerminal t = new MockTerminal();
final OptionParser parser = command.getParser();
// Try running it before the shard is closed, it should flip out because it can't acquire the lock
try {
final OptionSet options = parser.parse("-d", indexPath.toString());
command.execute(t, options, environment);
fail("expected the command to fail not being able to acquire the lock");
} catch (Exception e) {
assertThat(e.getMessage(), containsString("Failed to lock shard's directory"));
}
// close shard
closeShards(indexShard);
// Try running it before the shard is corrupted
try {
final OptionSet options = parser.parse("-d", indexPath.toString());
command.execute(t, options, environment);
fail("expected the command to fail not being able to find a corrupt file marker");
} catch (ElasticsearchException e) {
assertThat(e.getMessage(), startsWith("Shard does not seem to be corrupted at"));
assertThat(t.getOutput(), containsString("Lucene index is clean at"));
}
}
public void testCorruptedIndex() throws Exception {
final int numDocs = indexDocs(indexShard, true);
// close shard
closeShards(indexShard);
final boolean corruptSegments = randomBoolean();
CorruptionUtils.corruptIndex(random(), indexPath, corruptSegments);
// test corrupted shard
final IndexShard corruptedShard = reopenIndexShard(true);
allowShardFailures();
expectThrows(IndexShardRecoveryException.class, () -> newStartedShard(p -> corruptedShard, true));
closeShards(corruptedShard);
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
final MockTerminal t = new MockTerminal();
final OptionParser parser = command.getParser();
// run command with dry-run
t.addTextInput("n"); // mean dry run
final OptionSet options = parser.parse("-d", indexPath.toString());
t.setVerbosity(Terminal.Verbosity.VERBOSE);
try {
command.execute(t, options, environment);
fail();
} catch (ElasticsearchException e) {
if (corruptSegments) {
assertThat(e.getMessage(), is("Index is unrecoverable"));
} else {
assertThat(e.getMessage(), containsString("aborted by user"));
}
}
logger.info("--> output:\n{}", t.getOutput());
if (corruptSegments == false) {
// run command without dry-run
t.addTextInput("y");
command.execute(t, options, environment);
final String output = t.getOutput();
logger.info("--> output:\n{}", output);
// reopen shard
failOnShardFailures();
final IndexShard newShard = newStartedShard(p -> reopenIndexShard(false), true);
final Set<String> shardDocUIDs = getShardDocUIDs(newShard);
final Pattern pattern = Pattern.compile("Corrupted Lucene index segments found -\\s+(?<docs>\\d+) documents will be lost.");
final Matcher matcher = pattern.matcher(output);
assertThat(matcher.find(), equalTo(true));
final int expectedNumDocs = numDocs - Integer.parseInt(matcher.group("docs"));
assertThat(shardDocUIDs.size(), equalTo(expectedNumDocs));
closeShards(newShard);
}
}
public void testCorruptedTranslog() throws Exception {
final int numDocsToKeep = indexDocs(indexShard, false);
// close shard
closeShards(indexShard);
TestTranslog.corruptRandomTranslogFile(logger, random(), Arrays.asList(translogPath));
// test corrupted shard
final IndexShard corruptedShard = reopenIndexShard(true);
allowShardFailures();
// it has to fail on start up due to index.shard.check_on_startup = checksum
final Exception exception = expectThrows(Exception.class, () -> newStartedShard(p -> corruptedShard, true));
final Throwable cause = exception.getCause() instanceof EngineException ? exception.getCause().getCause() : exception.getCause();
assertThat(cause, instanceOf(TranslogCorruptedException.class));
closeShards(corruptedShard);
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
final MockTerminal t = new MockTerminal();
final OptionParser parser = command.getParser();
final OptionSet options = parser.parse("-d", translogPath.toString());
// run command with dry-run
t.addTextInput("n"); // mean dry run
t.setVerbosity(Terminal.Verbosity.VERBOSE);
try {
command.execute(t, options, environment);
fail();
} catch (ElasticsearchException e) {
assertThat(e.getMessage(), containsString("aborted by user"));
assertThat(t.getOutput(), containsString("Continue and remove corrupted data from the shard ?"));
}
logger.info("--> output:\n{}", t.getOutput());
// run command without dry-run
t.reset();
t.addTextInput("y");
command.execute(t, options, environment);
final String output = t.getOutput();
logger.info("--> output:\n{}", output);
// reopen shard
failOnShardFailures();
final IndexShard newShard = newStartedShard(p -> reopenIndexShard(false), true);
final Set<String> shardDocUIDs = getShardDocUIDs(newShard);
assertThat(shardDocUIDs.size(), equalTo(numDocsToKeep));
closeShards(newShard);
}
public void testCorruptedBothIndexAndTranslog() throws Exception {
// index some docs in several segments
final int numDocsToKeep = indexDocs(indexShard, false);
// close shard
closeShards(indexShard);
CorruptionUtils.corruptIndex(random(), indexPath, false);
// test corrupted shard
final IndexShard corruptedShard = reopenIndexShard(true);
allowShardFailures();
expectThrows(IndexShardRecoveryException.class, () -> newStartedShard(p -> corruptedShard, true));
closeShards(corruptedShard);
TestTranslog.corruptRandomTranslogFile(logger, random(), Arrays.asList(translogPath));
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
final MockTerminal t = new MockTerminal();
final OptionParser parser = command.getParser();
final OptionSet options = parser.parse("-d", translogPath.toString());
// run command with dry-run
t.addTextInput("n"); // mean dry run
t.addTextInput("n"); // mean dry run
t.setVerbosity(Terminal.Verbosity.VERBOSE);
try {
command.execute(t, options, environment);
fail();
} catch (ElasticsearchException e) {
assertThat(e.getMessage(), containsString("aborted by user"));
assertThat(t.getOutput(), containsString("Continue and remove corrupted data from the shard ?"));
}
logger.info("--> output:\n{}", t.getOutput());
// run command without dry-run
t.reset();
t.addTextInput("y");
command.execute(t, options, environment);
final String output = t.getOutput();
logger.info("--> output:\n{}", output);
// reopen shard
failOnShardFailures();
final IndexShard newShard = newStartedShard(p -> reopenIndexShard(false), true);
final Set<String> shardDocUIDs = getShardDocUIDs(newShard);
final Pattern pattern = Pattern.compile("Corrupted Lucene index segments found -\\s+(?<docs>\\d+) documents will be lost.");
final Matcher matcher = pattern.matcher(output);
assertThat(matcher.find(), equalTo(true));
final int expectedNumDocs = numDocsToKeep - Integer.parseInt(matcher.group("docs"));
assertThat(shardDocUIDs.size(), equalTo(expectedNumDocs));
closeShards(newShard);
}
public void testResolveIndexDirectory() throws Exception {
// index a single doc to have files on a disk
indexDoc(indexShard, "_doc", "0", "{}");
flushShard(indexShard, true);
writeIndexState();
// close shard
closeShards(indexShard);
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
final OptionParser parser = command.getParser();
// `--index index_name --shard-id 0` has to be resolved to indexPath
final OptionSet options = parser.parse("--index", shardId.getIndex().getName(),
"--shard-id", Integer.toString(shardId.id()));
command.findAndProcessShardPath(options, environment,
shardPath -> assertThat(shardPath.resolveIndex(), equalTo(indexPath)));
final OptionSet options2 = parser.parse("--dir", indexPath.toAbsolutePath().toString());
command.findAndProcessShardPath(options2, environment,
shardPath -> assertThat(shardPath.resolveIndex(), equalTo(indexPath)));
}
private IndexShard reopenIndexShard(boolean corrupted) throws IOException {
// open shard with the same location
final ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(indexShard.routingEntry(),
RecoverySource.ExistingStoreRecoverySource.INSTANCE
);
final IndexMetaData metaData = IndexMetaData.builder(indexMetaData)
.settings(Settings.builder()
.put(indexShard.indexSettings().getSettings())
.put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), "checksum"))
.build();
CheckedFunction<IndexSettings, Store, IOException> storeProvider =
corrupted == false ? null :
indexSettings -> {
final ShardId shardId = shardPath.getShardId();
final BaseDirectoryWrapper baseDirectoryWrapper = newFSDirectory(shardPath.resolveIndex());
// index is corrupted - don't even try to check index on close - it fails
baseDirectoryWrapper.setCheckIndexOnClose(false);
return new Store(shardId, indexSettings, baseDirectoryWrapper, new DummyShardLock(shardId));
};
return newShard(shardRouting, shardPath, metaData, storeProvider, null,
indexShard.engineFactory, indexShard.getGlobalCheckpointSyncer(), EMPTY_EVENT_LISTENER);
}
private int indexDocs(IndexShard indexShard, boolean flushLast) throws IOException {
// index some docs in several segments
int numDocs = 0;
int numDocsToKeep = 0;
for (int i = 0, attempts = randomIntBetween(5, 10); i < attempts; i++) {
final int numExtraDocs = between(10, 100);
for (long j = 0; j < numExtraDocs; j++) {
indexDoc(indexShard, "_doc", Long.toString(numDocs + j), "{}");
}
numDocs += numExtraDocs;
if (flushLast || i < attempts - 1) {
numDocsToKeep += numExtraDocs;
flushShard(indexShard, true);
}
}
logger.info("--> indexed {} docs, {} to keep", numDocs, numDocsToKeep);
writeIndexState();
return numDocsToKeep;
}
private void writeIndexState() throws IOException {
// create _state of IndexMetaData
try(NodeEnvironment nodeEnvironment = new NodeEnvironment(environment.settings(), environment, nId -> {})) {
final Path[] paths = nodeEnvironment.indexPaths(indexMetaData.getIndex());
IndexMetaData.FORMAT.write(indexMetaData, paths);
logger.info("--> index metadata persisted to {} ", Arrays.toString(paths));
}
}
}

57
server/src/test/java/org/elasticsearch/index/similarity/NonNegativeScoresSimilarityTests.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.similarity;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;
public class NonNegativeScoresSimilarityTests extends ESTestCase {
public void testBasics() {
Similarity negativeScoresSim = new Similarity() {
@Override
public long computeNorm(FieldInvertState state) {
return state.getLength();
}
@Override
public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
return new SimScorer() {
@Override
public float score(float freq, long norm) {
return freq - 5;
}
};
}
};
Similarity assertingSimilarity = new NonNegativeScoresSimilarity(negativeScoresSim);
SimScorer scorer = assertingSimilarity.scorer(1f, null);
assertEquals(2f, scorer.score(7f, 1L), 0f);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> scorer.score(2f, 1L));
assertThat(e.getMessage(), Matchers.containsString("Similarities must not produce negative scores"));
}
}

78
server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java
View File

@ -18,12 +18,18 @@
*/
package org.elasticsearch.index.similarity;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.BooleanSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.elasticsearch.Version;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import org.hamcrest.Matchers;
import java.util.Collections;
@ -56,4 +62,76 @@ public class SimilarityServiceTests extends ESTestCase {
SimilarityService service = new SimilarityService(indexSettings, null, Collections.emptyMap());
assertTrue(service.getDefaultSimilarity() instanceof BooleanSimilarity);
}
public void testSimilarityValidation() {
Similarity negativeScoresSim = new Similarity() {
@Override
public long computeNorm(FieldInvertState state) {
return state.getLength();
}
@Override
public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
return new SimScorer() {
@Override
public float score(float freq, long norm) {
return -1;
}
};
}
};
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> SimilarityService.validateSimilarity(Version.V_7_0_0_alpha1, negativeScoresSim));
assertThat(e.getMessage(), Matchers.containsString("Similarities should not return negative scores"));
Similarity decreasingScoresWithFreqSim = new Similarity() {
@Override
public long computeNorm(FieldInvertState state) {
return state.getLength();
}
@Override
public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
return new SimScorer() {
@Override
public float score(float freq, long norm) {
return 1 / (freq + norm);
}
};
}
};
e = expectThrows(IllegalArgumentException.class,
() -> SimilarityService.validateSimilarity(Version.V_7_0_0_alpha1, decreasingScoresWithFreqSim));
assertThat(e.getMessage(), Matchers.containsString("Similarity scores should not decrease when term frequency increases"));
Similarity increasingScoresWithNormSim = new Similarity() {
@Override
public long computeNorm(FieldInvertState state) {
return state.getLength();
}
@Override
public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
return new SimScorer() {
@Override
public float score(float freq, long norm) {
return freq + norm;
}
};
}
};
e = expectThrows(IllegalArgumentException.class,
() -> SimilarityService.validateSimilarity(Version.V_7_0_0_alpha1, increasingScoresWithNormSim));
assertThat(e.getMessage(), Matchers.containsString("Similarity scores should not increase when norm increases"));
}
}

2
server/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
View File

@ -125,7 +125,7 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
}
}
Path translogDir = RandomPicks.randomFrom(random(), translogDirs);
TestTranslog.corruptRandomTranslogFile(logger, random(), translogDir, TestTranslog.minTranslogGenUsedInRecovery(translogDir));
TestTranslog.corruptRandomTranslogFile(logger, random(), Arrays.asList(translogDir));
}
/** Disables translog flushing for the specified index */

11
server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java
View File

@ -34,6 +34,7 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Collection;
import java.util.List;
import java.util.Random;
import java.util.Set;
@ -52,13 +53,19 @@ import static org.hamcrest.core.IsNot.not;
public class TestTranslog {
static final Pattern TRANSLOG_FILE_PATTERN = Pattern.compile("translog-(\\d+)\\.tlog");
public static void corruptRandomTranslogFile(Logger logger, Random random, Collection<Path> translogDirs) throws IOException {
for (Path translogDir : translogDirs) {
final long minTranslogGen = minTranslogGenUsedInRecovery(translogDir);
corruptRandomTranslogFile(logger, random, translogDir, minTranslogGen);
}
}
/**
* Corrupts random translog file (translog-N.tlog) from the given translog directory.
*
* @return a translog file which has been corrupted.
*/
public static Path corruptRandomTranslogFile(Logger logger, Random random, Path translogDir, long minGeneration) throws
IOException {
public static Path corruptRandomTranslogFile(Logger logger, Random random, Path translogDir, long minGeneration) throws IOException {
Set<Path> candidates = new TreeSet<>(); // TreeSet makes sure iteration order is deterministic
logger.info("--> Translog dir [{}], minUsedTranslogGen [{}]", translogDir, minGeneration);
try (DirectoryStream<Path> stream = Files.newDirectoryStream(translogDir)) {

382
server/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
View File

@ -1,382 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.translog;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.NativeFSLockFactory;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.cli.MockTerminal;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.ShardIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.MockEngineFactoryPlugin;
import org.elasticsearch.index.seqno.SeqNoStats;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalTestCluster;
import org.elasticsearch.test.engine.MockEngineSupport;
import org.elasticsearch.test.transport.MockTransportService;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0)
public class TruncateTranslogIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(MockTransportService.TestPlugin.class, MockEngineFactoryPlugin.class);
}
public void testCorruptTranslogTruncation() throws Exception {
internalCluster().startNodes(2, Settings.EMPTY);
final String replicaNode = internalCluster().getNodeNames()[1];
assertAcked(prepareCreate("test").setSettings(Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 1)
.put("index.refresh_interval", "-1")
.put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true) // never flush - always recover from translog
.put("index.routing.allocation.exclude._name", replicaNode)
));
ensureYellow();
assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
.put("index.routing.allocation.exclude._name", (String)null)
));
ensureGreen();
// Index some documents
int numDocsToKeep = randomIntBetween(0, 100);
logger.info("--> indexing [{}] docs to be kept", numDocsToKeep);
IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocsToKeep];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex("test", "type").setSource("foo", "bar");
}
indexRandom(false, false, false, Arrays.asList(builders));
flush("test");
disableTranslogFlush("test");
// having no extra docs is an interesting case for seq no based recoveries - test it more often
int numDocsToTruncate = randomBoolean() ? 0 : randomIntBetween(0, 100);
logger.info("--> indexing [{}] more doc to be truncated", numDocsToTruncate);
builders = new IndexRequestBuilder[numDocsToTruncate];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex("test", "type").setSource("foo", "bar");
}
indexRandom(false, false, false, Arrays.asList(builders));
Set<Path> translogDirs = getTranslogDirs("test");
TruncateTranslogCommand ttc = new TruncateTranslogCommand();
MockTerminal t = new MockTerminal();
OptionParser parser = ttc.getParser();
for (Path translogDir : translogDirs) {
OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString(), "-b");
// Try running it before the shard is closed, it should flip out because it can't acquire the lock
try {
logger.info("--> running truncate while index is open on [{}]", translogDir.toAbsolutePath());
ttc.execute(t, options, null /* TODO: env should be real here, and ttc should actually use it... */);
fail("expected the truncate command to fail not being able to acquire the lock");
} catch (Exception e) {
assertThat(e.getMessage(), containsString("Failed to lock shard's directory"));
}
}
if (randomBoolean() && numDocsToTruncate > 0) {
// flush the replica, so it will have more docs than what the primary will have
Index index = resolveIndex("test");
IndexShard replica = internalCluster().getInstance(IndicesService.class, replicaNode).getShardOrNull(new ShardId(index, 0));
replica.flush(new FlushRequest());
logger.info("--> performed extra flushing on replica");
}
// shut down the replica node to be tested later
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode));
// Corrupt the translog file
logger.info("--> corrupting translog");
corruptRandomTranslogFile("test");
// Restart the single node
logger.info("--> restarting node");
internalCluster().restartRandomDataNode();
client().admin().cluster().prepareHealth().setWaitForYellowStatus()
.setTimeout(new TimeValue(1000, TimeUnit.MILLISECONDS))
.setWaitForEvents(Priority.LANGUID)
.get();
try {
client().prepareSearch("test").setQuery(matchAllQuery()).get();
fail("all shards should be failed due to a corrupted translog");
} catch (SearchPhaseExecutionException e) {
// Good, all shards should be failed because there is only a
// single shard and its translog is corrupt
}
// Close the index so we can actually truncate the translog
logger.info("--> closing 'test' index");
client().admin().indices().prepareClose("test").get();
for (Path translogDir : translogDirs) {
final Path idxLocation = translogDir.getParent().resolve("index");
assertBusy(() -> {
logger.info("--> checking that lock has been released for {}", idxLocation);
try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE);
Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
// Great, do nothing, we just wanted to obtain the lock
} catch (LockObtainFailedException lofe) {
logger.info("--> failed acquiring lock for {}", idxLocation);
fail("still waiting for lock release at [" + idxLocation + "]");
} catch (IOException ioe) {
fail("Got an IOException: " + ioe);
}
});
OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString(), "-b");
logger.info("--> running truncate translog command for [{}]", translogDir.toAbsolutePath());
ttc.execute(t, options, null /* TODO: env should be real here, and ttc should actually use it... */);
logger.info("--> output:\n{}", t.getOutput());
}
// Re-open index
logger.info("--> opening 'test' index");
client().admin().indices().prepareOpen("test").get();
ensureYellow("test");
// Run a search and make sure it succeeds
assertHitCount(client().prepareSearch("test").setQuery(matchAllQuery()).get(), numDocsToKeep);
logger.info("--> starting the replica node to test recovery");
internalCluster().startNode();
ensureGreen("test");
for (String node : internalCluster().nodesInclude("test")) {
SearchRequestBuilder q = client().prepareSearch("test").setPreference("_only_nodes:" + node).setQuery(matchAllQuery());
assertHitCount(q.get(), numDocsToKeep);
}
final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").setActiveOnly(false).get();
final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get("test").stream()
.filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get();
assertThat(replicaRecoveryState.getIndex().toString(), replicaRecoveryState.getIndex().recoveredFileCount(), greaterThan(0));
// Ensure that the global checkpoint and local checkpoint are restored from the max seqno of the last commit.
final SeqNoStats seqNoStats = getSeqNoStats("test", 0);
assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
assertThat(seqNoStats.getLocalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
}
public void testCorruptTranslogTruncationOfReplica() throws Exception {
internalCluster().startNodes(2, Settings.EMPTY);
final String primaryNode = internalCluster().getNodeNames()[0];
final String replicaNode = internalCluster().getNodeNames()[1];
assertAcked(prepareCreate("test").setSettings(Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 1)
.put("index.refresh_interval", "-1")
.put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true) // never flush - always recover from translog
.put("index.routing.allocation.exclude._name", replicaNode)
));
ensureYellow();
assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
.put("index.routing.allocation.exclude._name", (String)null)
));
ensureGreen();
// Index some documents
int numDocsToKeep = randomIntBetween(0, 100);
logger.info("--> indexing [{}] docs to be kept", numDocsToKeep);
IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocsToKeep];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex("test", "type").setSource("foo", "bar");
}
indexRandom(false, false, false, Arrays.asList(builders));
flush("test");
disableTranslogFlush("test");
// having no extra docs is an interesting case for seq no based recoveries - test it more often
int numDocsToTruncate = randomBoolean() ? 0 : randomIntBetween(0, 100);
logger.info("--> indexing [{}] more docs to be truncated", numDocsToTruncate);
builders = new IndexRequestBuilder[numDocsToTruncate];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex("test", "type").setSource("foo", "bar");
}
indexRandom(false, false, false, Arrays.asList(builders));
final int totalDocs = numDocsToKeep + numDocsToTruncate;
// sample the replica node translog dirs
final ShardId shardId = new ShardId(resolveIndex("test"), 0);
Set<Path> translogDirs = getTranslogDirs(replicaNode, shardId);
Path tdir = randomFrom(translogDirs);
// stop the cluster nodes. we don't use full restart so the node start up order will be the same
// and shard roles will be maintained
internalCluster().stopRandomDataNode();
internalCluster().stopRandomDataNode();
// Corrupt the translog file
logger.info("--> corrupting translog");
TestTranslog.corruptRandomTranslogFile(logger, random(), tdir, TestTranslog.minTranslogGenUsedInRecovery(tdir));
// Restart the single node
logger.info("--> starting node");
internalCluster().startNode();
ensureYellow();
// Run a search and make sure it succeeds
assertHitCount(client().prepareSearch("test").setQuery(matchAllQuery()).get(), totalDocs);
TruncateTranslogCommand ttc = new TruncateTranslogCommand();
MockTerminal t = new MockTerminal();
OptionParser parser = ttc.getParser();
for (Path translogDir : translogDirs) {
final Path idxLocation = translogDir.getParent().resolve("index");
assertBusy(() -> {
logger.info("--> checking that lock has been released for {}", idxLocation);
try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE);
Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
// Great, do nothing, we just wanted to obtain the lock
} catch (LockObtainFailedException lofe) {
logger.info("--> failed acquiring lock for {}", idxLocation);
fail("still waiting for lock release at [" + idxLocation + "]");
} catch (IOException ioe) {
fail("Got an IOException: " + ioe);
}
});
OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString(), "-b");
logger.info("--> running truncate translog command for [{}]", translogDir.toAbsolutePath());
ttc.execute(t, options, null /* TODO: env should be real here, and ttc should actually use it... */);
logger.info("--> output:\n{}", t.getOutput());
}
logger.info("--> starting the replica node to test recovery");
internalCluster().startNode();
ensureGreen("test");
for (String node : internalCluster().nodesInclude("test")) {
assertHitCount(client().prepareSearch("test").setPreference("_only_nodes:" + node).setQuery(matchAllQuery()).get(), totalDocs);
}
final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").setActiveOnly(false).get();
final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get("test").stream()
.filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get();
// the replica translog was disabled so it doesn't know what hte global checkpoint is and thus can't do ops based recovery
assertThat(replicaRecoveryState.getIndex().toString(), replicaRecoveryState.getIndex().recoveredFileCount(), greaterThan(0));
// Ensure that the global checkpoint and local checkpoint are restored from the max seqno of the last commit.
final SeqNoStats seqNoStats = getSeqNoStats("test", 0);
assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
assertThat(seqNoStats.getLocalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
}
private Set<Path> getTranslogDirs(String indexName) throws IOException {
ClusterState state = client().admin().cluster().prepareState().get().getState();
GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{indexName}, false);
List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
ShardRouting shardRouting = shardIterator.nextOrNull();
assertNotNull(shardRouting);
assertTrue(shardRouting.primary());
assertTrue(shardRouting.assignedToNode());
String nodeId = shardRouting.currentNodeId();
ShardId shardId = shardRouting.shardId();
return getTranslogDirs(nodeId, shardId);
}
private Set<Path> getTranslogDirs(String nodeId, ShardId shardId) {
NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
Set<Path> translogDirs = new TreeSet<>(); // treeset makes sure iteration order is deterministic
for (FsInfo.Path fsPath : nodeStatses.getNodes().get(0).getFs()) {
String path = fsPath.getPath();
final String relativeDataLocationPath = "indices/"+ shardId.getIndex().getUUID() +"/" + Integer.toString(shardId.getId())
+ "/translog";
Path translogPath = PathUtils.get(path).resolve(relativeDataLocationPath);
if (Files.isDirectory(translogPath)) {
translogDirs.add(translogPath);
}
}
return translogDirs;
}
private void corruptRandomTranslogFile(String indexName) throws IOException {
Set<Path> translogDirs = getTranslogDirs(indexName);
Path translogDir = randomFrom(translogDirs);
TestTranslog.corruptRandomTranslogFile(logger, random(), translogDir, TestTranslog.minTranslogGenUsedInRecovery(translogDir));
}
/** Disables translog flushing for the specified index */
private static void disableTranslogFlush(String index) {
Settings settings = Settings.builder()
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB))
.build();
client().admin().indices().prepareUpdateSettings(index).setSettings(settings).get();
}
private SeqNoStats getSeqNoStats(String index, int shardId) {
final ShardStats[] shardStats = client().admin().indices()
.prepareStats(index).get()
.getIndices().get(index).getShards();
return shardStats[shardId].getSeqNoStats();
}
}

8
server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.indices;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.AlreadyClosedException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
@ -56,6 +57,7 @@ import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.IndexShardState;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.index.similarity.NonNegativeScoresSimilarity;
import org.elasticsearch.indices.IndicesService.ShardDeletionCheckResult;
import org.elasticsearch.plugins.EnginePlugin;
import org.elasticsearch.plugins.MapperPlugin;
@ -448,8 +450,10 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
.build();
MapperService mapperService = indicesService.createIndexMapperService(indexMetaData);
assertNotNull(mapperService.documentMapperParser().parserContext("type").typeParser("fake-mapper"));
assertThat(mapperService.documentMapperParser().parserContext("type").getSimilarity("test").get(),
instanceOf(BM25Similarity.class));
Similarity sim = mapperService.documentMapperParser().parserContext("type").getSimilarity("test").get();
assertThat(sim, instanceOf(NonNegativeScoresSimilarity.class));
sim = ((NonNegativeScoresSimilarity) sim).getDelegate();
assertThat(sim, instanceOf(BM25Similarity.class));
}
public void testStatsByShardDoesNotDieFromExpectedExceptions() {

230
server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
View File

@ -28,10 +28,12 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryCachingPolicy;
import org.apache.lucene.search.RandomApproximationQuery;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
@ -118,6 +120,209 @@ public class QueryProfilerTests extends ESTestCase {
assertThat(rewriteTime, greaterThan(0L));
}
public void testConstantScoreQuery() throws IOException {
QueryProfiler profiler = new QueryProfiler();
searcher.setProfiler(profiler);
Query query = new ConstantScoreQuery(new TermQuery(new Term("foo", "bar")));
searcher.search(query, 1);
List<ProfileResult> results = profiler.getTree();
assertEquals(1, results.size());
Map<String, Long> breakdownConstantScoreQuery = results.get(0).getTimeBreakdown();
assertEquals(1, results.get(0).getProfiledChildren().size());
Map<String, Long> breakdownTermQuery = results.get(0).getProfiledChildren().get(0).getTimeBreakdown();
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
assertEquals(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(),
breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue());
long rewriteTime = profiler.getRewriteTime();
assertThat(rewriteTime, greaterThan(0L));
}
public void testConstantScoreTotalHitsBeingCachedQuery() throws IOException {
Query query = new ConstantScoreQuery(new TermQuery(new Term("foo", "bar")));
//clean cache and make sure queries will be cached
searcher.setQueryCache(IndexSearcher.getDefaultQueryCache());
searcher.setQueryCachingPolicy(ALWAYS_CACHE_POLICY);
QueryProfiler profiler = new QueryProfiler();
searcher.setProfiler(profiler);
TotalHitCountCollector collector = new TotalHitCountCollector();
searcher.search(query, collector);
List<ProfileResult> results = profiler.getTree();
assertEquals(1, results.size());
Map<String, Long> breakdownConstantScoreQuery = results.get(0).getTimeBreakdown();
assertEquals(1, results.get(0).getProfiledChildren().size());
Map<String, Long> breakdownTermQuery = results.get(0).getProfiledChildren().get(0).getTimeBreakdown();
//In this case scorers for constant score query and term query are disconnected.
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
long rewriteTime = profiler.getRewriteTime();
assertThat(rewriteTime, greaterThan(0L));
}
public void testConstantScoreTotalHitsNotCachedQuery() throws IOException {
Query query = new ConstantScoreQuery(new TermQuery(new Term("foo", "bar")));
//clean cache and make sure queries will not be cached
searcher.setQueryCache(IndexSearcher.getDefaultQueryCache());
searcher.setQueryCachingPolicy(NEVER_CACHE_POLICY);
QueryProfiler profiler = new QueryProfiler();
searcher.setProfiler(profiler);
TotalHitCountCollector collector = new TotalHitCountCollector();
searcher.search(query, collector);
List<ProfileResult> results = profiler.getTree();
assertEquals(1, results.size());
Map<String, Long> breakdownConstantScoreQuery = results.get(0).getTimeBreakdown();
assertEquals(1, results.get(0).getProfiledChildren().size());
Map<String, Long> breakdownTermQuery = results.get(0).getProfiledChildren().get(0).getTimeBreakdown();
//Timing from the scorer of term query are inherited by constant score query scorer.
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
assertEquals(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(),
breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue());
long rewriteTime = profiler.getRewriteTime();
assertThat(rewriteTime, greaterThan(0L));
}
public void testConstantScoreTotalHitsCachedQuery() throws IOException {
Query query = new ConstantScoreQuery(new TermQuery(new Term("foo", "bar")));
//clean cache and make sure queries will be cached
searcher.setQueryCache(IndexSearcher.getDefaultQueryCache());
searcher.setQueryCachingPolicy(ALWAYS_CACHE_POLICY);
//Put query on cache
TotalHitCountCollector collector = new TotalHitCountCollector();
searcher.search(query, collector);
QueryProfiler profiler = new QueryProfiler();
searcher.setProfiler(profiler);
collector = new TotalHitCountCollector();
searcher.search(query, collector);
List<ProfileResult> results = profiler.getTree();
assertEquals(1, results.size());
Map<String, Long> breakdownConstantScoreQuery = results.get(0).getTimeBreakdown();
assertEquals(1, results.get(0).getProfiledChildren().size());
Map<String, Long> breakdownTermQuery = results.get(0).getProfiledChildren().get(0).getTimeBreakdown();
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
long rewriteTime = profiler.getRewriteTime();
assertThat(rewriteTime, greaterThan(0L));
}
public void testNoScoring() throws IOException {
QueryProfiler profiler = new QueryProfiler();
searcher.setProfiler(profiler);
@ -276,4 +481,29 @@ public class QueryProfilerTests extends ESTestCase {
reader.close();
dir.close();
}
private static final QueryCachingPolicy ALWAYS_CACHE_POLICY = new QueryCachingPolicy() {
@Override
public void onUse(Query query) {}
@Override
public boolean shouldCache(Query query) throws IOException {
return true;
}
};
private static final QueryCachingPolicy NEVER_CACHE_POLICY = new QueryCachingPolicy() {
@Override
public void onUse(Query query) {}
@Override
public boolean shouldCache(Query query) throws IOException {
return false;
}
};
}

4
server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.transport;
import java.util.function.Supplier;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.LatchedActionListener;
@ -56,6 +55,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import static org.hamcrest.CoreMatchers.containsString;
@ -578,7 +578,7 @@ public class RemoteClusterServiceTests extends ESTestCase {
remoteClusterService.collectSearchShards(IndicesOptions.lenientExpandOpen(), "index_not_found",
null, remoteIndicesByCluster,
new LatchedActionListener<>(ActionListener.wrap(response::set, failure::set), latch));
assertTrue(latch.await(1, TimeUnit.SECONDS));
assertTrue(latch.await(2, TimeUnit.SECONDS));
assertNull(response.get());
assertNotNull(failure.get());
assertThat(failure.get(), instanceOf(RemoteTransportException.class));

9
test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
View File

@ -133,7 +133,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
super.setUp();
threadPool = new TestThreadPool(getClass().getName(), threadPoolSettings());
primaryTerm = randomIntBetween(1, 100); // use random but fixed term for creating shards
failOnShardFailures.set(true);
failOnShardFailures();
}
@Override
@ -154,6 +154,10 @@ public abstract class IndexShardTestCase extends ESTestCase {
failOnShardFailures.set(false);
}
protected void failOnShardFailures() {
failOnShardFailures.set(true);
}
public Settings threadPoolSettings() {
return Settings.EMPTY;
}
@ -233,7 +237,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
.settings(indexSettings)
.primaryTerm(0, primaryTerm)
.putMapping("_doc", "{ \"properties\": {} }");
return newShard(shardRouting, metaData.build(), engineFactory, listeners);
return newShard(shardRouting, metaData.build(), null, engineFactory, () -> {}, listeners);
}
/**
@ -279,7 +283,6 @@ public abstract class IndexShardTestCase extends ESTestCase {
return newShard(shardRouting, indexMetaData, searcherWrapper, new InternalEngineFactory(), globalCheckpointSyncer);
}
/**
* creates a new initializing shard. The shard will will be put in its proper path under the
* current node id the shard is assigned to.

18
test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.test;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@ -47,6 +48,23 @@ public final class CorruptionUtils {
private static Logger logger = ESLoggerFactory.getLogger("test");
private CorruptionUtils() {}
public static void corruptIndex(Random random, Path indexPath, boolean corruptSegments) throws IOException {
// corrupt files
final Path[] filesToCorrupt =
Files.walk(indexPath)
.filter(p -> {
final String name = p.getFileName().toString();
boolean segmentFile = name.startsWith("segments_") || name.endsWith(".si");
return Files.isRegularFile(p)
&& name.startsWith("extra") == false // Skip files added by Lucene's ExtrasFS
&& IndexWriter.WRITE_LOCK_NAME.equals(name) == false
&& (corruptSegments ? segmentFile : segmentFile == false);
}
)
.toArray(Path[]::new);
corruptFile(random, filesToCorrupt);
}
/**
* Corrupts a random file at a random position
*/

1
test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java
View File

@ -182,6 +182,7 @@ public class MockTcpTransport extends TcpTransport {
executor.submit(() -> {
try {
socket.connect(address);
socket.setSoLinger(false, 0);
channel.loopRead(executor);
connectListener.onResponse(null);
} catch (Exception ex) {

1
test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java
View File

@ -193,6 +193,7 @@ public class MockNioTransport extends TcpTransport {
BytesChannelContext context = new BytesChannelContext(nioChannel, selector, (e) -> exceptionCaught(nioChannel, e),
readWriteHandler, new InboundChannelBuffer(pageSupplier));
nioChannel.setContext(context);
nioChannel.setSoLinger(0);
return nioChannel;
}

4
x-pack/plugin/ccr/qa/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java
View File

@ -199,13 +199,13 @@ public class FollowIndexSecurityIT extends ESRestTestCase {
private static void followIndex(String leaderIndex, String followIndex) throws IOException {
final Request request = new Request("POST", "/" + followIndex + "/_ccr/follow");
request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"idle_shard_retry_delay\": \"10ms\"}");
request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"poll_timeout\": \"10ms\"}");
assertOK(client().performRequest(request));
}
private static void createAndFollowIndex(String leaderIndex, String followIndex) throws IOException {
final Request request = new Request("POST", "/" + followIndex + "/_ccr/create_and_follow");
request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"idle_shard_retry_delay\": \"10ms\"}");
request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"poll_timeout\": \"10ms\"}");
assertOK(client().performRequest(request));
}

4
x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
View File

@ -141,13 +141,13 @@ public class FollowIndexIT extends ESRestTestCase {
private static void followIndex(String leaderIndex, String followIndex) throws IOException {
final Request request = new Request("POST", "/" + followIndex + "/_ccr/follow");
request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"idle_shard_retry_delay\": \"10ms\"}");
request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"poll_timeout\": \"10ms\"}");
assertOK(client().performRequest(request));
}
private static void createAndFollowIndex(String leaderIndex, String followIndex) throws IOException {
final Request request = new Request("POST", "/" + followIndex + "/_ccr/create_and_follow");
request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"idle_shard_retry_delay\": \"10ms\"}");
request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"poll_timeout\": \"10ms\"}");
assertOK(client().performRequest(request));
}

16
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
View File

@ -297,12 +297,16 @@ public class AutoFollowCoordinator implements ClusterStateApplier {
String leaderIndexNameWithClusterAliasPrefix = clusterAlias.equals("_local_") ? leaderIndexName :
clusterAlias + ":" + leaderIndexName;
FollowIndexAction.Request request =
new FollowIndexAction.Request(leaderIndexNameWithClusterAliasPrefix, followIndexName,
pattern.getMaxBatchOperationCount(), pattern.getMaxConcurrentReadBatches(),
pattern.getMaxOperationSizeInBytes(), pattern.getMaxConcurrentWriteBatches(),
pattern.getMaxWriteBufferSize(), pattern.getMaxRetryDelay(),
pattern.getIdleShardRetryDelay());
FollowIndexAction.Request request = new FollowIndexAction.Request();
request.setLeaderIndex(leaderIndexNameWithClusterAliasPrefix);
request.setFollowerIndex(followIndexName);
request.setMaxBatchOperationCount(pattern.getMaxBatchOperationCount());
request.setMaxConcurrentReadBatches(pattern.getMaxConcurrentReadBatches());
request.setMaxOperationSizeInBytes(pattern.getMaxOperationSizeInBytes());
request.setMaxConcurrentWriteBatches(pattern.getMaxConcurrentWriteBatches());
request.setMaxWriteBufferSize(pattern.getMaxWriteBufferSize());
request.setMaxRetryDelay(pattern.getMaxRetryDelay());
request.setPollTimeout(pattern.getPollTimeout());
// Execute if the create and follow api call succeeds:
Runnable successHandler = () -> {

5
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
View File

@ -32,7 +32,6 @@ import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.ccr.action.FollowIndexAction;
import java.io.IOException;
import java.util.ArrayList;
@ -64,8 +63,8 @@ public class ShardChangesAction extends Action<ShardChangesAction.Response> {
private int maxOperationCount;
private ShardId shardId;
private String expectedHistoryUUID;
private TimeValue pollTimeout = FollowIndexAction.DEFAULT_POLL_TIMEOUT;
private long maxOperationSizeInBytes = FollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES;
private TimeValue pollTimeout = TransportFollowIndexAction.DEFAULT_POLL_TIMEOUT;
private long maxOperationSizeInBytes = TransportFollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES;
public Request(ShardId shardId, String expectedHistoryUUID) {
super(shardId.getIndexName());

87
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowIndexAction.java
View File

@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexingSlowLog;
@ -55,6 +56,14 @@ import java.util.stream.Collectors;
public class TransportFollowIndexAction extends HandledTransportAction<FollowIndexAction.Request, AcknowledgedResponse> {
static final long DEFAULT_MAX_BATCH_SIZE_IN_BYTES = Long.MAX_VALUE;
private static final TimeValue DEFAULT_MAX_RETRY_DELAY = new TimeValue(500);
private static final int DEFAULT_MAX_CONCURRENT_WRITE_BATCHES = 1;
private static final int DEFAULT_MAX_WRITE_BUFFER_SIZE = 10240;
private static final int DEFAULT_MAX_BATCH_OPERATION_COUNT = 1024;
private static final int DEFAULT_MAX_CONCURRENT_READ_BATCHES = 1;
static final TimeValue DEFAULT_POLL_TIMEOUT = TimeValue.timeValueMinutes(1);
private final Client client;
private final ThreadPool threadPool;
private final ClusterService clusterService;
@ -179,19 +188,8 @@ public class TransportFollowIndexAction extends HandledTransportAction<FollowInd
String[] recordedLeaderShardHistoryUUIDs = extractIndexShardHistoryUUIDs(ccrIndexMetadata);
String recordedLeaderShardHistoryUUID = recordedLeaderShardHistoryUUIDs[shardId];
ShardFollowTask shardFollowTask = new ShardFollowTask(
clusterNameAlias,
new ShardId(followIndexMetadata.getIndex(), shardId),
new ShardId(leaderIndexMetadata.getIndex(), shardId),
request.getMaxBatchOperationCount(),
request.getMaxConcurrentReadBatches(),
request.getMaxOperationSizeInBytes(),
request.getMaxConcurrentWriteBatches(),
request.getMaxWriteBufferSize(),
request.getMaxRetryDelay(),
request.getPollTimeout(),
recordedLeaderShardHistoryUUID,
filteredHeaders);
final ShardFollowTask shardFollowTask = createShardFollowTask(shardId, clusterNameAlias, request,
leaderIndexMetadata, followIndexMetadata, recordedLeaderShardHistoryUUID, filteredHeaders);
persistentTasksService.sendStartRequest(taskId, ShardFollowTask.NAME, shardFollowTask,
new ActionListener<PersistentTasksCustomMetaData.PersistentTask<ShardFollowTask>>() {
@Override
@ -299,6 +297,69 @@ public class TransportFollowIndexAction extends HandledTransportAction<FollowInd
followerMapperService.merge(leaderIndex, MapperService.MergeReason.MAPPING_RECOVERY);
}
private static ShardFollowTask createShardFollowTask(
int shardId,
String clusterAliasName,
FollowIndexAction.Request request,
IndexMetaData leaderIndexMetadata,
IndexMetaData followIndexMetadata,
String recordedLeaderShardHistoryUUID,
Map<String, String> filteredHeaders
) {
int maxBatchOperationCount;
if (request.getMaxBatchOperationCount() != null) {
maxBatchOperationCount = request.getMaxBatchOperationCount();
} else {
maxBatchOperationCount = DEFAULT_MAX_BATCH_OPERATION_COUNT;
}
int maxConcurrentReadBatches;
if (request.getMaxConcurrentReadBatches() != null){
maxConcurrentReadBatches = request.getMaxConcurrentReadBatches();
} else {
maxConcurrentReadBatches = DEFAULT_MAX_CONCURRENT_READ_BATCHES;
}
long maxOperationSizeInBytes;
if (request.getMaxOperationSizeInBytes() != null) {
maxOperationSizeInBytes = request.getMaxOperationSizeInBytes();
} else {
maxOperationSizeInBytes = DEFAULT_MAX_BATCH_SIZE_IN_BYTES;
}
int maxConcurrentWriteBatches;
if (request.getMaxConcurrentWriteBatches() != null) {
maxConcurrentWriteBatches = request.getMaxConcurrentWriteBatches();
} else {
maxConcurrentWriteBatches = DEFAULT_MAX_CONCURRENT_WRITE_BATCHES;
}
int maxWriteBufferSize;
if (request.getMaxWriteBufferSize() != null) {
maxWriteBufferSize = request.getMaxWriteBufferSize();
} else {
maxWriteBufferSize = DEFAULT_MAX_WRITE_BUFFER_SIZE;
}
TimeValue maxRetryDelay = request.getMaxRetryDelay() == null ? DEFAULT_MAX_RETRY_DELAY : request.getMaxRetryDelay();
TimeValue pollTimeout = request.getPollTimeout() == null ? DEFAULT_POLL_TIMEOUT : request.getPollTimeout();
return new ShardFollowTask(
clusterAliasName,
new ShardId(followIndexMetadata.getIndex(), shardId),
new ShardId(leaderIndexMetadata.getIndex(), shardId),
maxBatchOperationCount,
maxConcurrentReadBatches,
maxOperationSizeInBytes,
maxConcurrentWriteBatches,
maxWriteBufferSize,
maxRetryDelay,
pollTimeout,
recordedLeaderShardHistoryUUID,
filteredHeaders
);
}
private static String[] extractIndexShardHistoryUUIDs(Map<String, String> ccrIndexMetaData) {
String historyUUIDs = ccrIndexMetaData.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS);
return historyUUIDs.split(",");

2
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
View File

@ -156,7 +156,7 @@ public class TransportPutAutoFollowPatternAction extends
request.getMaxConcurrentWriteBatches(),
request.getMaxWriteBufferSize(),
request.getMaxRetryDelay(),
request.getIdleShardRetryDelay(),
request.getPollTimeout(),
filteredHeaders);
patterns.put(request.getLeaderClusterAlias(), autoFollowPattern);
ClusterState.Builder newState = ClusterState.builder(localState);

16
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java
View File

@ -192,16 +192,12 @@ public class CcrLicenseIT extends ESSingleNodeTestCase {
}
private FollowIndexAction.Request getFollowRequest() {
return new FollowIndexAction.Request(
"leader",
"follower",
FollowIndexAction.DEFAULT_MAX_BATCH_OPERATION_COUNT,
FollowIndexAction.DEFAULT_MAX_CONCURRENT_READ_BATCHES,
FollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES,
FollowIndexAction.DEFAULT_MAX_CONCURRENT_WRITE_BATCHES,
FollowIndexAction.DEFAULT_MAX_WRITE_BUFFER_SIZE,
TimeValue.timeValueMillis(10),
TimeValue.timeValueMillis(10));
FollowIndexAction.Request request = new FollowIndexAction.Request();
request.setLeaderIndex("leader");
request.setFollowerIndex("follower");
request.setMaxRetryDelay(TimeValue.timeValueMillis(10));
request.setPollTimeout(TimeValue.timeValueMillis(10));
return request;
}
}

45
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/ShardChangesIT.java
View File

@ -319,9 +319,11 @@ public class ShardChangesIT extends ESIntegTestCase {
long numDocsIndexed = Math.min(3000 * 2, randomLongBetween(maxReadSize, maxReadSize * 10));
atLeastDocsIndexed("index1", numDocsIndexed / 3);
final FollowIndexAction.Request followRequest = new FollowIndexAction.Request("index1", "index2", maxReadSize,
randomIntBetween(2, 10), Long.MAX_VALUE, randomIntBetween(2, 10),
randomIntBetween(1024, 10240), TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
FollowIndexAction.Request followRequest = createFollowRequest("index1", "index2");
followRequest.setMaxBatchOperationCount(maxReadSize);
followRequest.setMaxConcurrentReadBatches(randomIntBetween(2, 10));
followRequest.setMaxConcurrentWriteBatches(randomIntBetween(2, 10));
followRequest.setMaxWriteBufferSize(randomIntBetween(1024, 10240));
CreateAndFollowIndexAction.Request createAndFollowRequest = new CreateAndFollowIndexAction.Request(followRequest);
client().execute(CreateAndFollowIndexAction.INSTANCE, createAndFollowRequest).get();
@ -358,9 +360,10 @@ public class ShardChangesIT extends ESIntegTestCase {
});
thread.start();
final FollowIndexAction.Request followRequest = new FollowIndexAction.Request("index1", "index2", randomIntBetween(32, 2048),
randomIntBetween(2, 10), Long.MAX_VALUE, randomIntBetween(2, 10),
FollowIndexAction.DEFAULT_MAX_WRITE_BUFFER_SIZE, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
FollowIndexAction.Request followRequest = createFollowRequest("index1", "index2");
followRequest.setMaxBatchOperationCount(randomIntBetween(32, 2048));
followRequest.setMaxConcurrentReadBatches(randomIntBetween(2, 10));
followRequest.setMaxConcurrentWriteBatches(randomIntBetween(2, 10));
client().execute(CreateAndFollowIndexAction.INSTANCE, new CreateAndFollowIndexAction.Request(followRequest)).get();
long maxNumDocsReplicated = Math.min(1000, randomLongBetween(followRequest.getMaxBatchOperationCount(),
@ -447,7 +450,7 @@ public class ShardChangesIT extends ESIntegTestCase {
.actionGet());
}
public void testFollowIndex_lowMaxTranslogBytes() throws Exception {
public void testFollowIndexMaxOperationSizeInBytes() throws Exception {
final String leaderIndexSettings = getIndexSettings(1, between(0, 1),
singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true"));
assertAcked(client().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON));
@ -460,8 +463,8 @@ public class ShardChangesIT extends ESIntegTestCase {
client().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get();
}
final FollowIndexAction.Request followRequest = new FollowIndexAction.Request("index1", "index2", 1024, 1, 1024L,
1, 10240, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
FollowIndexAction.Request followRequest = createFollowRequest("index1", "index2");
followRequest.setMaxOperationSizeInBytes(1L);
final CreateAndFollowIndexAction.Request createAndFollowRequest = new CreateAndFollowIndexAction.Request(followRequest);
client().execute(CreateAndFollowIndexAction.INSTANCE, createAndFollowRequest).get();
@ -489,25 +492,21 @@ public class ShardChangesIT extends ESIntegTestCase {
assertAcked(client().admin().indices().prepareCreate("index3").setSource(leaderIndexSettings, XContentType.JSON));
ensureGreen("index3");
FollowIndexAction.Request followRequest = new FollowIndexAction.Request("index1", "index2", 1024, 1, 1024L,
1, 10240, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
FollowIndexAction.Request followRequest = createFollowRequest("index1", "index2");
CreateAndFollowIndexAction.Request createAndFollowRequest = new CreateAndFollowIndexAction.Request(followRequest);
client().execute(CreateAndFollowIndexAction.INSTANCE, createAndFollowRequest).get();
followRequest = new FollowIndexAction.Request("index3", "index4", 1024, 1, 1024L,
1, 10240, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
followRequest = createFollowRequest("index3", "index4");
createAndFollowRequest = new CreateAndFollowIndexAction.Request(followRequest);
client().execute(CreateAndFollowIndexAction.INSTANCE, createAndFollowRequest).get();
unfollowIndex("index2", "index4");
FollowIndexAction.Request wrongRequest1 = new FollowIndexAction.Request("index1", "index4", 1024, 1, 1024L,
1, 10240, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
FollowIndexAction.Request wrongRequest1 = createFollowRequest("index1", "index4");
Exception e = expectThrows(IllegalArgumentException.class,
() -> client().execute(FollowIndexAction.INSTANCE, wrongRequest1).actionGet());
assertThat(e.getMessage(), containsString("follow index [index4] should reference"));
FollowIndexAction.Request wrongRequest2 = new FollowIndexAction.Request("index3", "index2", 1024, 1, 1024L,
1, 10240, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
FollowIndexAction.Request wrongRequest2 = createFollowRequest("index3", "index2");
e = expectThrows(IllegalArgumentException.class, () -> client().execute(FollowIndexAction.INSTANCE, wrongRequest2).actionGet());
assertThat(e.getMessage(), containsString("follow index [index2] should reference"));
}
@ -716,10 +715,12 @@ public class ShardChangesIT extends ESIntegTestCase {
}, 60, TimeUnit.SECONDS);
}
public static FollowIndexAction.Request createFollowRequest(String leaderIndex, String followIndex) {
return new FollowIndexAction.Request(leaderIndex, followIndex, FollowIndexAction.DEFAULT_MAX_BATCH_OPERATION_COUNT,
FollowIndexAction.DEFAULT_MAX_CONCURRENT_READ_BATCHES, FollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES,
FollowIndexAction.DEFAULT_MAX_CONCURRENT_WRITE_BATCHES, FollowIndexAction.DEFAULT_MAX_WRITE_BUFFER_SIZE,
TimeValue.timeValueMillis(10), TimeValue.timeValueMillis(10));
public static FollowIndexAction.Request createFollowRequest(String leaderIndex, String followerIndex) {
FollowIndexAction.Request request = new FollowIndexAction.Request();
request.setLeaderIndex(leaderIndex);
request.setFollowerIndex(followerIndex);
request.setMaxRetryDelay(TimeValue.timeValueMillis(10));
request.setPollTimeout(TimeValue.timeValueMillis(10));
return request;
}
}

6
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowTests.java
View File

@ -136,7 +136,7 @@ public class AutoFollowTests extends ESSingleNodeTestCase {
request.setMaxRetryDelay(TimeValue.timeValueMillis(500));
}
if (randomBoolean()) {
request.setIdleShardRetryDelay(TimeValue.timeValueMillis(500));
request.setPollTimeout(TimeValue.timeValueMillis(500));
}
assertTrue(client().execute(PutAutoFollowPatternAction.INSTANCE, request).actionGet().isAcknowledged());
@ -167,8 +167,8 @@ public class AutoFollowTests extends ESSingleNodeTestCase {
if (request.getMaxRetryDelay() != null) {
assertThat(shardFollowTask.getMaxRetryDelay(), equalTo(request.getMaxRetryDelay()));
}
if (request.getIdleShardRetryDelay() != null) {
assertThat(shardFollowTask.getPollTimeout(), equalTo(request.getIdleShardRetryDelay()));
if (request.getPollTimeout() != null) {
assertThat(shardFollowTask.getPollTimeout(), equalTo(request.getPollTimeout()));
}
});
}

39
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowIndexRequestTests.java
View File

@ -40,24 +40,49 @@ public class FollowIndexRequestTests extends AbstractStreamableXContentTestCase<
}
static FollowIndexAction.Request createTestRequest() {
return new FollowIndexAction.Request(randomAlphaOfLength(4), randomAlphaOfLength(4), randomIntBetween(1, Integer.MAX_VALUE),
randomIntBetween(1, Integer.MAX_VALUE), randomNonNegativeLong(), randomIntBetween(1, Integer.MAX_VALUE),
randomIntBetween(1, Integer.MAX_VALUE), TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(500));
FollowIndexAction.Request request = new FollowIndexAction.Request();
request.setLeaderIndex(randomAlphaOfLength(4));
request.setFollowerIndex(randomAlphaOfLength(4));
if (randomBoolean()) {
request.setMaxBatchOperationCount(randomIntBetween(1, Integer.MAX_VALUE));
}
if (randomBoolean()) {
request.setMaxConcurrentReadBatches(randomIntBetween(1, Integer.MAX_VALUE));
}
if (randomBoolean()) {
request.setMaxConcurrentWriteBatches(randomIntBetween(1, Integer.MAX_VALUE));
}
if (randomBoolean()) {
request.setMaxOperationSizeInBytes(randomNonNegativeLong());
}
if (randomBoolean()) {
request.setMaxWriteBufferSize(randomIntBetween(1, Integer.MAX_VALUE));
}
if (randomBoolean()) {
request.setMaxRetryDelay(TimeValue.timeValueMillis(500));
}
if (randomBoolean()) {
request.setPollTimeout(TimeValue.timeValueMillis(500));
}
return request;
}
public void testValidate() {
FollowIndexAction.Request request = new FollowIndexAction.Request("index1", "index2", null, null, null, null,
null, TimeValue.ZERO, null);
FollowIndexAction.Request request = new FollowIndexAction.Request();
request.setLeaderIndex("index1");
request.setFollowerIndex("index2");
request.setMaxRetryDelay(TimeValue.ZERO);
ActionRequestValidationException validationException = request.validate();
assertThat(validationException, notNullValue());
assertThat(validationException.getMessage(), containsString("[max_retry_delay] must be positive but was [0ms]"));
request = new FollowIndexAction.Request("index1", "index2", null, null, null, null, null, TimeValue.timeValueMinutes(10), null);
request.setMaxRetryDelay(TimeValue.timeValueMinutes(10));
validationException = request.validate();
assertThat(validationException, notNullValue());
assertThat(validationException.getMessage(), containsString("[max_retry_delay] must be less than [5m] but was [10m]"));
request = new FollowIndexAction.Request("index1", "index2", null, null, null, null, null, TimeValue.timeValueMinutes(1), null);
request.setMaxRetryDelay(TimeValue.timeValueMinutes(1));
validationException = request.validate();
assertThat(validationException, nullValue());
}

2
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
View File

@ -45,7 +45,7 @@ public class PutAutoFollowPatternRequestTests extends AbstractStreamableXContent
request.setFollowIndexNamePattern(randomAlphaOfLength(4));
}
if (randomBoolean()) {
request.setIdleShardRetryDelay(TimeValue.timeValueMillis(500));
request.setPollTimeout(TimeValue.timeValueMillis(500));
}
if (randomBoolean()) {
request.setMaxRetryDelay(TimeValue.timeValueMillis(500));

3
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskRandomTests.java
View File

@ -15,7 +15,6 @@ import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsResponse;
import org.elasticsearch.xpack.core.ccr.action.FollowIndexAction;
import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
import java.nio.charset.StandardCharsets;
@ -81,7 +80,7 @@ public class ShardFollowNodeTaskRandomTests extends ESTestCase {
new ShardId("leader_index", "", 0),
testRun.maxOperationCount,
concurrency,
FollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES,
TransportFollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES,
concurrency,
10240,
TimeValue.timeValueMillis(10),

69
x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
View File

@ -81,31 +81,21 @@ public class SourceOnlySnapshot {
String segmentFileName;
try (Lock writeLock = targetDirectory.obtainLock(IndexWriter.WRITE_LOCK_NAME);
StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(commit)) {
SegmentInfos segmentInfos = reader.getSegmentInfos();
SegmentInfos segmentInfos = reader.getSegmentInfos().clone();
DirectoryReader wrappedReader = wrapReader(reader);
List<SegmentCommitInfo> newInfos = new ArrayList<>();
for (LeafReaderContext ctx : reader.leaves()) {
for (LeafReaderContext ctx : wrappedReader.leaves()) {
LeafReader leafReader = ctx.reader();
SegmentCommitInfo info = reader.getSegmentInfos().info(ctx.ord);
assert info.info.equals(Lucene.segmentReader(ctx.reader()).getSegmentInfo().info);
/* We could do this totally different without wrapping this dummy directory reader if FilterCodecReader would have a
* getDelegate method. This is fixed in LUCENE-8502 but we need to wait for it to come in 7.5.1 or 7.6.
* The reason here is that the ctx.ord is not guaranteed to be equivalent to the SegmentCommitInfo ord in the SegmentInfo
* object since we might drop fully deleted segments. if that happens we are using the wrong reader for the SI and
* might almost certainly expose deleted documents.
*/
DirectoryReader wrappedReader = wrapReader(new DummyDirectoryReader(reader.directory(), leafReader));
if (wrappedReader.leaves().isEmpty() == false) {
leafReader = wrappedReader.leaves().get(0).reader();
SegmentCommitInfo info = Lucene.segmentReader(leafReader).getSegmentInfo();
LiveDocs liveDocs = getLiveDocs(leafReader);
if (leafReader.numDocs() != 0) { // fully deleted segments don't need to be processed
SegmentCommitInfo newInfo = syncSegment(info, liveDocs, leafReader.getFieldInfos(), existingSegments, createdFiles);
newInfos.add(newInfo);
}
}
}
segmentInfos.clear();
segmentInfos.addAll(newInfos);
segmentInfos.setNextWriteGeneration(Math.max(segmentInfos.getGeneration(), generation)+1);
segmentInfos.setNextWriteGeneration(Math.max(segmentInfos.getGeneration(), generation) + 1);
String pendingSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS,
"", segmentInfos.getGeneration());
try (IndexOutput segnOutput = targetDirectory.createOutput(pendingSegmentFileName, IOContext.DEFAULT)) {
@ -250,7 +240,7 @@ public class SourceOnlySnapshot {
private boolean assertLiveDocs(Bits liveDocs, int deletes) {
int actualDeletes = 0;
for (int i = 0; i < liveDocs.length(); i++ ) {
for (int i = 0; i < liveDocs.length(); i++) {
if (liveDocs.get(i) == false) {
actualDeletes++;
}
@ -268,51 +258,4 @@ public class SourceOnlySnapshot {
this.bits = bits;
}
}
private static class DummyDirectoryReader extends DirectoryReader {
protected DummyDirectoryReader(Directory directory, LeafReader... segmentReaders) throws IOException {
super(directory, segmentReaders);
}
@Override
protected DirectoryReader doOpenIfChanged() throws IOException {
return null;
}
@Override
protected DirectoryReader doOpenIfChanged(IndexCommit commit) throws IOException {
return null;
}
@Override
protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws IOException {
return null;
}
@Override
public long getVersion() {
return 0;
}
@Override
public boolean isCurrent() throws IOException {
return false;
}
@Override
public IndexCommit getIndexCommit() throws IOException {
return null;
}
@Override
protected void doClose() throws IOException {
}
@Override
public CacheHelper getReaderCacheHelper() {
return null;
}
}
}

28
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java
View File

@ -171,7 +171,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
public static final ParseField MAX_CONCURRENT_WRITE_BATCHES = new ParseField("max_concurrent_write_batches");
public static final ParseField MAX_WRITE_BUFFER_SIZE = new ParseField("max_write_buffer_size");
public static final ParseField MAX_RETRY_DELAY = new ParseField("max_retry_delay");
public static final ParseField IDLE_SHARD_RETRY_DELAY = new ParseField("idle_shard_retry_delay");
public static final ParseField POLL_TIMEOUT = new ParseField("poll_timeout");
private static final ParseField HEADERS = new ParseField("headers");
@SuppressWarnings("unchecked")
@ -193,8 +193,8 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
(p, c) -> TimeValue.parseTimeValue(p.text(), MAX_RETRY_DELAY.getPreferredName()),
MAX_RETRY_DELAY, ObjectParser.ValueType.STRING);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> TimeValue.parseTimeValue(p.text(), IDLE_SHARD_RETRY_DELAY.getPreferredName()),
IDLE_SHARD_RETRY_DELAY, ObjectParser.ValueType.STRING);
(p, c) -> TimeValue.parseTimeValue(p.text(), POLL_TIMEOUT.getPreferredName()),
POLL_TIMEOUT, ObjectParser.ValueType.STRING);
PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.mapStrings(), HEADERS);
}
@ -206,7 +206,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
private final Integer maxConcurrentWriteBatches;
private final Integer maxWriteBufferSize;
private final TimeValue maxRetryDelay;
private final TimeValue idleShardRetryDelay;
private final TimeValue pollTimeout;
private final Map<String, String> headers;
public AutoFollowPattern(List<String> leaderIndexPatterns,
@ -217,7 +217,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
Integer maxConcurrentWriteBatches,
Integer maxWriteBufferSize,
TimeValue maxRetryDelay,
TimeValue idleShardRetryDelay,
TimeValue pollTimeout,
Map<String, String> headers) {
this.leaderIndexPatterns = leaderIndexPatterns;
this.followIndexPattern = followIndexPattern;
@ -227,7 +227,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
this.maxConcurrentWriteBatches = maxConcurrentWriteBatches;
this.maxWriteBufferSize = maxWriteBufferSize;
this.maxRetryDelay = maxRetryDelay;
this.idleShardRetryDelay = idleShardRetryDelay;
this.pollTimeout = pollTimeout;
this.headers = headers != null ? Collections.unmodifiableMap(headers) : Collections.emptyMap();
}
@ -240,7 +240,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
maxConcurrentWriteBatches = in.readOptionalVInt();
maxWriteBufferSize = in.readOptionalVInt();
maxRetryDelay = in.readOptionalTimeValue();
idleShardRetryDelay = in.readOptionalTimeValue();
pollTimeout = in.readOptionalTimeValue();
this.headers = Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString));
}
@ -284,8 +284,8 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
return maxRetryDelay;
}
public TimeValue getIdleShardRetryDelay() {
return idleShardRetryDelay;
public TimeValue getPollTimeout() {
return pollTimeout;
}
public Map<String, String> getHeaders() {
@ -302,7 +302,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
out.writeOptionalVInt(maxConcurrentWriteBatches);
out.writeOptionalVInt(maxWriteBufferSize);
out.writeOptionalTimeValue(maxRetryDelay);
out.writeOptionalTimeValue(idleShardRetryDelay);
out.writeOptionalTimeValue(pollTimeout);
out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString);
}
@ -330,8 +330,8 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
if (maxRetryDelay != null) {
builder.field(MAX_RETRY_DELAY.getPreferredName(), maxRetryDelay);
}
if (idleShardRetryDelay != null) {
builder.field(IDLE_SHARD_RETRY_DELAY.getPreferredName(), idleShardRetryDelay);
if (pollTimeout != null) {
builder.field(POLL_TIMEOUT.getPreferredName(), pollTimeout);
}
builder.field(HEADERS.getPreferredName(), headers);
return builder;
@ -355,7 +355,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
Objects.equals(maxConcurrentWriteBatches, that.maxConcurrentWriteBatches) &&
Objects.equals(maxWriteBufferSize, that.maxWriteBufferSize) &&
Objects.equals(maxRetryDelay, that.maxRetryDelay) &&
Objects.equals(idleShardRetryDelay, that.idleShardRetryDelay) &&
Objects.equals(pollTimeout, that.pollTimeout) &&
Objects.equals(headers, that.headers);
}
@ -370,7 +370,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
maxConcurrentWriteBatches,
maxWriteBufferSize,
maxRetryDelay,
idleShardRetryDelay,
pollTimeout,
headers
);
}

228
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowIndexAction.java
View File

@ -14,7 +14,6 @@ import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -30,14 +29,7 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
public static final FollowIndexAction INSTANCE = new FollowIndexAction();
public static final String NAME = "cluster:admin/xpack/ccr/follow_index";
public static final int DEFAULT_MAX_WRITE_BUFFER_SIZE = 10240;
public static final int DEFAULT_MAX_BATCH_OPERATION_COUNT = 1024;
public static final int DEFAULT_MAX_CONCURRENT_READ_BATCHES = 1;
public static final int DEFAULT_MAX_CONCURRENT_WRITE_BATCHES = 1;
public static final long DEFAULT_MAX_BATCH_SIZE_IN_BYTES = Long.MAX_VALUE;
static final TimeValue DEFAULT_MAX_RETRY_DELAY = new TimeValue(500);
static final TimeValue MAX_RETRY_DELAY = TimeValue.timeValueMinutes(5);
public static final TimeValue DEFAULT_POLL_TIMEOUT = TimeValue.timeValueMinutes(1);
public static final TimeValue MAX_RETRY_DELAY = TimeValue.timeValueMinutes(5);
private FollowIndexAction() {
super(NAME);
@ -59,30 +51,23 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
private static final ParseField MAX_WRITE_BUFFER_SIZE = new ParseField("max_write_buffer_size");
private static final ParseField MAX_RETRY_DELAY_FIELD = new ParseField("max_retry_delay");
private static final ParseField POLL_TIMEOUT = new ParseField("poll_timeout");
private static final ConstructingObjectParser<Request, String> PARSER = new ConstructingObjectParser<>(NAME, true,
(args, followerIndex) -> {
if (args[1] != null) {
followerIndex = (String) args[1];
}
return new Request((String) args[0], followerIndex, (Integer) args[2], (Integer) args[3], (Long) args[4],
(Integer) args[5], (Integer) args[6], (TimeValue) args[7], (TimeValue) args[8]);
});
private static final ObjectParser<Request, String> PARSER = new ObjectParser<>(NAME, Request::new);
static {
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), LEADER_INDEX_FIELD);
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FOLLOWER_INDEX_FIELD);
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_BATCH_OPERATION_COUNT);
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_CONCURRENT_READ_BATCHES);
PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MAX_BATCH_SIZE_IN_BYTES);
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_CONCURRENT_WRITE_BATCHES);
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_WRITE_BUFFER_SIZE);
PARSER.declareString(Request::setLeaderIndex, LEADER_INDEX_FIELD);
PARSER.declareString(Request::setFollowerIndex, FOLLOWER_INDEX_FIELD);
PARSER.declareInt(Request::setMaxBatchOperationCount, MAX_BATCH_OPERATION_COUNT);
PARSER.declareInt(Request::setMaxConcurrentReadBatches, MAX_CONCURRENT_READ_BATCHES);
PARSER.declareLong(Request::setMaxOperationSizeInBytes, MAX_BATCH_SIZE_IN_BYTES);
PARSER.declareInt(Request::setMaxConcurrentWriteBatches, MAX_CONCURRENT_WRITE_BATCHES);
PARSER.declareInt(Request::setMaxWriteBufferSize, MAX_WRITE_BUFFER_SIZE);
PARSER.declareField(
ConstructingObjectParser.optionalConstructorArg(),
Request::setMaxRetryDelay,
(p, c) -> TimeValue.parseTimeValue(p.text(), MAX_RETRY_DELAY_FIELD.getPreferredName()),
MAX_RETRY_DELAY_FIELD,
ObjectParser.ValueType.STRING);
PARSER.declareField(
ConstructingObjectParser.optionalConstructorArg(),
Request::setPollTimeout,
(p, c) -> TimeValue.parseTimeValue(p.text(), POLL_TIMEOUT.getPreferredName()),
POLL_TIMEOUT,
ObjectParser.ValueType.STRING);
@ -108,6 +93,9 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
return leaderIndex;
}
public void setLeaderIndex(String leaderIndex) {
this.leaderIndex = leaderIndex;
}
private String followerIndex;
@ -115,38 +103,66 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
return followerIndex;
}
private int maxBatchOperationCount;
public void setFollowerIndex(String followerIndex) {
this.followerIndex = followerIndex;
}
public int getMaxBatchOperationCount() {
private Integer maxBatchOperationCount;
public Integer getMaxBatchOperationCount() {
return maxBatchOperationCount;
}
private int maxConcurrentReadBatches;
public void setMaxBatchOperationCount(Integer maxBatchOperationCount) {
this.maxBatchOperationCount = maxBatchOperationCount;
}
public int getMaxConcurrentReadBatches() {
private Integer maxConcurrentReadBatches;
public Integer getMaxConcurrentReadBatches() {
return maxConcurrentReadBatches;
}
private long maxOperationSizeInBytes;
public void setMaxConcurrentReadBatches(Integer maxConcurrentReadBatches) {
this.maxConcurrentReadBatches = maxConcurrentReadBatches;
}
public long getMaxOperationSizeInBytes() {
private Long maxOperationSizeInBytes;
public Long getMaxOperationSizeInBytes() {
return maxOperationSizeInBytes;
}
private int maxConcurrentWriteBatches;
public void setMaxOperationSizeInBytes(Long maxOperationSizeInBytes) {
this.maxOperationSizeInBytes = maxOperationSizeInBytes;
}
public int getMaxConcurrentWriteBatches() {
private Integer maxConcurrentWriteBatches;
public Integer getMaxConcurrentWriteBatches() {
return maxConcurrentWriteBatches;
}
private int maxWriteBufferSize;
public void setMaxConcurrentWriteBatches(Integer maxConcurrentWriteBatches) {
this.maxConcurrentWriteBatches = maxConcurrentWriteBatches;
}
public int getMaxWriteBufferSize() {
private Integer maxWriteBufferSize;
public Integer getMaxWriteBufferSize() {
return maxWriteBufferSize;
}
public void setMaxWriteBufferSize(Integer maxWriteBufferSize) {
this.maxWriteBufferSize = maxWriteBufferSize;
}
private TimeValue maxRetryDelay;
public void setMaxRetryDelay(TimeValue maxRetryDelay) {
this.maxRetryDelay = maxRetryDelay;
}
public TimeValue getMaxRetryDelay() {
return maxRetryDelay;
}
@ -157,88 +173,50 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
return pollTimeout;
}
public Request(
final String leaderIndex,
final String followerIndex,
final Integer maxBatchOperationCount,
final Integer maxConcurrentReadBatches,
final Long maxOperationSizeInBytes,
final Integer maxConcurrentWriteBatches,
final Integer maxWriteBufferSize,
final TimeValue maxRetryDelay,
final TimeValue pollTimeout) {
if (leaderIndex == null) {
throw new IllegalArgumentException(LEADER_INDEX_FIELD.getPreferredName() + " is missing");
}
if (followerIndex == null) {
throw new IllegalArgumentException(FOLLOWER_INDEX_FIELD.getPreferredName() + " is missing");
}
final int actualMaxBatchOperationCount =
maxBatchOperationCount == null ? DEFAULT_MAX_BATCH_OPERATION_COUNT : maxBatchOperationCount;
if (actualMaxBatchOperationCount < 1) {
throw new IllegalArgumentException(MAX_BATCH_OPERATION_COUNT.getPreferredName() + " must be larger than 0");
}
final int actualMaxConcurrentReadBatches =
maxConcurrentReadBatches == null ? DEFAULT_MAX_CONCURRENT_READ_BATCHES : maxConcurrentReadBatches;
if (actualMaxConcurrentReadBatches < 1) {
throw new IllegalArgumentException(MAX_CONCURRENT_READ_BATCHES.getPreferredName() + " must be larger than 0");
}
final long actualMaxOperationSizeInBytes =
maxOperationSizeInBytes == null ? DEFAULT_MAX_BATCH_SIZE_IN_BYTES : maxOperationSizeInBytes;
if (actualMaxOperationSizeInBytes <= 0) {
throw new IllegalArgumentException(MAX_BATCH_SIZE_IN_BYTES.getPreferredName() + " must be larger than 0");
}
final int actualMaxConcurrentWriteBatches =
maxConcurrentWriteBatches == null ? DEFAULT_MAX_CONCURRENT_WRITE_BATCHES : maxConcurrentWriteBatches;
if (actualMaxConcurrentWriteBatches < 1) {
throw new IllegalArgumentException(MAX_CONCURRENT_WRITE_BATCHES.getPreferredName() + " must be larger than 0");
}
final int actualMaxWriteBufferSize = maxWriteBufferSize == null ? DEFAULT_MAX_WRITE_BUFFER_SIZE : maxWriteBufferSize;
if (actualMaxWriteBufferSize < 1) {
throw new IllegalArgumentException(MAX_WRITE_BUFFER_SIZE.getPreferredName() + " must be larger than 0");
}
final TimeValue actualRetryTimeout = maxRetryDelay == null ? DEFAULT_MAX_RETRY_DELAY : maxRetryDelay;
final TimeValue actualPollTimeout = pollTimeout == null ? DEFAULT_POLL_TIMEOUT : pollTimeout;
this.leaderIndex = leaderIndex;
this.followerIndex = followerIndex;
this.maxBatchOperationCount = actualMaxBatchOperationCount;
this.maxConcurrentReadBatches = actualMaxConcurrentReadBatches;
this.maxOperationSizeInBytes = actualMaxOperationSizeInBytes;
this.maxConcurrentWriteBatches = actualMaxConcurrentWriteBatches;
this.maxWriteBufferSize = actualMaxWriteBufferSize;
this.maxRetryDelay = actualRetryTimeout;
this.pollTimeout = actualPollTimeout;
public void setPollTimeout(TimeValue pollTimeout) {
this.pollTimeout = pollTimeout;
}
public Request() {
}
@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
ActionRequestValidationException e = null;
if (maxRetryDelay.millis() <= 0) {
if (leaderIndex == null) {
e = addValidationError(LEADER_INDEX_FIELD.getPreferredName() + " is missing", e);
}
if (followerIndex == null) {
e = addValidationError(FOLLOWER_INDEX_FIELD.getPreferredName() + " is missing", e);
}
if (maxBatchOperationCount != null && maxBatchOperationCount < 1) {
e = addValidationError(MAX_BATCH_OPERATION_COUNT.getPreferredName() + " must be larger than 0", e);
}
if (maxConcurrentReadBatches != null && maxConcurrentReadBatches < 1) {
e = addValidationError(MAX_CONCURRENT_READ_BATCHES.getPreferredName() + " must be larger than 0", e);
}
if (maxOperationSizeInBytes != null && maxOperationSizeInBytes <= 0) {
e = addValidationError(MAX_BATCH_SIZE_IN_BYTES.getPreferredName() + " must be larger than 0", e);
}
if (maxConcurrentWriteBatches != null && maxConcurrentWriteBatches < 1) {
e = addValidationError(MAX_CONCURRENT_WRITE_BATCHES.getPreferredName() + " must be larger than 0", e);
}
if (maxWriteBufferSize != null && maxWriteBufferSize < 1) {
e = addValidationError(MAX_WRITE_BUFFER_SIZE.getPreferredName() + " must be larger than 0", e);
}
if (maxRetryDelay != null && maxRetryDelay.millis() <= 0) {
String message = "[" + MAX_RETRY_DELAY_FIELD.getPreferredName() + "] must be positive but was [" +
maxRetryDelay.getStringRep() + "]";
validationException = addValidationError(message, validationException);
e = addValidationError(message, e);
}
if (maxRetryDelay.millis() > FollowIndexAction.MAX_RETRY_DELAY.millis()) {
if (maxRetryDelay != null && maxRetryDelay.millis() > FollowIndexAction.MAX_RETRY_DELAY.millis()) {
String message = "[" + MAX_RETRY_DELAY_FIELD.getPreferredName() + "] must be less than [" + MAX_RETRY_DELAY +
"] but was [" + maxRetryDelay.getStringRep() + "]";
validationException = addValidationError(message, validationException);
e = addValidationError(message, e);
}
return validationException;
return e;
}
@Override
@ -246,11 +224,11 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
super.readFrom(in);
leaderIndex = in.readString();
followerIndex = in.readString();
maxBatchOperationCount = in.readVInt();
maxConcurrentReadBatches = in.readVInt();
maxOperationSizeInBytes = in.readVLong();
maxConcurrentWriteBatches = in.readVInt();
maxWriteBufferSize = in.readVInt();
maxBatchOperationCount = in.readOptionalVInt();
maxConcurrentReadBatches = in.readOptionalVInt();
maxOperationSizeInBytes = in.readOptionalLong();
maxConcurrentWriteBatches = in.readOptionalVInt();
maxWriteBufferSize = in.readOptionalVInt();
maxRetryDelay = in.readOptionalTimeValue();
pollTimeout = in.readOptionalTimeValue();
}
@ -260,11 +238,11 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
super.writeTo(out);
out.writeString(leaderIndex);
out.writeString(followerIndex);
out.writeVInt(maxBatchOperationCount);
out.writeVInt(maxConcurrentReadBatches);
out.writeVLong(maxOperationSizeInBytes);
out.writeVInt(maxConcurrentWriteBatches);
out.writeVInt(maxWriteBufferSize);
out.writeOptionalVInt(maxBatchOperationCount);
out.writeOptionalVInt(maxConcurrentReadBatches);
out.writeOptionalLong(maxOperationSizeInBytes);
out.writeOptionalVInt(maxConcurrentWriteBatches);
out.writeOptionalVInt(maxWriteBufferSize);
out.writeOptionalTimeValue(maxRetryDelay);
out.writeOptionalTimeValue(pollTimeout);
}
@ -275,14 +253,28 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
{
builder.field(LEADER_INDEX_FIELD.getPreferredName(), leaderIndex);
builder.field(FOLLOWER_INDEX_FIELD.getPreferredName(), followerIndex);
if (maxBatchOperationCount != null) {
builder.field(MAX_BATCH_OPERATION_COUNT.getPreferredName(), maxBatchOperationCount);
}
if (maxOperationSizeInBytes != null) {
builder.field(MAX_BATCH_SIZE_IN_BYTES.getPreferredName(), maxOperationSizeInBytes);
}
if (maxWriteBufferSize != null) {
builder.field(MAX_WRITE_BUFFER_SIZE.getPreferredName(), maxWriteBufferSize);
}
if (maxConcurrentReadBatches != null) {
builder.field(MAX_CONCURRENT_READ_BATCHES.getPreferredName(), maxConcurrentReadBatches);
}
if (maxConcurrentWriteBatches != null) {
builder.field(MAX_CONCURRENT_WRITE_BATCHES.getPreferredName(), maxConcurrentWriteBatches);
}
if (maxRetryDelay != null) {
builder.field(MAX_RETRY_DELAY_FIELD.getPreferredName(), maxRetryDelay.getStringRep());
}
if (pollTimeout != null) {
builder.field(POLL_TIMEOUT.getPreferredName(), pollTimeout.getStringRep());
}
}
builder.endObject();
return builder;
}
@ -292,11 +284,11 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Request request = (Request) o;
return maxBatchOperationCount == request.maxBatchOperationCount &&
maxConcurrentReadBatches == request.maxConcurrentReadBatches &&
maxOperationSizeInBytes == request.maxOperationSizeInBytes &&
maxConcurrentWriteBatches == request.maxConcurrentWriteBatches &&
maxWriteBufferSize == request.maxWriteBufferSize &&
return Objects.equals(maxBatchOperationCount, request.maxBatchOperationCount) &&
Objects.equals(maxConcurrentReadBatches, request.maxConcurrentReadBatches) &&
Objects.equals(maxOperationSizeInBytes, request.maxOperationSizeInBytes) &&
Objects.equals(maxConcurrentWriteBatches, request.maxConcurrentWriteBatches) &&
Objects.equals(maxWriteBufferSize, request.maxWriteBufferSize) &&
Objects.equals(maxRetryDelay, request.maxRetryDelay) &&
Objects.equals(pollTimeout, request.pollTimeout) &&
Objects.equals(leaderIndex, request.leaderIndex) &&

28
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
View File

@ -59,9 +59,9 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
PARSER.declareField(Request::setMaxRetryDelay,
(p, c) -> TimeValue.parseTimeValue(p.text(), AutoFollowPattern.MAX_RETRY_DELAY.getPreferredName()),
AutoFollowPattern.MAX_RETRY_DELAY, ObjectParser.ValueType.STRING);
PARSER.declareField(Request::setIdleShardRetryDelay,
(p, c) -> TimeValue.parseTimeValue(p.text(), AutoFollowPattern.IDLE_SHARD_RETRY_DELAY.getPreferredName()),
AutoFollowPattern.IDLE_SHARD_RETRY_DELAY, ObjectParser.ValueType.STRING);
PARSER.declareField(Request::setPollTimeout,
(p, c) -> TimeValue.parseTimeValue(p.text(), AutoFollowPattern.POLL_TIMEOUT.getPreferredName()),
AutoFollowPattern.POLL_TIMEOUT, ObjectParser.ValueType.STRING);
}
public static Request fromXContent(XContentParser parser, String remoteClusterAlias) throws IOException {
@ -88,7 +88,7 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
private Integer maxConcurrentWriteBatches;
private Integer maxWriteBufferSize;
private TimeValue maxRetryDelay;
private TimeValue idleShardRetryDelay;
private TimeValue pollTimeout;
@Override
public ActionRequestValidationException validate() {
@ -189,12 +189,12 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
this.maxRetryDelay = maxRetryDelay;
}
public TimeValue getIdleShardRetryDelay() {
return idleShardRetryDelay;
public TimeValue getPollTimeout() {
return pollTimeout;
}
public void setIdleShardRetryDelay(TimeValue idleShardRetryDelay) {
this.idleShardRetryDelay = idleShardRetryDelay;
public void setPollTimeout(TimeValue pollTimeout) {
this.pollTimeout = pollTimeout;
}
@Override
@ -209,7 +209,7 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
maxConcurrentWriteBatches = in.readOptionalVInt();
maxWriteBufferSize = in.readOptionalVInt();
maxRetryDelay = in.readOptionalTimeValue();
idleShardRetryDelay = in.readOptionalTimeValue();
pollTimeout = in.readOptionalTimeValue();
}
@Override
@ -224,7 +224,7 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
out.writeOptionalVInt(maxConcurrentWriteBatches);
out.writeOptionalVInt(maxWriteBufferSize);
out.writeOptionalTimeValue(maxRetryDelay);
out.writeOptionalTimeValue(idleShardRetryDelay);
out.writeOptionalTimeValue(pollTimeout);
}
@Override
@ -254,8 +254,8 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
if (maxRetryDelay != null) {
builder.field(AutoFollowPattern.MAX_RETRY_DELAY.getPreferredName(), maxRetryDelay.getStringRep());
}
if (idleShardRetryDelay != null) {
builder.field(AutoFollowPattern.IDLE_SHARD_RETRY_DELAY.getPreferredName(), idleShardRetryDelay.getStringRep());
if (pollTimeout != null) {
builder.field(AutoFollowPattern.POLL_TIMEOUT.getPreferredName(), pollTimeout.getStringRep());
}
}
builder.endObject();
@ -276,7 +276,7 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
Objects.equals(maxConcurrentWriteBatches, request.maxConcurrentWriteBatches) &&
Objects.equals(maxWriteBufferSize, request.maxWriteBufferSize) &&
Objects.equals(maxRetryDelay, request.maxRetryDelay) &&
Objects.equals(idleShardRetryDelay, request.idleShardRetryDelay);
Objects.equals(pollTimeout, request.pollTimeout);
}
@Override
@ -291,7 +291,7 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
maxConcurrentWriteBatches,
maxWriteBufferSize,
maxRetryDelay,
idleShardRetryDelay
pollTimeout
);
}
}

83
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/JobStorageDeletionTask.java
View File

@ -11,8 +11,12 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Client;
@ -23,10 +27,12 @@ import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
import org.elasticsearch.index.query.IdsQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.DeleteByQueryAction;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskId;
import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction;
@ -69,11 +75,16 @@ public class JobStorageDeletionTask extends Task {
final String indexName = AnomalyDetectorsIndex.getPhysicalIndexFromState(state, jobId);
final String indexPattern = indexName + "-*";
ActionListener<Boolean> deleteAliasHandler = ActionListener.wrap(finishedHandler, failureHandler);
final ActionListener<AcknowledgedResponse> completionHandler = ActionListener.wrap(
response -> finishedHandler.accept(response.isAcknowledged()),
failureHandler);
// Step 5. DBQ state done, delete the aliases
// Step 7. If we did not drop the index and after DBQ state done, we delete the aliases
ActionListener<BulkByScrollResponse> dbqHandler = ActionListener.wrap(
bulkByScrollResponse -> {
if (bulkByScrollResponse == null) { // no action was taken by DBQ, assume Index was deleted
completionHandler.onResponse(new AcknowledgedResponse(true));
} else {
if (bulkByScrollResponse.isTimedOut()) {
logger.warn("[{}] DeleteByQuery for indices [{}, {}] timed out.", jobId, indexName, indexPattern);
}
@ -85,13 +96,15 @@ public class JobStorageDeletionTask extends Task {
logger.warn("DBQ failure: " + failure);
}
}
deleteAliases(jobId, client, deleteAliasHandler);
deleteAliases(jobId, client, completionHandler);
}
},
failureHandler);
// Step 4. Delete categorizer state done, DeleteByQuery on the index, matching all docs with the right job_id
ActionListener<Boolean> deleteCategorizerStateHandler = ActionListener.wrap(
// Step 6. If we did not delete the index, we run a delete by query
ActionListener<Boolean> deleteByQueryExecutor = ActionListener.wrap(
response -> {
if (response) {
logger.info("Running DBQ on [" + indexName + "," + indexPattern + "] for job [" + jobId + "]");
DeleteByQueryRequest request = new DeleteByQueryRequest(indexName, indexPattern);
ConstantScoreQueryBuilder query =
@ -103,9 +116,62 @@ public class JobStorageDeletionTask extends Task {
request.setRefresh(true);
executeAsyncWithOrigin(client, ML_ORIGIN, DeleteByQueryAction.INSTANCE, request, dbqHandler);
} else { // We did not execute DBQ, no need to delete aliases or check the response
dbqHandler.onResponse(null);
}
},
failureHandler);
// Step 5. If we have any hits, that means we are NOT the only job on this index, and should not delete it
// if we do not have any hits, we can drop the index and then skip the DBQ and alias deletion
ActionListener<SearchResponse> customIndexSearchHandler = ActionListener.wrap(
searchResponse -> {
if (searchResponse == null || searchResponse.getHits().totalHits > 0) {
deleteByQueryExecutor.onResponse(true); // We need to run DBQ and alias deletion
} else {
logger.info("Running DELETE Index on [" + indexName + "] for job [" + jobId + "]");
DeleteIndexRequest request = new DeleteIndexRequest(indexName);
request.indicesOptions(IndicesOptions.lenientExpandOpen());
// If we have deleted the index, then we don't need to delete the aliases or run the DBQ
executeAsyncWithOrigin(
client.threadPool().getThreadContext(),
ML_ORIGIN,
request,
ActionListener.<AcknowledgedResponse>wrap(
response -> deleteByQueryExecutor.onResponse(false), // skip DBQ && Alias
failureHandler),
client.admin().indices()::delete);
}
},
failure -> {
if (failure.getClass() == IndexNotFoundException.class) { // assume the index is already deleted
deleteByQueryExecutor.onResponse(false); // skip DBQ && Alias
} else {
failureHandler.accept(failure);
}
}
);
// Step 4. Determine if we are on a shared index by looking at `.ml-anomalies-shared` or the custom index's aliases
ActionListener<Boolean> deleteCategorizerStateHandler = ActionListener.wrap(
response -> {
if (indexName.equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX +
AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT)) {
customIndexSearchHandler.onResponse(null); //don't bother searching the index any further, we are on the default shared
} else {
SearchSourceBuilder source = new SearchSourceBuilder()
.size(1)
.query(QueryBuilders.boolQuery().filter(
QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId))));
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(source);
executeAsyncWithOrigin(client, ML_ORIGIN, SearchAction.INSTANCE, searchRequest, customIndexSearchHandler);
}
},
failureHandler
);
// Step 3. Delete quantiles done, delete the categorizer state
ActionListener<Boolean> deleteQuantilesHandler = ActionListener.wrap(
response -> deleteCategorizerState(jobId, client, 1, deleteCategorizerStateHandler),
@ -189,7 +255,7 @@ public class JobStorageDeletionTask extends Task {
}));
}
private void deleteAliases(String jobId, Client client, ActionListener<Boolean> finishedHandler) {
private void deleteAliases(String jobId, Client client, ActionListener<AcknowledgedResponse> finishedHandler) {
final String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);
final String writeAliasName = AnomalyDetectorsIndex.resultsWriteAlias(jobId);
@ -204,11 +270,12 @@ public class JobStorageDeletionTask extends Task {
if (removeRequest == null) {
// don't error if the job's aliases have already been deleted - carry on and delete the
// rest of the job's data
finishedHandler.onResponse(true);
finishedHandler.onResponse(new AcknowledgedResponse(true));
return;
}
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, removeRequest,
ActionListener.<AcknowledgedResponse>wrap(removeResponse -> finishedHandler.onResponse(true),
ActionListener.<AcknowledgedResponse>wrap(
finishedHandler::onResponse,
finishedHandler::onFailure),
client.admin().indices()::aliases);
},

19
x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponseTests.java
View File

@ -13,6 +13,8 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Predicate;
@ -88,12 +90,29 @@ public class GraphExploreResponseTests extends AbstractXContentTestCase< GraphEx
assertThat(newInstance.getTook(), equalTo(expectedInstance.getTook()));
assertThat(newInstance.isTimedOut(), equalTo(expectedInstance.isTimedOut()));
Comparator<Connection> connComparator = new Comparator<Connection>() {
@Override
public int compare(Connection o1, Connection o2) {
return o1.getId().toString().compareTo(o2.getId().toString());
}
};
Connection[] newConns = newInstance.getConnections().toArray(new Connection[0]);
Connection[] expectedConns = expectedInstance.getConnections().toArray(new Connection[0]);
Arrays.sort(newConns, connComparator);
Arrays.sort(expectedConns, connComparator);
assertArrayEquals(expectedConns, newConns);
//Sort the vertices lists before equality test (map insertion sequences can cause order differences)
Comparator<Vertex> comparator = new Comparator<Vertex>() {
@Override
public int compare(Vertex o1, Vertex o2) {
return o1.getId().toString().compareTo(o2.getId().toString());
}
};
Vertex[] newVertices = newInstance.getVertices().toArray(new Vertex[0]);
Vertex[] expectedVertices = expectedInstance.getVertices().toArray(new Vertex[0]);
Arrays.sort(newVertices, comparator);
Arrays.sort(expectedVertices, comparator);
assertArrayEquals(expectedVertices, newVertices);
ShardOperationFailedException[] newFailures = newInstance.getShardFailures();

40
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
View File

@ -192,6 +192,7 @@ public class MlJobIT extends ESRestTestCase {
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
{ //create jobId1 docs
String id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1234", 300);
Request createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/doc/" + id);
createResultRequest.setJsonEntity(String.format(Locale.ROOT,
@ -215,10 +216,36 @@ public class MlJobIT extends ESRestTestCase {
responseAsString = EntityUtils.toString(client().performRequest(
new Request("GET", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/_search")).getEntity());
assertThat(responseAsString, containsString("\"total\":2"));
}
{ //create jobId2 docs
String id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId2, "1234", 300);
Request createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/doc/" + id);
createResultRequest.setJsonEntity(String.format(Locale.ROOT,
"{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}",
jobId2, "1234", 1));
client().performRequest(createResultRequest);
id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId2, "1236", 300);
createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/doc/" + id);
createResultRequest.setJsonEntity(String.format(Locale.ROOT,
"{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}",
jobId2, "1236", 1));
client().performRequest(createResultRequest);
client().performRequest(new Request("POST", "/_refresh"));
responseAsString = EntityUtils.toString(client().performRequest(
new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2 + "/results/buckets")).getEntity());
assertThat(responseAsString, containsString("\"count\":2"));
responseAsString = EntityUtils.toString(client().performRequest(
new Request("GET", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/_search")).getEntity());
assertThat(responseAsString, containsString("\"total\":2"));
}
client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1));
// check that indices still exist, but are empty and aliases are gone
// check that indices still exist, but no longer have job1 entries and aliases are gone
responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists
@ -230,7 +257,16 @@ public class MlJobIT extends ESRestTestCase {
responseAsString = EntityUtils.toString(client().performRequest(
new Request("GET", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "/_count")).getEntity());
assertThat(responseAsString, containsString("\"count\":0"));
assertThat(responseAsString, containsString("\"count\":2"));
// Delete the second job and verify aliases are gone, and original concrete/custom index is gone
client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2));
responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
client().performRequest(new Request("POST", "/_refresh"));
responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)));
}
public void testCreateJobInSharedIndexUpdatesMapping() throws Exception {

7
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java
View File

@ -20,7 +20,6 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.CustomAnalyzer;
import org.elasticsearch.index.analysis.CustomAnalyzerProvider;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.indices.analysis.AnalysisModule;
@ -217,6 +216,8 @@ public class CategorizationAnalyzer implements Closeable {
Tuple<String, TokenizerFactory> tokenizerFactory,
List<CharFilterFactory> charFilterFactoryList) throws IOException {
List<CategorizationAnalyzerConfig.NameOrDefinition> tokenFilters = config.getTokenFilters();
TransportAnalyzeAction.DeferredTokenFilterRegistry deferredRegistry
= new TransportAnalyzeAction.DeferredTokenFilterRegistry(analysisRegistry, null);
final List<TokenFilterFactory> tokenFilterFactoryList = new ArrayList<>();
for (CategorizationAnalyzerConfig.NameOrDefinition tokenFilter : tokenFilters) {
TokenFilterFactory tokenFilterFactory;
@ -241,8 +242,8 @@ public class CategorizationAnalyzer implements Closeable {
// Need to set anonymous "name" of token_filter
tokenFilterFactory = tokenFilterFactoryFactory.get(buildDummyIndexSettings(settings), environment, "_anonymous_tokenfilter",
settings);
tokenFilterFactory = CustomAnalyzerProvider.checkAndApplySynonymFilter(tokenFilterFactory, tokenizerFactory.v1(),
tokenizerFactory.v2(), tokenFilterFactoryList, charFilterFactoryList, environment);
tokenFilterFactory = tokenFilterFactory.getChainAwareTokenFilterFactory(tokenizerFactory.v2(),
charFilterFactoryList, tokenFilterFactoryList, deferredRegistry);
}
if (tokenFilterFactory == null) {
throw new IllegalArgumentException("Failed to find or create token filter [" + tokenFilter + "]");

3
x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
View File

@ -422,9 +422,6 @@ public class MonitoringIT extends ESSingleNodeTestCase {
assertThat((String) indexStats.get("uuid"), not(isEmptyOrNullString()));
assertThat(indexStats.get("created"), notNullValue());
assertThat((String) indexStats.get("status"), not(isEmptyOrNullString()));
assertThat(indexStats.get("version"), notNullValue());
final Map<String, Object> version = (Map<String, Object>) indexStats.get("version");
assertEquals(2, version.size());
assertThat(indexStats.get("shards"), notNullValue());
final Map<String, Object> shards = (Map<String, Object>) indexStats.get("shards");
assertEquals(11, shards.size());

10
x-pack/plugin/sql/src/main/antlr/SqlBase.g4
View File

@ -163,14 +163,18 @@ expression
booleanExpression
: NOT booleanExpression #logicalNot
| EXISTS '(' query ')' #exists
| QUERY '(' queryString=string (',' options=string)* ')' #stringQuery
| MATCH '(' singleField=qualifiedName ',' queryString=string (',' options=string)* ')' #matchQuery
| MATCH '(' multiFields=string ',' queryString=string (',' options=string)* ')' #multiMatchQuery
| QUERY '(' queryString=string matchQueryOptions ')' #stringQuery
| MATCH '(' singleField=qualifiedName ',' queryString=string matchQueryOptions ')' #matchQuery
| MATCH '(' multiFields=string ',' queryString=string matchQueryOptions ')' #multiMatchQuery
| predicated #booleanDefault
| left=booleanExpression operator=AND right=booleanExpression #logicalBinary
| left=booleanExpression operator=OR right=booleanExpression #logicalBinary
;
matchQueryOptions
: (',' string)*
;
// workaround for:
// https://github.com/antlr/antlr4/issues/780
// https://github.com/antlr/antlr4/issues/781

12
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/fulltext/FullTextUtils.java
View File

@ -5,16 +5,16 @@
*/
package org.elasticsearch.xpack.sql.expression.predicate.fulltext;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.elasticsearch.common.Strings;
import org.elasticsearch.xpack.sql.expression.predicate.fulltext.FullTextPredicate.Operator;
import org.elasticsearch.xpack.sql.parser.ParsingException;
import org.elasticsearch.xpack.sql.tree.Location;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import static java.util.Collections.emptyMap;
abstract class FullTextUtils {
@ -26,7 +26,7 @@ abstract class FullTextUtils {
return emptyMap();
}
String[] list = Strings.delimitedListToStringArray(options, DELIMITER);
Map<String, String> op = new LinkedHashMap<String, String>(list.length);
Map<String, String> op = new LinkedHashMap<>(list.length);
for (String entry : list) {
String[] split = splitInTwo(entry, "=");

17
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
View File

@ -67,6 +67,7 @@ import org.elasticsearch.xpack.sql.parser.SqlBaseParser.LikePatternContext;
import org.elasticsearch.xpack.sql.parser.SqlBaseParser.LogicalBinaryContext;
import org.elasticsearch.xpack.sql.parser.SqlBaseParser.LogicalNotContext;
import org.elasticsearch.xpack.sql.parser.SqlBaseParser.MatchQueryContext;
import org.elasticsearch.xpack.sql.parser.SqlBaseParser.MatchQueryOptionsContext;
import org.elasticsearch.xpack.sql.parser.SqlBaseParser.MultiMatchQueryContext;
import org.elasticsearch.xpack.sql.parser.SqlBaseParser.NullLiteralContext;
import org.elasticsearch.xpack.sql.parser.SqlBaseParser.OrderByContext;
@ -99,6 +100,7 @@ import java.math.BigInteger;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.StringJoiner;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.sql.type.DataTypeConversion.conversionFor;
@ -324,18 +326,27 @@ abstract class ExpressionBuilder extends IdentifierBuilder {
//
@Override
public Object visitStringQuery(StringQueryContext ctx) {
return new StringQueryPredicate(source(ctx), string(ctx.queryString), string(ctx.options));
return new StringQueryPredicate(source(ctx), string(ctx.queryString), getQueryOptions(ctx.matchQueryOptions()));
}
@Override
public Object visitMatchQuery(MatchQueryContext ctx) {
return new MatchQueryPredicate(source(ctx), new UnresolvedAttribute(source(ctx.singleField),
visitQualifiedName(ctx.singleField)), string(ctx.queryString), string(ctx.options));
visitQualifiedName(ctx.singleField)), string(ctx.queryString), getQueryOptions(ctx.matchQueryOptions()));
}
@Override
public Object visitMultiMatchQuery(MultiMatchQueryContext ctx) {
return new MultiMatchQueryPredicate(source(ctx), string(ctx.multiFields), string(ctx.queryString), string(ctx.options));
return new MultiMatchQueryPredicate(source(ctx), string(ctx.multiFields), string(ctx.queryString),
getQueryOptions(ctx.matchQueryOptions()));
}
private String getQueryOptions(MatchQueryOptionsContext optionsCtx) {
StringJoiner sj = new StringJoiner(";");
for (StringContext sc: optionsCtx.string()) {
sj.add(string(sc));
}
return sj.toString();
}
@Override

12
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java
View File

@ -527,6 +527,18 @@ class SqlBaseBaseListener implements SqlBaseListener {
* <p>The default implementation does nothing.</p>
*/
@Override public void exitLogicalBinary(SqlBaseParser.LogicalBinaryContext ctx) { }
/**
* {@inheritDoc}
*
* <p>The default implementation does nothing.</p>
*/
@Override public void enterMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx) { }
/**
* {@inheritDoc}
*
* <p>The default implementation does nothing.</p>
*/
@Override public void exitMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx) { }
/**
* {@inheritDoc}
*

7
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java
View File

@ -312,6 +312,13 @@ class SqlBaseBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements SqlBa
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override public T visitLogicalBinary(SqlBaseParser.LogicalBinaryContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
*
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override public T visitMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
*

10
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseListener.java
View File

@ -489,6 +489,16 @@ interface SqlBaseListener extends ParseTreeListener {
* @param ctx the parse tree
*/
void exitLogicalBinary(SqlBaseParser.LogicalBinaryContext ctx);
/**
* Enter a parse tree produced by {@link SqlBaseParser#matchQueryOptions}.
* @param ctx the parse tree
*/
void enterMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx);
/**
* Exit a parse tree produced by {@link SqlBaseParser#matchQueryOptions}.
* @param ctx the parse tree
*/
void exitMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx);
/**
* Enter a parse tree produced by {@link SqlBaseParser#predicated}.
* @param ctx the parse tree

1743
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java
View File

File diff suppressed because it is too large Load Diff

6
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseVisitor.java
View File

@ -294,6 +294,12 @@ interface SqlBaseVisitor<T> extends ParseTreeVisitor<T> {
* @return the visitor result
*/
T visitLogicalBinary(SqlBaseParser.LogicalBinaryContext ctx);
/**
* Visit a parse tree produced by {@link SqlBaseParser#matchQueryOptions}.
* @param ctx the parse tree
* @return the visitor result
*/
T visitMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx);
/**
* Visit a parse tree produced by {@link SqlBaseParser#predicated}.
* @param ctx the parse tree

31
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java
View File

@ -5,16 +5,20 @@
*/
package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
import org.elasticsearch.bootstrap.JavaVersion;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.NamedDateTimeProcessor.NameExtractor;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.junit.Assume;
import java.io.IOException;
import java.util.TimeZone;
public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase<NamedDateTimeProcessor> {
private static final TimeZone UTC = TimeZone.getTimeZone("UTC");
public static NamedDateTimeProcessor randomNamedDateTimeProcessor() {
@ -37,8 +41,8 @@ public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase
return new NamedDateTimeProcessor(replaced, UTC);
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33621")
public void testValidDayNamesInUTC() {
assumeJava9PlusAndCompatLocaleProviderSetting();
NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.DAY_NAME, UTC);
assertEquals("Thursday", proc.process("0"));
assertEquals("Saturday", proc.process("-64164233612338"));
@ -50,8 +54,8 @@ public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase
assertEquals("Tuesday", proc.process(new DateTime(10902, 8, 22, 11, 11, DateTimeZone.UTC)));
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33621")
public void testValidDayNamesWithNonUTCTimeZone() {
assumeJava9PlusAndCompatLocaleProviderSetting();
NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.DAY_NAME, TimeZone.getTimeZone("GMT-10:00"));
assertEquals("Wednesday", proc.process("0"));
assertEquals("Friday", proc.process("-64164233612338"));
@ -64,8 +68,8 @@ public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase
assertEquals("Monday", proc.process(new DateTime(10902, 8, 22, 9, 59, DateTimeZone.UTC)));
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33621")
public void testValidMonthNamesInUTC() {
assumeJava9PlusAndCompatLocaleProviderSetting();
NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.MONTH_NAME, UTC);
assertEquals("January", proc.process("0"));
assertEquals("September", proc.process("-64164233612338"));
@ -77,8 +81,8 @@ public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase
assertEquals("August", proc.process(new DateTime(10902, 8, 22, 11, 11, DateTimeZone.UTC)));
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33621")
public void testValidMonthNamesWithNonUTCTimeZone() {
assumeJava9PlusAndCompatLocaleProviderSetting();
NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.MONTH_NAME, TimeZone.getTimeZone("GMT-3:00"));
assertEquals("December", proc.process("0"));
assertEquals("August", proc.process("-64165813612338")); // GMT: Tuesday, September 1, -0064 2:53:07.662 AM
@ -90,4 +94,23 @@ public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase
assertEquals("July", proc.process(new DateTime(10902, 8, 1, 2, 59, DateTimeZone.UTC)));
assertEquals("August", proc.process(new DateTime(10902, 8, 1, 3, 00, DateTimeZone.UTC)));
}
/*
* This method checks the existence of a jvm parameter that should exist in ES jvm.options for Java 9+. If the parameter is
* missing, the tests will be skipped. Not doing this, the tests will fail because the day and month names will be in the narrow
* format (Mon, Tue, Jan, Feb etc) instead of full format (Monday, Tuesday, January, February etc).
*
* Related infra issue: https://github.com/elastic/elasticsearch/issues/33796
*/
private void assumeJava9PlusAndCompatLocaleProviderSetting() {
// at least Java 9
if (JavaVersion.current().compareTo(JavaVersion.parse("9")) < 0) {
return;
}
String beforeJava9CompatibleLocale = System.getProperty("java.locale.providers");
// and COMPAT setting needs to be first on the list
boolean isBeforeJava9Compatible = beforeJava9CompatibleLocale != null
&& Strings.tokenizeToStringArray(beforeJava9CompatibleLocale, ",")[0].equals("COMPAT");
Assume.assumeTrue(isBeforeJava9Compatible);
}
}

41
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/fulltext/FullTextUtilsTests.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.expression.predicate.fulltext;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.sql.parser.ParsingException;
import org.elasticsearch.xpack.sql.tree.Location;
import java.util.Map;
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.is;
public class FullTextUtilsTests extends ESTestCase {
public void testColonDelimited() {
Map<String, String> options = FullTextUtils.parseSettings("k1=v1;k2=v2", new Location(1, 1));
assertThat(options.size(), is(2));
assertThat(options, hasEntry("k1", "v1"));
assertThat(options, hasEntry("k2", "v2"));
}
public void testColonDelimitedErrorString() {
ParsingException e = expectThrows(ParsingException.class,
() -> FullTextUtils.parseSettings("k1=v1;k2v2", new Location(1, 1)));
assertThat(e.getMessage(), is("line 1:3: Cannot parse entry k2v2 in options k1=v1;k2v2"));
assertThat(e.getLineNumber(), is(1));
assertThat(e.getColumnNumber(), is(3));
}
public void testColonDelimitedErrorDuplicate() {
ParsingException e = expectThrows(ParsingException.class,
() -> FullTextUtils.parseSettings("k1=v1;k1=v2", new Location(1, 1)));
assertThat(e.getMessage(), is("line 1:3: Duplicate option k1=v2 detected in options k1=v1;k1=v2"));
assertThat(e.getLineNumber(), is(1));
assertThat(e.getColumnNumber(), is(3));
}
}

44
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java
View File

@ -11,6 +11,10 @@ import org.elasticsearch.xpack.sql.expression.Order;
import org.elasticsearch.xpack.sql.expression.UnresolvedAttribute;
import org.elasticsearch.xpack.sql.expression.UnresolvedStar;
import org.elasticsearch.xpack.sql.expression.function.UnresolvedFunction;
import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MatchQueryPredicate;
import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MultiMatchQueryPredicate;
import org.elasticsearch.xpack.sql.expression.predicate.fulltext.StringQueryPredicate;
import org.elasticsearch.xpack.sql.plan.logical.Filter;
import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.sql.plan.logical.OrderBy;
import org.elasticsearch.xpack.sql.plan.logical.Project;
@ -19,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import static java.util.stream.Collectors.toList;
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.instanceOf;
@ -92,6 +97,45 @@ public class SqlParserTests extends ESTestCase {
assertEquals("baz", a.name());
}
public void testStringQuery() {
LogicalPlan plan =
parseStatement("SELECT * FROM FOO WHERE " +
"QUERY('foo', 'default_field=last_name;lenient=true', 'fuzzy_rewrite=scoring_boolean')");
StringQueryPredicate sqp = (StringQueryPredicate) ((Filter) plan.children().get(0).children().get(0)).condition();
assertEquals("foo", sqp.query());
assertEquals(3, sqp.optionMap().size());
assertThat(sqp.optionMap(), hasEntry("default_field", "last_name"));
assertThat(sqp.optionMap(), hasEntry("lenient", "true"));
assertThat(sqp.optionMap(), hasEntry("fuzzy_rewrite", "scoring_boolean"));
}
public void testMatchQuery() {
LogicalPlan plan = parseStatement("SELECT * FROM FOO WHERE " +
"MATCH(first_name, 'foo', 'operator=AND;lenient=true', 'fuzzy_rewrite=scoring_boolean')");
MatchQueryPredicate mqp = (MatchQueryPredicate) ((Filter) plan.children().get(0).children().get(0)).condition();
assertEquals("foo", mqp.query());
assertEquals("?first_name", mqp.field().toString());
assertEquals(3, mqp.optionMap().size());
assertThat(mqp.optionMap(), hasEntry("operator", "AND"));
assertThat(mqp.optionMap(), hasEntry("lenient", "true"));
assertThat(mqp.optionMap(), hasEntry("fuzzy_rewrite", "scoring_boolean"));
}
public void testMultiMatchQuery() {
LogicalPlan plan = parseStatement("SELECT * FROM FOO WHERE " +
"MATCH('first_name,last_name', 'foo', 'operator=AND;type=best_fields', 'fuzzy_rewrite=scoring_boolean')");
MultiMatchQueryPredicate mmqp = (MultiMatchQueryPredicate) ((Filter) plan.children().get(0).children().get(0)).condition();
assertEquals("foo", mmqp.query());
assertEquals("first_name,last_name", mmqp.fieldString());
assertEquals(3, mmqp.optionMap().size());
assertThat(mmqp.optionMap(), hasEntry("operator", "AND"));
assertThat(mmqp.optionMap(), hasEntry("type", "best_fields"));
assertThat(mmqp.optionMap(), hasEntry("fuzzy_rewrite", "scoring_boolean"));
}
private LogicalPlan parseStatement(String sql) {
return new SqlParser().createStatement(sql);
}

Some files were not shown because too many files have changed in this diff Show More