Merge remote-tracking branch 'origin/master' into index-lifecycle

buildSrc/src/test/java/org/elasticsearch/gradle/precommit/JarHellTaskIT.java

@@ -34,7 +34,7 @@ public class JarHellTaskIT extends GradleIntegrationTestCase {
         assertTaskFailed(result, ":jarHell");
         assertOutputContains(
             result.getOutput(),
-            "Exception in thread \"main\" java.lang.IllegalStateException: jar hell!",
+            "java.lang.IllegalStateException: jar hell!",
             "class: org.apache.logging.log4j.Logger"
         );
     }

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

@@ -28,6 +28,7 @@ import org.apache.http.entity.ByteArrayEntity;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.client.RequestConverters.EndpointBuilder;
 import org.elasticsearch.client.ml.CloseJobRequest;
+import org.elasticsearch.client.ml.DeleteCalendarRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteForecastRequest;
 import org.elasticsearch.client.ml.DeleteJobRequest;
@@ -372,4 +373,15 @@ final class MLRequestConverters {
         request.setEntity(createEntity(getCalendarsRequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
+
+    static Request deleteCalendar(DeleteCalendarRequest deleteCalendarRequest) {
+        String endpoint = new EndpointBuilder()
+                .addPathPartAsIs("_xpack")
+                .addPathPartAsIs("ml")
+                .addPathPartAsIs("calendars")
+                .addPathPart(deleteCalendarRequest.getCalendarId())
+                .build();
+        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
+        return request;
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java

@@ -22,6 +22,7 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ml.CloseJobRequest;
 import org.elasticsearch.client.ml.CloseJobResponse;
+import org.elasticsearch.client.ml.DeleteCalendarRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteForecastRequest;
 import org.elasticsearch.client.ml.DeleteJobRequest;
@@ -910,4 +911,44 @@ public final class MachineLearningClient {
                 listener,
                 Collections.emptySet());
     }
+
+    /**
+     * Deletes the given Machine Learning Calendar
+     * <p>
+     * For additional info see
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-calendar.html">
+     *     ML Delete calendar documentation</a>
+     *
+     * @param request The request to delete the calendar
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return action acknowledgement
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public AcknowledgedResponse deleteCalendar(DeleteCalendarRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+                MLRequestConverters::deleteCalendar,
+                options,
+                AcknowledgedResponse::fromXContent,
+                Collections.emptySet());
+    }
+
+    /**
+     * Deletes the given Machine Learning Job asynchronously and notifies the listener on completion
+     * <p>
+     * For additional info see
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-calendar.html">
+     *     ML Delete calendar documentation</a>
+     *
+     * @param request  The request to delete the calendar
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void deleteCalendarAsync(DeleteCalendarRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+                MLRequestConverters::deleteCalendar,
+                options,
+                AcknowledgedResponse::fromXContent,
+                listener,
+                Collections.emptySet());
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteCalendarRequest.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+
+import java.util.Objects;
+
+/**
+ * Request to delete a Machine Learning Calendar
+ */
+public class DeleteCalendarRequest extends ActionRequest {
+
+    private final String calendarId;
+
+    /**
+     * The constructor requires a single calendar id.
+     * @param calendarId The calendar to delete. Must be {@code non-null}
+     */
+    public DeleteCalendarRequest(String calendarId) {
+        this.calendarId = Objects.requireNonNull(calendarId, "[calendar_id] must not be null");
+    }
+
+    public String getCalendarId() {
+        return calendarId;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(calendarId);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        DeleteCalendarRequest other = (DeleteCalendarRequest) obj;
+        return Objects.equals(calendarId, other.calendarId);
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java

@@ -24,6 +24,7 @@ import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.client.ml.CloseJobRequest;
+import org.elasticsearch.client.ml.DeleteCalendarRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteForecastRequest;
 import org.elasticsearch.client.ml.DeleteJobRequest;
@@ -438,6 +439,13 @@ public class MLRequestConvertersTests extends ESTestCase {
         }
     }
 
+    public void testDeleteCalendar() {
+        DeleteCalendarRequest deleteCalendarRequest = new DeleteCalendarRequest(randomAlphaOfLength(10));
+        Request request = MLRequestConverters.deleteCalendar(deleteCalendarRequest);
+        assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
+        assertEquals("/_xpack/ml/calendars/" + deleteCalendarRequest.getCalendarId(), request.getEndpoint());
+    }
+
     private static Job createValidJob(String jobId) {
         AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList(
                 Detector.builder().setFunction("count").build()));

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -25,6 +25,7 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ml.CloseJobRequest;
 import org.elasticsearch.client.ml.CloseJobResponse;
+import org.elasticsearch.client.ml.DeleteCalendarRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteForecastRequest;
 import org.elasticsearch.client.ml.DeleteJobRequest;
@@ -517,6 +518,24 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
         assertEquals(calendar1, getCalendarsResponse.calendars().get(0));
     }
 
+    public void testDeleteCalendar() throws IOException {
+        Calendar calendar = CalendarTests.testInstance();
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        execute(new PutCalendarRequest(calendar), machineLearningClient::putCalendar,
+                machineLearningClient::putCalendarAsync);
+
+        AcknowledgedResponse response = execute(new DeleteCalendarRequest(calendar.getId()),
+                machineLearningClient::deleteCalendar,
+                machineLearningClient::deleteCalendarAsync);
+        assertTrue(response.isAcknowledged());
+
+        // calendar is missing
+        ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class,
+                () -> execute(new DeleteCalendarRequest(calendar.getId()), machineLearningClient::deleteCalendar,
+                        machineLearningClient::deleteCalendarAsync));
+        assertThat(exception.status().getStatus(), equalTo(404));
+    }
+
     public static String randomValidJobId() {
         CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz0123456789".toCharArray());
         return generator.ofCodePointsLength(random(), 10, 10);

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -34,6 +34,7 @@ import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.ml.CloseJobRequest;
 import org.elasticsearch.client.ml.CloseJobResponse;
+import org.elasticsearch.client.ml.DeleteCalendarRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteForecastRequest;
 import org.elasticsearch.client.ml.DeleteJobRequest;
@@ -1591,4 +1592,50 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
         }
     }
+
+    public void testDeleteCalendar() throws IOException, InterruptedException {
+        RestHighLevelClient client = highLevelClient();
+
+        Calendar calendar = new Calendar("holidays", Collections.singletonList("job_1"), "A calendar for public holidays");
+        PutCalendarRequest putCalendarRequest = new PutCalendarRequest(calendar);
+        client.machineLearning().putCalendar(putCalendarRequest, RequestOptions.DEFAULT);
+
+        //tag::x-pack-ml-delete-calendar-request
+        DeleteCalendarRequest request = new DeleteCalendarRequest("holidays"); // <1>
+        //end::x-pack-ml-delete-calendar-request
+
+        //tag::x-pack-ml-delete-calendar-execute
+        AcknowledgedResponse response = client.machineLearning().deleteCalendar(request, RequestOptions.DEFAULT);
+        //end::x-pack-ml-delete-calendar-execute
+
+        //tag::x-pack-ml-delete-calendar-response
+        boolean isAcknowledged = response.isAcknowledged(); // <1>
+        //end::x-pack-ml-delete-calendar-response
+
+        assertTrue(isAcknowledged);
+
+        // tag::x-pack-ml-delete-calendar-listener
+        ActionListener<AcknowledgedResponse> listener = new ActionListener<AcknowledgedResponse>() {
+            @Override
+            public void onResponse(AcknowledgedResponse response) {
+                // <1>
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                // <2>
+            }
+        };
+        // end::x-pack-ml-delete-calendar-listener
+
+        // Replace the empty listener by a blocking listener in test
+        final CountDownLatch latch = new CountDownLatch(1);
+        listener = new LatchedActionListener<>(listener, latch);
+
+        // tag::x-pack-ml-delete-calendar-execute-async
+        client.machineLearning().deleteCalendarAsync(request, RequestOptions.DEFAULT, listener); // <1>
+        // end::x-pack-ml-delete-calendar-execute-async
+
+        assertTrue(latch.await(30L, TimeUnit.SECONDS));
+    }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteCalendarRequestTests.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+
+public class DeleteCalendarRequestTests extends ESTestCase {
+
+    public void testWithNullId() {
+        NullPointerException ex = expectThrows(NullPointerException.class, () -> new DeleteCalendarRequest(null));
+        assertEquals("[calendar_id] must not be null", ex.getMessage());
+    }
+
+    public void testEqualsAndHash() {
+        String id1 = randomAlphaOfLength(8);
+        String id2 = id1 + "_a";
+        assertThat(new DeleteCalendarRequest(id1), equalTo(new DeleteCalendarRequest(id1)));
+        assertThat(new DeleteCalendarRequest(id1).hashCode(), equalTo(new DeleteCalendarRequest(id1).hashCode()));
+        assertThat(new DeleteCalendarRequest(id1), not(equalTo(new DeleteCalendarRequest(id2))));
+        assertThat(new DeleteCalendarRequest(id1).hashCode(), not(equalTo(new DeleteCalendarRequest(id2).hashCode())));
+    }
+}

distribution/src/bin/elasticsearch-shard

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ES_MAIN_CLASS=org.elasticsearch.index.shard.ShardToolCli \
+  "`dirname "$0"`"/elasticsearch-cli \
+  "$@"

distribution/src/bin/elasticsearch-shard.bat

@@ -0,0 +1,12 @@
+@echo off
+
+setlocal enabledelayedexpansion
+setlocal enableextensions
+
+set ES_MAIN_CLASS=org.elasticsearch.index.shard.ShardToolCli
+call "%~dp0elasticsearch-cli.bat" ^
+  %%* ^
+  || exit /b 1
+
+endlocal
+endlocal

docs/java-rest/high-level/ml/delete-calendar.asciidoc

@@ -0,0 +1,59 @@
+[[java-rest-high-x-pack-ml-delete-calendar]]
+=== Delete Calendar API
+Delete a {ml} calendar.
+The API accepts a `DeleteCalendarRequest` and responds
+with a `AcknowledgedResponse` object.
+
+[[java-rest-high-x-pack-ml-delete-calendar-request]]
+==== Delete Calendar Request
+
+A `DeleteCalendar` object requires a non-null `calendarId`.
+
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-calendar-request]
+---------------------------------------------------
+<1> Constructing a new request referencing an existing Calendar
+
+[[java-rest-high-x-pack-ml-delete-calendar-response]]
+==== Delete Calendar Response
+
+The returned `AcknowledgedResponse` object indicates the acknowledgement of the request:
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-calendar-response]
+---------------------------------------------------
+<1> `isAcknowledged` was the deletion request acknowledged or not
+
+[[java-rest-high-x-pack-ml-delete-calendar-execution]]
+==== Execution
+The request can be executed through the `MachineLearningClient` contained
+in the `RestHighLevelClient` object, accessed via the `machineLearningClient()` method.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-calendar-execute]
+--------------------------------------------------
+
+[[java-rest-high-x-pack-ml-delete-calendar-async]]
+==== Delete Calendar Asynchronously
+
+This request can also be made asynchronously.
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-calendar-execute-async]
+---------------------------------------------------
+<1> The `DeleteCalendarRequest` to execute and the `ActionListener` to alert on completion or error.
+
+The deletion request returns immediately. Once the request is completed, the `ActionListener` is
+called back using the `onResponse` or `onFailure`. The latter indicates some failure occurred when
+making the request.
+
+A typical listener for a `DeleteCalendarRequest` could be defined as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-delete-calendar-listener]
+---------------------------------------------------
+<1> The action to be taken when it is completed
+<2> What to do when a failure occurs

docs/java-rest/high-level/supported-apis.asciidoc

@@ -233,6 +233,7 @@ The Java High Level REST Client supports the following Machine Learning APIs:
 * <<java-rest-high-x-pack-ml-get-categories>>
 * <<java-rest-high-x-pack-ml-get-calendars>>
 * <<java-rest-high-x-pack-ml-put-calendar>>
+* <<java-rest-high-x-pack-ml-delete-calendar>>
 
 include::ml/put-job.asciidoc[]
 include::ml/get-job.asciidoc[]
@@ -255,6 +256,7 @@ include::ml/get-influencers.asciidoc[]
 include::ml/get-categories.asciidoc[]
 include::ml/get-calendars.asciidoc[]
 include::ml/put-calendar.asciidoc[]
+include::ml/delete-calendar.asciidoc[]
 
 == Migration APIs
 

docs/painless/painless-getting-started.asciidoc

@@ -198,13 +198,11 @@ POST hockey/player/1/_update
 ==== Dates
 
 Date fields are exposed as
-`ReadableDateTime` or
-so they support methods like
-`getYear`,
-and `getDayOfWeek`.
-To get milliseconds since epoch call
-`getMillis`.
-For example, the following returns every hockey player's birth year:
+`ReadableDateTime`, so they support methods like `getYear`, `getDayOfWeek` 
+or e.g. getting milliseconds since epoch with `getMillis`. To use these
+in a script, leave out the `get` prefix and continue with lowercasing the
+rest of the method name. For example, the following returns every hockey 
+player's birth year:
 
 [source,js]
 ----------------------------------------------------------------

docs/reference/analysis/tokenfilters/multiplexer-tokenfilter.asciidoc

@@ -113,4 +113,12 @@ And it'd respond:
 // TESTRESPONSE
 
 <1> The stemmer has also emitted a token `home` at position 1, but because it is a
-duplicate of this token it has been removed from the token stream
+duplicate of this token it has been removed from the token stream
+
+NOTE: The synonym and synonym_graph filters use their preceding analysis chain to
+parse and analyse their synonym lists, and ignore any token filters in the chain
+that produce multiple tokens at the same position.  This means that any filters
+within the multiplexer will be ignored for the purpose of synonyms.  If you want to
+use filters contained within the multiplexer for parsing synonyms (for example, to
+apply stemming to the synonym lists), then you should append the synonym filter
+to the relevant multiplexer filter list.

docs/reference/commands/index.asciidoc

@@ -12,6 +12,7 @@ tasks from the command line:
 * <<migrate-tool>>
 * <<saml-metadata>>
 * <<setup-passwords>>
+* <<shard-tool>>
 * <<syskeygen>>
 * <<users-command>>
 
@@ -22,5 +23,6 @@ include::certutil.asciidoc[]
 include::migrate-tool.asciidoc[]
 include::saml-metadata.asciidoc[]
 include::setup-passwords.asciidoc[]
+include::shard-tool.asciidoc[]
 include::syskeygen.asciidoc[]
 include::users-command.asciidoc[]

docs/reference/commands/shard-tool.asciidoc

@@ -0,0 +1,107 @@
+[[shard-tool]]
+== elasticsearch-shard
+
+In some cases the Lucene index or translog of a shard copy can become
+corrupted. The `elasticsearch-shard` command enables you to remove corrupted
+parts of the shard if a good copy of the shard cannot be recovered
+automatically or restored from backup.
+
+[WARNING]
+You will lose the corrupted data when you run `elasticsearch-shard`.  This tool
+should only be used as a last resort if there is no way to recover from another
+copy of the shard or restore a snapshot.
+
+When Elasticsearch detects that a shard's data is corrupted, it fails that
+shard copy and refuses to use it. Under normal conditions, the shard is
+automatically recovered from another copy. If no good copy of the shard is
+available and you cannot restore from backup, you can use `elasticsearch-shard`
+to remove the corrupted data and restore access to any remaining data in
+unaffected segments.
+
+[WARNING]
+Stop Elasticsearch before running `elasticsearch-shard`.
+
+To remove corrupted shard data use the `remove-corrupted-data` subcommand.
+
+There are two ways to specify the path:
+
+* Specify the index name and shard name with the `--index` and `--shard-id`
+  options.
+* Use the `--dir` option to specify the full path to the corrupted index or
+  translog files.
+
+[float]
+=== Removing corrupted data
+
+`elasticsearch-shard` analyses the shard copy and provides an overview of the
+corruption found. To proceed you must then confirm that you want to remove the
+corrupted data.
+
+[WARNING]
+Back up your data before running `elasticsearch-shard`. This is a destructive
+operation that removes corrupted data from the shard.
+
+[source,txt]
+--------------------------------------------------
+$ bin/elasticsearch-shard remove-corrupted-data --index twitter --shard-id 0
+
+
+    WARNING: Elasticsearch MUST be stopped before running this tool.
+
+  Please make a complete backup of your index before using this tool.
+
+
+Opening Lucene index at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+
+ >> Lucene index is corrupted at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+
+Opening translog at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
+
+
+ >> Translog is clean at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
+
+
+  Corrupted Lucene index segments found - 32 documents will be lost.
+
+            WARNING:              YOU WILL LOSE DATA.
+
+Continue and remove docs from the index ? Y
+
+WARNING: 1 broken segments (containing 32 documents) detected
+Took 0.056 sec total.
+Writing...
+OK
+Wrote new segments file "segments_c"
+Marking index with the new history uuid : 0pIBd9VTSOeMfzYT6p0AsA
+Changing allocation id V8QXk-QXSZinZMT-NvEq4w to tjm9Ve6uTBewVFAlfUMWjA
+
+You should run the following command to allocate this shard:
+
+POST /_cluster/reroute
+{
+  "commands" : [
+    {
+      "allocate_stale_primary" : {
+        "index" : "index42",
+        "shard" : 0,
+        "node" : "II47uXW2QvqzHBnMcl2o_Q",
+        "accept_data_loss" : false
+      }
+    }
+  ]
+}
+
+You must accept the possibility of data loss by changing parameter `accept_data_loss` to `true`.
+
+Deleted corrupt marker corrupted_FzTSBSuxT7i3Tls_TgwEag from /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+
+--------------------------------------------------
+
+When you use `elasticsearch-shard` to drop the corrupted data, the shard's
+allocation ID changes. After restarting the node, you must use the
+<<cluster-reroute,cluster reroute API>> to tell Elasticsearch to use the new
+ID. The `elasticsearch-shard` command shows the request that
+you need to submit.
+
+You can also use the `-h` option to get a list of all options and parameters
+that the `elasticsearch-shard` tool supports.

docs/reference/index-modules/translog.asciidoc

@@ -92,6 +92,10 @@ The maximum duration for which translog files will be kept. Defaults to `12h`.
 [[corrupt-translog-truncation]]
 === What to do if the translog becomes corrupted?
 
+[WARNING]
+This tool is deprecated and will be completely removed in 7.0.
+Use the <<shard-tool,elasticsearch-shard tool>> instead of this one.
+
 In some cases (a bad drive, user error) the translog on a shard copy can become
 corrupted. When this corruption is detected by Elasticsearch due to mismatching
 checksums, Elasticsearch will fail that shard copy and refuse to use that copy

docs/reference/modules/snapshots.asciidoc

@@ -1,22 +1,47 @@
 [[modules-snapshots]]
 == Snapshot And Restore
 
-You can store snapshots of individual indices or an entire cluster in
-a remote repository like a shared file system, S3, or HDFS. These snapshots
-are great for backups because they can be restored relatively quickly. However,
-snapshots can only be restored to versions of Elasticsearch that can read the
-indices:
+A snapshot is a backup taken from a running Elasticsearch cluster. You can take
+a snapshot of individual indices or of the entire cluster and store it in a
+repository on a shared filesystem, and there are plugins that support remote
+repositories on S3, HDFS, Azure, Google Cloud Storage and more.
+
+Snapshots are taken incrementally. This means that when creating a snapshot of
+an index Elasticsearch will avoid copying any data that is already stored in
+the repository as part of an earlier snapshot of the same index. Therefore it
+can be efficient to take snapshots of your cluster quite frequently.
+
+Snapshots can be restored into a running cluster via the restore API. When
+restoring an index it is possible to alter the name of the restored index as
+well as some of its settings, allowing a great deal of flexibility in how the
+snapshot and restore functionality can be used.
+
+WARNING: It is not possible to back up an Elasticsearch cluster simply by
+taking a copy of the data directories of all of its nodes. Elasticsearch may be
+making changes to the contents of its data directories while it is running, and
+this means that copying its data directories cannot be expected to capture a
+consistent picture of their contents. Attempts to restore a cluster from such a
+backup may fail, reporting corruption and/or missing files, or may appear to
+have succeeded having silently lost some of its data. The only reliable way to
+back up a cluster is by using the snapshot and restore functionality.
+
+[float]
+=== Version compatibility
+
+A snapshot contains a copy of the on-disk data structures that make up an
+index. This means that snapshots can only be restored to versions of
+Elasticsearch that can read the indices:
 
 * A snapshot of an index created in 5.x can be restored to 6.x.
 * A snapshot of an index created in 2.x can be restored to 5.x.
 * A snapshot of an index created in 1.x can be restored to 2.x.
 
-Conversely, snapshots of indices created in 1.x **cannot** be restored to
-5.x or 6.x, and snapshots of indices created in 2.x **cannot** be restored
-to 6.x.
+Conversely, snapshots of indices created in 1.x **cannot** be restored to 5.x
+or 6.x, and snapshots of indices created in 2.x **cannot** be restored to 6.x.
 
-Snapshots are incremental and can contain indices created in various
-versions of Elasticsearch. If any indices in a snapshot were created in an
+Each snapshot can contain indices created in various versions of Elasticsearch,
+and when restoring a snapshot it must be possible to restore all of the indices
+into the target cluster. If any indices in a snapshot were created in an
 incompatible version, you will not be able restore the snapshot.
 
 IMPORTANT: When backing up your data prior to an upgrade, keep in mind that you
@@ -28,8 +53,8 @@ that is incompatible with the version of the cluster you are currently running,
 you can restore it on the latest compatible version and use
 <<reindex-from-remote,reindex-from-remote>> to rebuild the index on the current
 version. Reindexing from remote is only possible if the original index has
-source enabled. Retrieving and reindexing the data can take significantly longer
-than simply restoring a snapshot. If you have a large amount of data, we
+source enabled. Retrieving and reindexing the data can take significantly
+longer than simply restoring a snapshot. If you have a large amount of data, we
 recommend testing the reindex from remote process with a subset of your data to
 understand the time requirements before proceeding.
 

libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java

@@ -85,12 +85,17 @@ public abstract class Terminal {
 
     /** Prints message to the terminal at {@code verbosity} level, without a newline. */
     public final void print(Verbosity verbosity, String msg) {
-        if (this.verbosity.ordinal() >= verbosity.ordinal()) {
+        if (isPrintable(verbosity)) {
             getWriter().print(msg);
             getWriter().flush();
         }
     }
 
+    /** Checks if is enough {@code verbosity} level to be printed */
+    public final boolean isPrintable(Verbosity verbosity) {
+        return this.verbosity.ordinal() >= verbosity.ordinal();
+    }
+
     /**
      * Prompt for a yes or no answer from the user. This method will loop until 'y' or 'n'
      * (or the default empty value) is entered.

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java

@@ -29,33 +29,20 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
-import org.elasticsearch.index.analysis.ReferringFilterFactory;
+import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.function.Function;
 
-public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory implements ReferringFilterFactory {
+public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory {
 
-    private List<TokenFilterFactory> filters;
     private List<String> filterNames;
     private final boolean preserveOriginal;
 
-    private static final TokenFilterFactory IDENTITY_FACTORY = new TokenFilterFactory() {
-        @Override
-        public String name() {
-            return "identity";
-        }
-
-        @Override
-        public TokenStream create(TokenStream tokenStream) {
-            return tokenStream;
-        }
-    };
-
     public MultiplexerTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) throws IOException {
         super(indexSettings, name, settings);
         this.filterNames = settings.getAsList("filters");
@@ -64,31 +51,56 @@ public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory im
 
     @Override
     public TokenStream create(TokenStream tokenStream) {
-        List<Function<TokenStream, TokenStream>> functions = new ArrayList<>();
-        for (TokenFilterFactory tff : filters) {
-            functions.add(tff::create);
-        }
-        return new RemoveDuplicatesTokenFilter(new MultiplexTokenFilter(tokenStream, functions));
+        throw new UnsupportedOperationException("TokenFilterFactory.getChainAwareTokenFilterFactory() must be called first");
     }
 
     @Override
-    public void setReferences(Map<String, TokenFilterFactory> factories) {
-        filters = new ArrayList<>();
+    public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
+                                                              List<TokenFilterFactory> previousTokenFilters,
+                                                              Function<String, TokenFilterFactory> allFilters) {
+        List<TokenFilterFactory> filters = new ArrayList<>();
         if (preserveOriginal) {
-            filters.add(IDENTITY_FACTORY);
+            filters.add(IDENTITY_FILTER);
         }
         for (String filter : filterNames) {
             String[] parts = Strings.tokenizeToStringArray(filter, ",");
             if (parts.length == 1) {
-                filters.add(resolveFilterFactory(factories, parts[0]));
+                TokenFilterFactory factory = resolveFilterFactory(allFilters, parts[0]);
+                factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, previousTokenFilters, allFilters);
+                filters.add(factory);
             } else {
+                List<TokenFilterFactory> existingChain = new ArrayList<>(previousTokenFilters);
                 List<TokenFilterFactory> chain = new ArrayList<>();
                 for (String subfilter : parts) {
-                    chain.add(resolveFilterFactory(factories, subfilter));
+                    TokenFilterFactory factory = resolveFilterFactory(allFilters, subfilter);
+                    factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, existingChain, allFilters);
+                    chain.add(factory);
+                    existingChain.add(factory);
                 }
                 filters.add(chainFilters(filter, chain));
             }
         }
+
+        return new TokenFilterFactory() {
+            @Override
+            public String name() {
+                return MultiplexerTokenFilterFactory.this.name();
+            }
+
+            @Override
+            public TokenStream create(TokenStream tokenStream) {
+                List<Function<TokenStream, TokenStream>> functions = new ArrayList<>();
+                for (TokenFilterFactory tff : filters) {
+                    functions.add(tff::create);
+                }
+                return new RemoveDuplicatesTokenFilter(new MultiplexTokenFilter(tokenStream, functions));
+            }
+
+            @Override
+            public TokenFilterFactory getSynonymFilter() {
+                return IDENTITY_FILTER;
+            }
+        };
     }
 
     private TokenFilterFactory chainFilters(String name, List<TokenFilterFactory> filters) {
@@ -108,11 +120,12 @@ public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory im
         };
     }
 
-    private TokenFilterFactory resolveFilterFactory(Map<String, TokenFilterFactory> factories, String name) {
-        if (factories.containsKey(name) == false) {
+    private TokenFilterFactory resolveFilterFactory(Function<String, TokenFilterFactory> factories, String name) {
+        TokenFilterFactory factory = factories.apply(name);
+        if (factory == null) {
             throw new IllegalArgumentException("Multiplexing filter [" + name() + "] refers to undefined tokenfilter [" + name + "]");
         } else {
-            return factories.get(name);
+            return factory;
         }
     }
 

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java

@@ -24,26 +24,24 @@ import org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
-import org.elasticsearch.index.analysis.ReferringFilterFactory;
+import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.function.Function;
 
 /**
  * A factory for a conditional token filter that only applies child filters if the underlying token
  * matches an {@link AnalysisPredicateScript}
  */
-public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFactory implements ReferringFilterFactory {
+public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFactory {
 
     private final AnalysisPredicateScript.Factory factory;
-    private final List<TokenFilterFactory> filters = new ArrayList<>();
     private final List<String> filterNames;
 
     ScriptedConditionTokenFilterFactory(IndexSettings indexSettings, String name,
@@ -65,13 +63,43 @@ public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFact
 
     @Override
     public TokenStream create(TokenStream tokenStream) {
-        Function<TokenStream, TokenStream> filter = in -> {
-            for (TokenFilterFactory tff : filters) {
-                in = tff.create(in);
+        throw new UnsupportedOperationException("getChainAwareTokenFilterFactory should be called first");
+    }
+
+    @Override
+    public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
+                                                              List<TokenFilterFactory> previousTokenFilters,
+                                                              Function<String, TokenFilterFactory> allFilters) {
+        List<TokenFilterFactory> filters = new ArrayList<>();
+        List<TokenFilterFactory> existingChain = new ArrayList<>(previousTokenFilters);
+        for (String filter : filterNames) {
+            TokenFilterFactory tff = allFilters.apply(filter);
+            if (tff == null) {
+                throw new IllegalArgumentException("ScriptedConditionTokenFilter [" + name() +
+                    "] refers to undefined token filter [" + filter + "]");
+            }
+            tff = tff.getChainAwareTokenFilterFactory(tokenizer, charFilters, existingChain, allFilters);
+            filters.add(tff);
+            existingChain.add(tff);
+        }
+
+        return new TokenFilterFactory() {
+            @Override
+            public String name() {
+                return ScriptedConditionTokenFilterFactory.this.name();
+            }
+
+            @Override
+            public TokenStream create(TokenStream tokenStream) {
+                Function<TokenStream, TokenStream> filter = in -> {
+                    for (TokenFilterFactory tff : filters) {
+                        in = tff.create(in);
+                    }
+                    return in;
+                };
+                return new ScriptedConditionTokenFilter(tokenStream, filter, factory.newInstance());
             }
-            return in;
         };
-        return new ScriptedConditionTokenFilter(tokenStream, filter, factory.newInstance());
     }
 
     private static class ScriptedConditionTokenFilter extends ConditionalTokenFilter {
@@ -80,29 +108,17 @@ public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFact
         private final AnalysisPredicateScript.Token token;
 
         ScriptedConditionTokenFilter(TokenStream input, Function<TokenStream, TokenStream> inputFactory,
-                                               AnalysisPredicateScript script) {
+                                     AnalysisPredicateScript script) {
             super(input, inputFactory);
             this.script = script;
             this.token = new AnalysisPredicateScript.Token(this);
         }
 
         @Override
-        protected boolean shouldFilter() throws IOException {
+        protected boolean shouldFilter() {
             token.updatePosition();
             return script.execute(token);
         }
     }
 
-    @Override
-    public void setReferences(Map<String, TokenFilterFactory> factories) {
-        for (String filter : filterNames) {
-            TokenFilterFactory tff = factories.get(filter);
-            if (tff == null) {
-                throw new IllegalArgumentException("ScriptedConditionTokenFilter [" + name() +
-                    "] refers to undefined token filter [" + filter + "]");
-            }
-            filters.add(tff);
-        }
-    }
-
 }

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.elasticsearch.Version;
@@ -117,6 +118,26 @@ public class SynonymsAnalysisTests extends ESTestCase {
         }
     }
 
+    public void testSynonymsWithMultiplexer() throws IOException {
+        Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put("path.home", createTempDir().toString())
+            .put("index.analysis.filter.synonyms.type", "synonym")
+            .putList("index.analysis.filter.synonyms.synonyms", "programmer, developer")
+            .put("index.analysis.filter.my_english.type", "stemmer")
+            .put("index.analysis.filter.my_english.language", "porter2")
+            .put("index.analysis.filter.stem_repeat.type", "multiplexer")
+            .putList("index.analysis.filter.stem_repeat.filters", "my_english, synonyms")
+            .put("index.analysis.analyzer.synonymAnalyzer.tokenizer", "standard")
+            .putList("index.analysis.analyzer.synonymAnalyzer.filter", "lowercase", "stem_repeat")
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+        indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
+
+        BaseTokenStreamTestCase.assertAnalyzesTo(indexAnalyzers.get("synonymAnalyzer"), "Some developers are odd",
+            new String[]{ "some", "developers", "develop", "programm", "are", "odd" },
+            new int[]{ 1, 1, 0, 0, 1, 1 });
+    }
 
     private void match(String analyzerName, String source, String target) throws IOException {
         Analyzer analyzer = indexAnalyzers.get(analyzerName).analyzer();

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java

@@ -325,4 +325,21 @@ public abstract class ArchiveTestCase extends PackagingTestCase {
         }
     }
 
+    public void test100RepairIndexCliPackaging() {
+        assumeThat(installation, is(notNullValue()));
+
+        final Installation.Executables bin = installation.executables();
+        final Shell sh = new Shell();
+
+        Platforms.PlatformAction action = () -> {
+            final Result result = sh.run(bin.elasticsearchShard + " help");
+            assertThat(result.stdout, containsString("A CLI tool to remove corrupted parts of unrecoverable shards"));
+        };
+
+        if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
+            Platforms.onLinux(action);
+            Platforms.onWindows(action);
+        }
+    }
+
 }

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java

@@ -186,6 +186,7 @@ public class Archives {
             "elasticsearch-env",
             "elasticsearch-keystore",
             "elasticsearch-plugin",
+            "elasticsearch-shard",
             "elasticsearch-translog"
         ).forEach(executable -> {
 

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java

@@ -100,8 +100,9 @@ public class Installation {
         public final Path elasticsearch = platformExecutable("elasticsearch");
         public final Path elasticsearchPlugin = platformExecutable("elasticsearch-plugin");
         public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore");
-        public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog");
         public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil");
+        public final Path elasticsearchShard = platformExecutable("elasticsearch-shard");
+        public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog");
 
         private Path platformExecutable(String name) {
             final String platformExecutableName = Platforms.WINDOWS

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java

@@ -187,6 +187,7 @@ public class Packages {
             "elasticsearch",
             "elasticsearch-plugin",
             "elasticsearch-keystore",
+            "elasticsearch-shard",
             "elasticsearch-translog"
         ).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
 

qa/vagrant/src/test/resources/packaging/utils/packages.bash

@@ -95,6 +95,7 @@ verify_package_installation() {
     assert_file "$ESHOME/bin" d root root 755
     assert_file "$ESHOME/bin/elasticsearch" f root root 755
     assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755
+    assert_file "$ESHOME/bin/elasticsearch-shard" f root root 755
     assert_file "$ESHOME/bin/elasticsearch-translog" f root root 755
     assert_file "$ESHOME/lib" d root root 755
     assert_file "$ESCONFIG" d root elasticsearch 2750

qa/vagrant/src/test/resources/packaging/utils/tar.bash

@@ -94,6 +94,7 @@ verify_archive_installation() {
     assert_file "$ESHOME/bin/elasticsearch-env" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-keystore" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755
+    assert_file "$ESHOME/bin/elasticsearch-shard" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-translog" f elasticsearch elasticsearch 755
     assert_file "$ESCONFIG" d elasticsearch elasticsearch 755
     assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660

server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java

@@ -48,11 +48,9 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.CustomAnalyzer;
-import org.elasticsearch.index.analysis.CustomAnalyzerProvider;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.analysis.ReferringFilterFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -66,6 +64,7 @@ import org.elasticsearch.transport.TransportService;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -73,6 +72,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.function.Function;
 
 /**
  * Transport action used to execute analyze requests
@@ -571,11 +571,48 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
         return charFilterFactoryList;
     }
 
+    public static class DeferredTokenFilterRegistry implements Function<String, TokenFilterFactory> {
+
+        private final AnalysisRegistry analysisRegistry;
+        private final IndexSettings indexSettings;
+        Map<String, TokenFilterFactory> prebuiltFilters;
+
+        public DeferredTokenFilterRegistry(AnalysisRegistry analysisRegistry, IndexSettings indexSettings) {
+            this.analysisRegistry = analysisRegistry;
+            if (indexSettings == null) {
+                // Settings are null when _analyze is called with no index name, so
+                // we create dummy settings which will make prebuilt analysis components
+                // available
+                Settings settings = Settings.builder()
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+                    .build();
+                IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
+                indexSettings = new IndexSettings(metaData, Settings.EMPTY);
+            }
+            this.indexSettings = indexSettings;
+        }
+
+        @Override
+        public TokenFilterFactory apply(String s) {
+            if (prebuiltFilters == null) {
+                try {
+                    prebuiltFilters = analysisRegistry.buildTokenFilterFactories(indexSettings);
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+            return prebuiltFilters.get(s);
+        }
+    }
+
     private static List<TokenFilterFactory> parseTokenFilterFactories(AnalyzeRequest request, IndexSettings indexSettings, AnalysisRegistry analysisRegistry,
                                                                 Environment environment, Tuple<String, TokenizerFactory> tokenizerFactory,
                                                                 List<CharFilterFactory> charFilterFactoryList, boolean normalizer) throws IOException {
         List<TokenFilterFactory> tokenFilterFactoryList = new ArrayList<>();
-        List<ReferringFilterFactory> referringFilters = new ArrayList<>();
+        DeferredTokenFilterRegistry deferredRegistry = new DeferredTokenFilterRegistry(analysisRegistry, indexSettings);
         if (request.tokenFilters() != null && request.tokenFilters().size() > 0) {
             List<AnalyzeRequest.NameOrDefinition> tokenFilters = request.tokenFilters();
             for (AnalyzeRequest.NameOrDefinition tokenFilter : tokenFilters) {
@@ -594,11 +631,8 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
                     }
                     // Need to set anonymous "name" of tokenfilter
                     tokenFilterFactory = tokenFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenfilter", settings);
-                    tokenFilterFactory = CustomAnalyzerProvider.checkAndApplySynonymFilter(tokenFilterFactory, tokenizerFactory.v1(), tokenizerFactory.v2(), tokenFilterFactoryList,
-                        charFilterFactoryList, environment);
-                    if (tokenFilterFactory instanceof ReferringFilterFactory) {
-                        referringFilters.add((ReferringFilterFactory)tokenFilterFactory);
-                    }
+                    tokenFilterFactory = tokenFilterFactory.getChainAwareTokenFilterFactory(tokenizerFactory.v2(), charFilterFactoryList,
+                        tokenFilterFactoryList, deferredRegistry);
 
                 } else {
                     AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory;
@@ -616,8 +650,8 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
                         Settings settings = AnalysisRegistry.getSettingsFromIndexSettings(indexSettings,
                             AnalysisRegistry.INDEX_ANALYSIS_FILTER + "." + tokenFilter.name);
                         tokenFilterFactory = tokenFilterFactoryFactory.get(indexSettings, environment, tokenFilter.name, settings);
-                        tokenFilterFactory = CustomAnalyzerProvider.checkAndApplySynonymFilter(tokenFilterFactory, tokenizerFactory.v1(), tokenizerFactory.v2(), tokenFilterFactoryList,
-                            charFilterFactoryList, environment);
+                        tokenFilterFactory = tokenFilterFactory.getChainAwareTokenFilterFactory(tokenizerFactory.v2(), charFilterFactoryList,
+                            tokenFilterFactoryList, deferredRegistry);
                     }
                 }
                 if (tokenFilterFactory == null) {
@@ -633,26 +667,6 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
                 tokenFilterFactoryList.add(tokenFilterFactory);
             }
         }
-        if (referringFilters.isEmpty() == false) {
-            // The request included at least one custom referring tokenfilter that has not already been built by the
-            // analysis registry, so we need to set its references.  Note that this will only apply pre-built
-            // tokenfilters
-            if (indexSettings == null) {
-                Settings settings = Settings.builder()
-                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                    .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
-                    .build();
-                IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
-                indexSettings = new IndexSettings(metaData, Settings.EMPTY);
-            }
-            Map<String, TokenFilterFactory> prebuiltFilters = analysisRegistry.buildTokenFilterFactories(indexSettings);
-            for (ReferringFilterFactory rff : referringFilters) {
-                rff.setReferences(prebuiltFilters);
-            }
-
-        }
         return tokenFilterFactoryList;
     }
 

server/src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -30,6 +30,7 @@ import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FilterCodecReader;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.FilterLeafReader;
 import org.apache.lucene.index.IndexCommit;
@@ -726,6 +727,9 @@ public class Lucene {
         } else if (reader instanceof FilterLeafReader) {
             final FilterLeafReader fReader = (FilterLeafReader) reader;
             return segmentReader(FilterLeafReader.unwrap(fReader));
+        } else if (reader instanceof FilterCodecReader) {
+            final FilterCodecReader fReader = (FilterCodecReader) reader;
+            return segmentReader(FilterCodecReader.unwrap(fReader));
         }
         // hard fail - we can't get a SegmentReader
         throw new IllegalStateException("Can not extract segment reader from given index reader [" + reader + "]");

server/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java

@@ -70,7 +70,7 @@ public class RandomScoreFunction extends ScoreFunction {
             public double score(int docId, float subQueryScore) throws IOException {
                 int hash;
                 if (values == null) {
-                    hash = BitMixer.mix(docId, saltedSeed);
+                    hash = BitMixer.mix(ctx.docBase + docId, saltedSeed);
                 } else if (values.advanceExact(docId)) {
                     hash = StringHelper.murmurhash3_x86_32(values.nextValue(), saltedSeed);
                 } else {

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -30,6 +30,8 @@ import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.NativeFSLockFactory;
 import org.apache.lucene.store.SimpleFSDirectory;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -75,6 +77,7 @@ import java.util.Set;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
 import static java.util.Collections.unmodifiableSet;
@@ -171,6 +174,63 @@ public final class NodeEnvironment  implements Closeable {
     public static final String INDICES_FOLDER = "indices";
     public static final String NODE_LOCK_FILENAME = "node.lock";
 
+    public static class NodeLock implements Releasable {
+
+        private final int nodeId;
+        private final Lock[] locks;
+        private final NodePath[] nodePaths;
+
+        /**
+         * Tries to acquire a node lock for a node id, throws {@code IOException} if it is unable to acquire it
+         * @param pathFunction function to check node path before attempt of acquiring a node lock
+         */
+        public NodeLock(final int nodeId, final Logger logger,
+                        final Environment environment,
+                        final CheckedFunction<Path, Boolean, IOException> pathFunction) throws IOException {
+            this.nodeId = nodeId;
+            nodePaths = new NodePath[environment.dataWithClusterFiles().length];
+            locks = new Lock[nodePaths.length];
+            try {
+                final Path[] dataPaths = environment.dataFiles();
+                for (int dirIndex = 0; dirIndex < dataPaths.length; dirIndex++) {
+                    Path dataDir = dataPaths[dirIndex];
+                    Path dir = resolveNodePath(dataDir, nodeId);
+                    if (pathFunction.apply(dir) == false) {
+                        continue;
+                    }
+                    try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
+                        logger.trace("obtaining node lock on {} ...", dir.toAbsolutePath());
+                        locks[dirIndex] = luceneDir.obtainLock(NODE_LOCK_FILENAME);
+                        nodePaths[dirIndex] = new NodePath(dir);
+                    } catch (IOException e) {
+                        logger.trace(() -> new ParameterizedMessage(
+                            "failed to obtain node lock on {}", dir.toAbsolutePath()), e);
+                        // release all the ones that were obtained up until now
+                        throw (e instanceof LockObtainFailedException ? e
+                            : new IOException("failed to obtain lock on " + dir.toAbsolutePath(), e));
+                    }
+                }
+            } catch (IOException e) {
+                close();
+                throw e;
+            }
+        }
+
+        public NodePath[] getNodePaths() {
+            return nodePaths;
+        }
+
+        @Override
+        public void close() {
+            for (int i = 0; i < locks.length; i++) {
+                if (locks[i] != null) {
+                    IOUtils.closeWhileHandlingException(locks[i]);
+                }
+                locks[i] = null;
+            }
+        }
+    }
+
     /**
      * Setup the environment.
      * @param settings settings from elasticsearch.yml
@@ -188,51 +248,39 @@ public final class NodeEnvironment  implements Closeable {
             nodeIdConsumer.accept(nodeMetaData.nodeId());
             return;
         }
-        final NodePath[] nodePaths = new NodePath[environment.dataWithClusterFiles().length];
-        final Lock[] locks = new Lock[nodePaths.length];
         boolean success = false;
+        NodeLock nodeLock = null;
 
         try {
             sharedDataPath = environment.sharedDataFile();
-            int nodeLockId = -1;
             IOException lastException = null;
             int maxLocalStorageNodes = MAX_LOCAL_STORAGE_NODES_SETTING.get(settings);
+
+            final AtomicReference<IOException> onCreateDirectoriesException = new AtomicReference<>();
             for (int possibleLockId = 0; possibleLockId < maxLocalStorageNodes; possibleLockId++) {
-                for (int dirIndex = 0; dirIndex < environment.dataFiles().length; dirIndex++) {
-                    Path dataDir = environment.dataFiles()[dirIndex];
-                    Path dir = resolveNodePath(dataDir, possibleLockId);
-                    Files.createDirectories(dir);
-
-                    try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
-                        logger.trace("obtaining node lock on {} ...", dir.toAbsolutePath());
-                        try {
-                            locks[dirIndex] = luceneDir.obtainLock(NODE_LOCK_FILENAME);
-                            nodePaths[dirIndex] = new NodePath(dir);
-                            nodeLockId = possibleLockId;
-                        } catch (LockObtainFailedException ex) {
-                            logger.trace(
-                                    new ParameterizedMessage("failed to obtain node lock on {}", dir.toAbsolutePath()), ex);
-                            // release all the ones that were obtained up until now
-                            releaseAndNullLocks(locks);
-                            break;
-                        }
-
-                    } catch (IOException e) {
-                        logger.trace(() -> new ParameterizedMessage(
-                            "failed to obtain node lock on {}", dir.toAbsolutePath()), e);
-                        lastException = new IOException("failed to obtain lock on " + dir.toAbsolutePath(), e);
-                        // release all the ones that were obtained up until now
-                        releaseAndNullLocks(locks);
-                        break;
-                    }
-                }
-                if (locks[0] != null) {
-                    // we found a lock, break
+                try {
+                    nodeLock = new NodeLock(possibleLockId, logger, environment,
+                        dir -> {
+                            try {
+                                Files.createDirectories(dir);
+                            } catch (IOException e) {
+                                onCreateDirectoriesException.set(e);
+                                throw e;
+                            }
+                            return true;
+                        });
                     break;
+                } catch (LockObtainFailedException e) {
+                    // ignore any LockObtainFailedException
+                } catch (IOException e) {
+                    if (onCreateDirectoriesException.get() != null) {
+                        throw onCreateDirectoriesException.get();
+                    }
+                    lastException = e;
                 }
             }
 
-            if (locks[0] == null) {
+            if (nodeLock == null) {
                 final String message = String.format(
                     Locale.ROOT,
                     "failed to obtain node locks, tried [%s] with lock id%s;" +
@@ -243,13 +291,12 @@ public final class NodeEnvironment  implements Closeable {
                     maxLocalStorageNodes);
                 throw new IllegalStateException(message, lastException);
             }
+            this.locks = nodeLock.locks;
+            this.nodePaths = nodeLock.nodePaths;
+            this.nodeLockId = nodeLock.nodeId;
             this.nodeMetaData = loadOrCreateNodeMetaData(settings, logger, nodePaths);
             nodeIdConsumer.accept(nodeMetaData.nodeId());
 
-            this.nodeLockId = nodeLockId;
-            this.locks = locks;
-            this.nodePaths = nodePaths;
-
             if (logger.isDebugEnabled()) {
                 logger.debug("using node location [{}], local_lock_id [{}]", nodePaths, nodeLockId);
             }
@@ -262,7 +309,7 @@ public final class NodeEnvironment  implements Closeable {
             success = true;
         } finally {
             if (success == false) {
-                IOUtils.closeWhileHandlingException(locks);
+                close();
             }
         }
     }
@@ -278,15 +325,6 @@ public final class NodeEnvironment  implements Closeable {
         return path.resolve(NODES_FOLDER).resolve(Integer.toString(nodeLockId));
     }
 
-    private static void releaseAndNullLocks(Lock[] locks) {
-        for (int i = 0; i < locks.length; i++) {
-            if (locks[i] != null) {
-                IOUtils.closeWhileHandlingException(locks[i]);
-            }
-            locks[i] = null;
-        }
-    }
-
     private void maybeLogPathDetails() throws IOException {
 
         // We do some I/O in here, so skip this if DEBUG/INFO are not enabled:
@@ -696,6 +734,13 @@ public final class NodeEnvironment  implements Closeable {
         return paths;
     }
 
+    /**
+     * Returns shared data path for this node environment
+     */
+    public Path sharedDataPath() {
+        return sharedDataPath;
+    }
+
     /**
      * returns the unique uuid describing this node. The uuid is persistent in the data folder of this node
      * and remains across restarts.
@@ -956,11 +1001,22 @@ public final class NodeEnvironment  implements Closeable {
      * @param indexSettings settings for the index
      */
     public Path resolveBaseCustomLocation(IndexSettings indexSettings) {
+        return resolveBaseCustomLocation(indexSettings, sharedDataPath, nodeLockId);
+    }
+
+    /**
+     * Resolve the custom path for a index's shard.
+     * Uses the {@code IndexMetaData.SETTING_DATA_PATH} setting to determine
+     * the root path for the index.
+     *
+     * @param indexSettings settings for the index
+     */
+    public static Path resolveBaseCustomLocation(IndexSettings indexSettings, Path sharedDataPath, int nodeLockId) {
         String customDataDir = indexSettings.customDataPath();
         if (customDataDir != null) {
             // This assert is because this should be caught by MetaDataCreateIndexService
             assert sharedDataPath != null;
-            return sharedDataPath.resolve(customDataDir).resolve(Integer.toString(this.nodeLockId));
+            return sharedDataPath.resolve(customDataDir).resolve(Integer.toString(nodeLockId));
         } else {
             throw new IllegalArgumentException("no custom " + IndexMetaData.SETTING_DATA_PATH + " setting available");
         }
@@ -974,7 +1030,11 @@ public final class NodeEnvironment  implements Closeable {
      * @param indexSettings settings for the index
      */
     private Path resolveIndexCustomLocation(IndexSettings indexSettings) {
-        return resolveBaseCustomLocation(indexSettings).resolve(indexSettings.getUUID());
+        return resolveIndexCustomLocation(indexSettings, sharedDataPath, nodeLockId);
+    }
+
+    private static Path resolveIndexCustomLocation(IndexSettings indexSettings, Path sharedDataPath, int nodeLockId) {
+        return resolveBaseCustomLocation(indexSettings, sharedDataPath, nodeLockId).resolve(indexSettings.getUUID());
     }
 
     /**
@@ -986,7 +1046,11 @@ public final class NodeEnvironment  implements Closeable {
      * @param shardId shard to resolve the path to
      */
     public Path resolveCustomLocation(IndexSettings indexSettings, final ShardId shardId) {
-        return resolveIndexCustomLocation(indexSettings).resolve(Integer.toString(shardId.id()));
+        return resolveCustomLocation(indexSettings, shardId, sharedDataPath, nodeLockId);
+    }
+
+    public static Path resolveCustomLocation(IndexSettings indexSettings, final ShardId shardId, Path sharedDataPath, int nodeLockId) {
+        return resolveIndexCustomLocation(indexSettings, sharedDataPath, nodeLockId).resolve(Integer.toString(shardId.id()));
     }
 
     /**

server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java

@@ -167,17 +167,7 @@ public final class AnalysisRegistry implements Closeable {
         tokenFilters.put("synonym", requiresAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings)));
         tokenFilters.put("synonym_graph", requiresAnalysisSettings((is, env, name, settings) -> new SynonymGraphTokenFilterFactory(is, env, this, name, settings)));
 
-        Map<String, TokenFilterFactory> mappings
-            = buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.preConfiguredTokenFilters);
-
-        // ReferringTokenFilters require references to other tokenfilters, so we pass these in
-        // after all factories have been registered
-        for (TokenFilterFactory tff : mappings.values()) {
-            if (tff instanceof ReferringFilterFactory) {
-                ((ReferringFilterFactory)tff).setReferences(mappings);
-            }
-        }
-        return mappings;
+        return buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.preConfiguredTokenFilters);
     }
 
     public Map<String, TokenizerFactory> buildTokenizerFactories(IndexSettings indexSettings) throws IOException {

server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java

@@ -81,9 +81,7 @@ public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider<Custom
             if (tokenFilter == null) {
                 throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find filter under name [" + tokenFilterName + "]");
             }
-            // no need offsetGap for tokenize synonyms
-            tokenFilter = checkAndApplySynonymFilter(tokenFilter, tokenizerName, tokenizer, tokenFilterList, charFiltersList,
-                this.environment);
+            tokenFilter = tokenFilter.getChainAwareTokenFilterFactory(tokenizer, charFiltersList, tokenFilterList, tokenFilters::get);
             tokenFilterList.add(tokenFilter);
         }
 
@@ -95,33 +93,6 @@ public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider<Custom
         );
     }
 
-    public static TokenFilterFactory checkAndApplySynonymFilter(TokenFilterFactory tokenFilter, String tokenizerName, TokenizerFactory tokenizer,
-                                                                List<TokenFilterFactory> tokenFilterList,
-                                                                List<CharFilterFactory> charFiltersList, Environment env) {
-        if (tokenFilter instanceof SynonymGraphTokenFilterFactory) {
-            List<TokenFilterFactory> tokenFiltersListForSynonym = new ArrayList<>(tokenFilterList);
-
-            try (CustomAnalyzer analyzer = new CustomAnalyzer(tokenizerName, tokenizer,
-                    charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]),
-                    tokenFiltersListForSynonym.toArray(new TokenFilterFactory[tokenFiltersListForSynonym.size()]),
-                    TextFieldMapper.Defaults.POSITION_INCREMENT_GAP,
-                    -1)){
-                tokenFilter = ((SynonymGraphTokenFilterFactory) tokenFilter).createPerAnalyzerSynonymGraphFactory(analyzer, env);
-            }
-
-        } else if (tokenFilter instanceof SynonymTokenFilterFactory) {
-            List<TokenFilterFactory> tokenFiltersListForSynonym = new ArrayList<>(tokenFilterList);
-            try (CustomAnalyzer analyzer = new CustomAnalyzer(tokenizerName, tokenizer,
-                    charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]),
-                    tokenFiltersListForSynonym.toArray(new TokenFilterFactory[tokenFiltersListForSynonym.size()]),
-                    TextFieldMapper.Defaults.POSITION_INCREMENT_GAP,
-                    -1)) {
-                tokenFilter = ((SynonymTokenFilterFactory) tokenFilter).createPerAnalyzerSynonymFactory(analyzer, env);
-            }
-        }
-        return tokenFilter;
-    }
-
     @Override
     public CustomAnalyzer get() {
         return this.customAnalyzer;

server/src/main/java/org/elasticsearch/index/analysis/SynonymGraphTokenFilterFactory.java

@@ -28,9 +28,11 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 
 import java.io.IOException;
-import java.io.Reader;
+import java.util.List;
+import java.util.function.Function;
 
 public class SynonymGraphTokenFilterFactory extends SynonymTokenFilterFactory {
+
     public SynonymGraphTokenFilterFactory(IndexSettings indexSettings, Environment env, AnalysisRegistry analysisRegistry,
                                      String name, Settings settings) throws IOException {
         super(indexSettings, env, analysisRegistry, name, settings);
@@ -41,42 +43,24 @@ public class SynonymGraphTokenFilterFactory extends SynonymTokenFilterFactory {
         throw new IllegalStateException("Call createPerAnalyzerSynonymGraphFactory to specialize this factory for an analysis chain first");
     }
 
-    Factory createPerAnalyzerSynonymGraphFactory(Analyzer analyzerForParseSynonym, Environment env){
-        return new Factory("synonymgraph", analyzerForParseSynonym, getRulesFromSettings(env));
-    }
-
-    public class Factory implements TokenFilterFactory{
-
-        private final String name;
-        private final SynonymMap synonymMap;
-
-        public Factory(String name, final Analyzer analyzerForParseSynonym, Reader rulesReader) {
-            this.name = name;
-
-            try {
-                SynonymMap.Builder parser;
-                if ("wordnet".equalsIgnoreCase(format)) {
-                    parser = new ESWordnetSynonymParser(true, expand, lenient, analyzerForParseSynonym);
-                    ((ESWordnetSynonymParser) parser).parse(rulesReader);
-                } else {
-                    parser = new ESSolrSynonymParser(true, expand, lenient, analyzerForParseSynonym);
-                    ((ESSolrSynonymParser) parser).parse(rulesReader);
-                }
-                synonymMap = parser.build();
-            } catch (Exception e) {
-                throw new IllegalArgumentException("failed to build synonyms", e);
+    @Override
+    public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
+                                                              List<TokenFilterFactory> previousTokenFilters,
+                                                              Function<String, TokenFilterFactory> allFilters) {
+        final Analyzer analyzer = buildSynonymAnalyzer(tokenizer, charFilters, previousTokenFilters);
+        final SynonymMap synonyms = buildSynonyms(analyzer, getRulesFromSettings(environment));
+        final String name = name();
+        return new TokenFilterFactory() {
+            @Override
+            public String name() {
+                return name;
             }
-        }
 
-        @Override
-        public String name() {
-            return this.name;
-        }
-
-        @Override
-        public TokenStream create(TokenStream tokenStream) {
-            // fst is null means no synonyms
-            return synonymMap.fst == null ? tokenStream : new SynonymGraphFilter(tokenStream, synonymMap, false);
-        }
+            @Override
+            public TokenStream create(TokenStream tokenStream) {
+                return synonyms.fst == null ? tokenStream : new SynonymGraphFilter(tokenStream, synonyms, false);
+            }
+        };
     }
+
 }

server/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java

@@ -31,6 +31,7 @@ import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
 import java.util.List;
+import java.util.function.Function;
 
 public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
 
@@ -38,6 +39,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
     protected final boolean expand;
     protected final boolean lenient;
     protected final Settings settings;
+    protected final Environment environment;
 
     public SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env, AnalysisRegistry analysisRegistry,
                                       String name, Settings settings) throws IOException {
@@ -53,6 +55,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
         this.expand = settings.getAsBoolean("expand", true);
         this.lenient = settings.getAsBoolean("lenient", false);
         this.format = settings.get("format", "");
+        this.environment = env;
     }
 
     @Override
@@ -60,6 +63,50 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
         throw new IllegalStateException("Call createPerAnalyzerSynonymFactory to specialize this factory for an analysis chain first");
     }
 
+    @Override
+    public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
+                                                              List<TokenFilterFactory> previousTokenFilters,
+                                                              Function<String, TokenFilterFactory> allFilters) {
+        final Analyzer analyzer = buildSynonymAnalyzer(tokenizer, charFilters, previousTokenFilters);
+        final SynonymMap synonyms = buildSynonyms(analyzer, getRulesFromSettings(environment));
+        final String name = name();
+        return new TokenFilterFactory() {
+            @Override
+            public String name() {
+                return name;
+            }
+
+            @Override
+            public TokenStream create(TokenStream tokenStream) {
+                return synonyms.fst == null ? tokenStream : new SynonymFilter(tokenStream, synonyms, false);
+            }
+        };
+    }
+
+    protected Analyzer buildSynonymAnalyzer(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
+                                            List<TokenFilterFactory> tokenFilters) {
+        return new CustomAnalyzer("synonyms", tokenizer, charFilters.toArray(new CharFilterFactory[0]),
+            tokenFilters.stream()
+                .map(TokenFilterFactory::getSynonymFilter)
+                .toArray(TokenFilterFactory[]::new));
+    }
+
+    protected SynonymMap buildSynonyms(Analyzer analyzer, Reader rules) {
+        try {
+            SynonymMap.Builder parser;
+            if ("wordnet".equalsIgnoreCase(format)) {
+                parser = new ESWordnetSynonymParser(true, expand, lenient, analyzer);
+                ((ESWordnetSynonymParser) parser).parse(rules);
+            } else {
+                parser = new ESSolrSynonymParser(true, expand, lenient, analyzer);
+                ((ESSolrSynonymParser) parser).parse(rules);
+            }
+            return parser.build();
+        } catch (Exception e) {
+            throw new IllegalArgumentException("failed to build synonyms", e);
+        }
+    }
+
     protected Reader getRulesFromSettings(Environment env) {
         Reader rulesReader;
         if (settings.getAsList("synonyms", null) != null) {
@@ -77,44 +124,4 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
         return rulesReader;
     }
 
-    Factory createPerAnalyzerSynonymFactory(Analyzer analyzerForParseSynonym, Environment env){
-        return new Factory("synonym", analyzerForParseSynonym, getRulesFromSettings(env));
-    }
-
-    public class Factory implements TokenFilterFactory{
-
-        private final String name;
-        private final SynonymMap synonymMap;
-
-        public Factory(String name, Analyzer analyzerForParseSynonym, Reader rulesReader) {
-
-            this.name = name;
-
-            try {
-                SynonymMap.Builder parser;
-                if ("wordnet".equalsIgnoreCase(format)) {
-                    parser = new ESWordnetSynonymParser(true, expand, lenient, analyzerForParseSynonym);
-                    ((ESWordnetSynonymParser) parser).parse(rulesReader);
-                } else {
-                    parser = new ESSolrSynonymParser(true, expand, lenient, analyzerForParseSynonym);
-                    ((ESSolrSynonymParser) parser).parse(rulesReader);
-                }
-                synonymMap = parser.build();
-            } catch (Exception e) {
-                throw new IllegalArgumentException("failed to build synonyms", e);
-            }
-        }
-
-        @Override
-        public String name() {
-            return this.name;
-        }
-
-        @Override
-        public TokenStream create(TokenStream tokenStream) {
-            // fst is null means no synonyms
-            return synonymMap.fst == null ? tokenStream : new SynonymFilter(tokenStream, synonymMap, false);
-        }
-    }
-
 }

server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java

@@ -23,6 +23,9 @@ import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter;
 
+import java.util.List;
+import java.util.function.Function;
+
 public interface TokenFilterFactory {
     String name();
 
@@ -36,4 +39,43 @@ public interface TokenFilterFactory {
     default boolean breaksFastVectorHighlighter() {
         return false;
     }
+
+    /**
+     * Rewrite the TokenFilterFactory to take into account the preceding analysis chain, or refer
+     * to other TokenFilterFactories
+     * @param tokenizer             the TokenizerFactory for the preceding chain
+     * @param charFilters           any CharFilterFactories for the preceding chain
+     * @param previousTokenFilters  a list of TokenFilterFactories in the preceding chain
+     * @param allFilters            access to previously defined TokenFilterFactories
+     */
+    default TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
+                                                               List<TokenFilterFactory> previousTokenFilters,
+                                                               Function<String, TokenFilterFactory> allFilters) {
+        return this;
+    }
+
+    /**
+     * Return a version of this TokenFilterFactory appropriate for synonym parsing
+     *
+     * Filters that should not be applied to synonyms (for example, those that produce
+     * multiple tokens) can return {@link #IDENTITY_FILTER}
+     */
+    default TokenFilterFactory getSynonymFilter() {
+        return this;
+    }
+
+    /**
+     * A TokenFilterFactory that does no filtering to its TokenStream
+     */
+    TokenFilterFactory IDENTITY_FILTER = new TokenFilterFactory() {
+        @Override
+        public String name() {
+            return "identity";
+        }
+
+        @Override
+        public TokenStream create(TokenStream tokenStream) {
+            return tokenStream;
+        }
+    };
 }

server/src/main/java/org/elasticsearch/index/engine/Engine.java

@@ -66,6 +66,7 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
@@ -175,6 +176,41 @@ public abstract class Engine implements Closeable {
     /** Returns how many bytes we are currently moving from heap to disk */
     public abstract long getWritingBytes();
 
+    /**
+     * Returns the {@link DocsStats} for this engine
+     */
+    public DocsStats docStats() {
+        // we calculate the doc stats based on the internal reader that is more up-to-date and not subject
+        // to external refreshes. For instance we don't refresh an external reader if we flush and indices with
+        // index.refresh_interval=-1 won't see any doc stats updates at all. This change will give more accurate statistics
+        // when indexing but not refreshing in general. Yet, if a refresh happens the internal reader is refresh as well so we are
+        // safe here.
+        try (Engine.Searcher searcher = acquireSearcher("docStats", Engine.SearcherScope.INTERNAL)) {
+           return docsStats(searcher.reader());
+        }
+    }
+
+    protected final DocsStats docsStats(IndexReader indexReader) {
+        long numDocs = 0;
+        long numDeletedDocs = 0;
+        long sizeInBytes = 0;
+        // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
+        // the next scheduled refresh to go through and refresh the stats as well
+        for (LeafReaderContext readerContext : indexReader.leaves()) {
+            // we go on the segment level here to get accurate numbers
+            final SegmentReader segmentReader = Lucene.segmentReader(readerContext.reader());
+            SegmentCommitInfo info = segmentReader.getSegmentInfo();
+            numDocs += readerContext.reader().numDocs();
+            numDeletedDocs += readerContext.reader().numDeletedDocs();
+            try {
+                sizeInBytes += info.sizeInBytes();
+            } catch (IOException e) {
+                logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e);
+            }
+        }
+        return new DocsStats(numDocs, numDeletedDocs, sizeInBytes);
+    }
+
     /**
      * A throttling class that can be activated, causing the
      * {@code acquireThrottle} method to block on a lock when throttling

server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java

@@ -34,6 +34,7 @@ import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogStats;
@@ -63,6 +64,7 @@ public final class ReadOnlyEngine extends Engine {
     private final SearcherManager searcherManager;
     private final IndexCommit indexCommit;
     private final Lock indexWriterLock;
+    private final DocsStats docsStats;
 
     /**
      * Creates a new ReadOnlyEngine. This ctor can also be used to open a read-only engine on top of an already opened
@@ -101,6 +103,7 @@ public final class ReadOnlyEngine extends Engine {
                 this.indexCommit = reader.getIndexCommit();
                 this.searcherManager = new SearcherManager(reader,
                     new RamAccountingSearcherFactory(engineConfig.getCircuitBreakerService()));
+                this.docsStats = docsStats(reader);
                 this.indexWriterLock = indexWriterLock;
                 success = true;
             } finally {
@@ -365,4 +368,9 @@ public final class ReadOnlyEngine extends Engine {
     @Override
     public void maybePruneDeletes() {
     }
+
+    @Override
+    public DocsStats docStats() {
+        return docsStats;
+    }
 }

server/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -21,13 +21,9 @@ package org.elasticsearch.index.shard;
 
 import com.carrotsearch.hppc.ObjectLongMap;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.index.CheckIndex;
 import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
-import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryCachingPolicy;
@@ -879,32 +875,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     }
 
     public DocsStats docStats() {
-        // we calculate the doc stats based on the internal reader that is more up-to-date and not subject
-        // to external refreshes. For instance we don't refresh an external reader if we flush and indices with
-        // index.refresh_interval=-1 won't see any doc stats updates at all. This change will give more accurate statistics
-        // when indexing but not refreshing in general. Yet, if a refresh happens the internal reader is refresh as well so we are
-        // safe here.
-        long numDocs = 0;
-        long numDeletedDocs = 0;
-        long sizeInBytes = 0;
-        try (Engine.Searcher searcher = acquireSearcher("docStats", Engine.SearcherScope.INTERNAL)) {
-            // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
-            // the next scheduled refresh to go through and refresh the stats as well
-            markSearcherAccessed();
-            for (LeafReaderContext reader : searcher.reader().leaves()) {
-                // we go on the segment level here to get accurate numbers
-                final SegmentReader segmentReader = Lucene.segmentReader(reader.reader());
-                SegmentCommitInfo info = segmentReader.getSegmentInfo();
-                numDocs += reader.reader().numDocs();
-                numDeletedDocs += reader.reader().numDeletedDocs();
-                try {
-                    sizeInBytes += info.sizeInBytes();
-                } catch (IOException e) {
-                    logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e);
-                }
-            }
-        }
-        return new DocsStats(numDocs, numDeletedDocs, sizeInBytes);
+        readAllowed();
+        DocsStats docsStats = getEngine().docStats();
+        markSearcherAccessed();
+        return docsStats;
     }
 
     /**

server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedLuceneSegmentsAction.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.shard;
+
+import org.apache.lucene.index.CheckIndex;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.Lock;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.collect.Tuple;
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+/**
+ * Removes corrupted Lucene index segments
+ */
+public class RemoveCorruptedLuceneSegmentsAction {
+
+    public Tuple<RemoveCorruptedShardDataCommand.CleanStatus, String> getCleanStatus(ShardPath shardPath,
+                                                                                     Directory indexDirectory,
+                                                                                     Lock writeLock,
+                                                                                     PrintStream printStream,
+                                                                                     boolean verbose) throws IOException {
+        if (RemoveCorruptedShardDataCommand.isCorruptMarkerFileIsPresent(indexDirectory) == false) {
+            return Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.CLEAN, null);
+        }
+
+        final CheckIndex.Status status;
+        try (CheckIndex checker = new CheckIndex(indexDirectory, writeLock)) {
+            checker.setChecksumsOnly(true);
+            checker.setInfoStream(printStream, verbose);
+
+            status = checker.checkIndex(null);
+
+            if (status.missingSegments) {
+                return Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.UNRECOVERABLE,
+                    "Index is unrecoverable - there are missing segments");
+            }
+
+            return status.clean
+                ? Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.CLEAN_WITH_CORRUPTED_MARKER, null)
+                : Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.CORRUPTED,
+                    "Corrupted Lucene index segments found - " + status.totLoseDocCount + " documents will be lost.");
+        }
+    }
+
+    public void execute(Terminal terminal,
+                        ShardPath shardPath,
+                        Directory indexDirectory,
+                        Lock writeLock,
+                        PrintStream printStream,
+                        boolean verbose) throws IOException {
+        checkCorruptMarkerFileIsPresent(indexDirectory);
+
+        final CheckIndex.Status status;
+        try (CheckIndex checker = new CheckIndex(indexDirectory, writeLock)) {
+
+            checker.setChecksumsOnly(true);
+            checker.setInfoStream(printStream, verbose);
+
+            status = checker.checkIndex(null);
+
+            if (status.missingSegments == false) {
+                if (status.clean == false) {
+                    terminal.println("Writing...");
+                    checker.exorciseIndex(status);
+
+                    terminal.println("OK");
+                    terminal.println("Wrote new segments file \"" + status.segmentsFileName + "\"");
+                }
+            } else {
+                throw new ElasticsearchException("Index is unrecoverable - there are missing segments");
+            }
+        }
+    }
+
+    protected void checkCorruptMarkerFileIsPresent(Directory directory) throws IOException {
+        if (RemoveCorruptedShardDataCommand.isCorruptMarkerFileIsPresent(directory) == false) {
+            throw new ElasticsearchException("There is no corruption file marker");
+        }
+    }
+
+}

server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java

@@ -0,0 +1,545 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.shard;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.NativeFSLockFactory;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.AllocationId;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.env.NodeMetaData;
+import org.elasticsearch.gateway.MetaDataStateFormat;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.translog.TruncateTranslogAction;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class RemoveCorruptedShardDataCommand extends EnvironmentAwareCommand {
+
+    private static final Logger logger = Loggers.getLogger(RemoveCorruptedShardDataCommand.class);
+
+    private final OptionSpec<String> folderOption;
+    private final OptionSpec<String> indexNameOption;
+    private final OptionSpec<Integer> shardIdOption;
+
+    private final RemoveCorruptedLuceneSegmentsAction removeCorruptedLuceneSegmentsAction;
+    private final TruncateTranslogAction truncateTranslogAction;
+    private final NamedXContentRegistry namedXContentRegistry;
+
+    public RemoveCorruptedShardDataCommand() {
+        this(false);
+    }
+
+    public RemoveCorruptedShardDataCommand(boolean translogOnly) {
+        super("Removes corrupted shard files");
+
+        folderOption = parser.acceptsAll(Arrays.asList("d", "dir"),
+            "Index directory location on disk")
+            .withRequiredArg();
+
+        indexNameOption = parser.accepts("index", "Index name")
+            .withRequiredArg();
+
+        shardIdOption = parser.accepts("shard-id", "Shard id")
+            .withRequiredArg()
+            .ofType(Integer.class);
+
+        namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
+
+        removeCorruptedLuceneSegmentsAction = translogOnly ? null : new RemoveCorruptedLuceneSegmentsAction();
+        truncateTranslogAction = new TruncateTranslogAction(namedXContentRegistry);
+    }
+
+    @Override
+    protected void printAdditionalHelp(Terminal terminal) {
+        if (removeCorruptedLuceneSegmentsAction == null) {
+            // that's only for 6.x branch for bwc with elasticsearch-translog
+            terminal.println("This tool truncates the translog and translog checkpoint files to create a new translog");
+        } else {
+            terminal.println("This tool attempts to detect and remove unrecoverable corrupted data in a shard.");
+        }
+    }
+
+    // Visible for testing
+    public OptionParser getParser() {
+        return this.parser;
+    }
+
+    @SuppressForbidden(reason = "Necessary to use the path passed in")
+    protected Path getPath(String dirValue) {
+        return PathUtils.get(dirValue, "", "");
+    }
+
+    protected void findAndProcessShardPath(OptionSet options, Environment environment, CheckedConsumer<ShardPath, IOException> consumer)
+    throws IOException {
+        final Settings settings = environment.settings();
+
+        final String indexName;
+        final int shardId;
+        final int fromNodeId;
+        final int toNodeId;
+
+        if (options.has(folderOption)) {
+            final Path path = getPath(folderOption.value(options)).getParent();
+            final Path shardParent = path.getParent();
+            final Path shardParentParent = shardParent.getParent();
+            final Path indexPath = path.resolve(ShardPath.INDEX_FOLDER_NAME);
+            if (Files.exists(indexPath) == false || Files.isDirectory(indexPath) == false) {
+                throw new ElasticsearchException("index directory [" + indexPath + "], must exist and be a directory");
+            }
+
+            final IndexMetaData indexMetaData =
+                IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, shardParent);
+
+            final String shardIdFileName = path.getFileName().toString();
+            final String nodeIdFileName = shardParentParent.getParent().getFileName().toString();
+            if (Files.isDirectory(path) && shardIdFileName.chars().allMatch(Character::isDigit) // SHARD-ID path element check
+                && NodeEnvironment.INDICES_FOLDER.equals(shardParentParent.getFileName().toString()) // `indices` check
+                && nodeIdFileName.chars().allMatch(Character::isDigit) // NODE-ID check
+                && NodeEnvironment.NODES_FOLDER.equals(shardParentParent.getParent().getParent().getFileName().toString()) // `nodes` check
+            ) {
+                shardId = Integer.parseInt(shardIdFileName);
+                indexName = indexMetaData.getIndex().getName();
+                fromNodeId = Integer.parseInt(nodeIdFileName);
+                toNodeId = fromNodeId + 1;
+            } else {
+                throw new ElasticsearchException("Unable to resolve shard id. Wrong folder structure at [ " + path.toString()
+                    + " ], expected .../nodes/[NODE-ID]/indices/[INDEX-UUID]/[SHARD-ID]");
+            }
+        } else {
+            // otherwise resolve shardPath based on the index name and shard id
+            indexName = Objects.requireNonNull(indexNameOption.value(options), "Index name is required");
+            shardId = Objects.requireNonNull(shardIdOption.value(options), "Shard ID is required");
+
+            // resolve shard path in case of multi-node layout per environment
+            fromNodeId = 0;
+            toNodeId = NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.get(settings);
+        }
+
+        // have to iterate over possibleLockId as NodeEnvironment; on a contrast to it - we have to fail if node is busy
+        for (int possibleLockId = fromNodeId; possibleLockId < toNodeId; possibleLockId++) {
+            try {
+                try (NodeEnvironment.NodeLock nodeLock = new NodeEnvironment.NodeLock(possibleLockId, logger, environment, Files::exists)) {
+                    final NodeEnvironment.NodePath[] nodePaths = nodeLock.getNodePaths();
+                    for (NodeEnvironment.NodePath nodePath : nodePaths) {
+                        if (Files.exists(nodePath.indicesPath)) {
+                            // have to scan all index uuid folders to resolve from index name
+                            try (DirectoryStream<Path> stream = Files.newDirectoryStream(nodePath.indicesPath)) {
+                                for (Path file : stream) {
+                                    if (Files.exists(file.resolve(MetaDataStateFormat.STATE_DIR_NAME)) == false) {
+                                        continue;
+                                    }
+
+                                    final IndexMetaData indexMetaData =
+                                        IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, file);
+                                    if (indexMetaData == null) {
+                                        continue;
+                                    }
+                                    final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings);
+                                    final Index index = indexMetaData.getIndex();
+                                    if (indexName.equals(index.getName()) == false) {
+                                        continue;
+                                    }
+                                    final ShardId shId = new ShardId(index, shardId);
+
+                                    final Path shardPathLocation = nodePath.resolve(shId);
+                                    if (Files.exists(shardPathLocation) == false) {
+                                        continue;
+                                    }
+                                    final ShardPath shardPath = ShardPath.loadShardPath(logger, shId, indexSettings,
+                                        new Path[]{shardPathLocation}, possibleLockId, nodePath.path);
+                                    if (shardPath != null) {
+                                        consumer.accept(shardPath);
+                                        return;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            } catch (LockObtainFailedException lofe) {
+                throw new ElasticsearchException("Failed to lock node's directory [" + lofe.getMessage()
+                    + "], is Elasticsearch still running ?");
+            }
+        }
+        throw new ElasticsearchException("Unable to resolve shard path for index [" + indexName + "] and shard id [" + shardId + "]");
+    }
+
+    public static boolean isCorruptMarkerFileIsPresent(final Directory directory) throws IOException {
+        boolean found = false;
+
+        final String[] files = directory.listAll();
+        for (String file : files) {
+            if (file.startsWith(Store.CORRUPTED)) {
+                found = true;
+                break;
+            }
+        }
+
+        return found;
+    }
+
+    protected void dropCorruptMarkerFiles(Terminal terminal, Path path, Directory directory, boolean clean) throws IOException {
+        if (clean) {
+            confirm("This shard has been marked as corrupted but no corruption can now be detected.\n"
+                + "This may indicate an intermittent hardware problem. The corruption marker can be \n"
+                + "removed, but there is a risk that data has been undetectably lost.\n\n"
+                + "Are you taking a risk of losing documents and proceed with removing a corrupted marker ?",
+                terminal);
+        }
+        String[] files = directory.listAll();
+        boolean found = false;
+        for (String file : files) {
+            if (file.startsWith(Store.CORRUPTED)) {
+                directory.deleteFile(file);
+
+                terminal.println("Deleted corrupt marker " + file + " from " + path);
+            }
+        }
+    }
+
+    private static void loseDataDetailsBanner(Terminal terminal, Tuple<CleanStatus, String> cleanStatus) {
+        if (cleanStatus.v2() != null) {
+            terminal.println("");
+            terminal.println("  " + cleanStatus.v2());
+            terminal.println("");
+        }
+    }
+
+    private static void confirm(String msg, Terminal terminal) {
+        terminal.println(msg);
+        String text = terminal.readText("Confirm [y/N] ");
+        if (text.equalsIgnoreCase("y") == false) {
+            throw new ElasticsearchException("aborted by user");
+        }
+    }
+
+    private void warnAboutESShouldBeStopped(Terminal terminal) {
+        terminal.println("-----------------------------------------------------------------------");
+        terminal.println("");
+        terminal.println("    WARNING: Elasticsearch MUST be stopped before running this tool.");
+        terminal.println("");
+        // that's only for 6.x branch for bwc with elasticsearch-translog
+        if (removeCorruptedLuceneSegmentsAction == null) {
+            terminal.println("  This tool is deprecated and will be completely removed in 7.0.");
+            terminal.println("  It is replaced by the elasticsearch-shard tool. ");
+            terminal.println("");
+        }
+        terminal.println("  Please make a complete backup of your index before using this tool.");
+        terminal.println("");
+        terminal.println("-----------------------------------------------------------------------");
+    }
+
+    // Visible for testing
+    @Override
+    public void execute(Terminal terminal, OptionSet options, Environment environment) throws Exception {
+        warnAboutESShouldBeStopped(terminal);
+
+        findAndProcessShardPath(options, environment, shardPath -> {
+            final Path indexPath = shardPath.resolveIndex();
+            final Path translogPath = shardPath.resolveTranslog();
+            final Path nodePath = getNodePath(shardPath);
+            if (Files.exists(translogPath) == false || Files.isDirectory(translogPath) == false) {
+                throw new ElasticsearchException("translog directory [" + translogPath + "], must exist and be a directory");
+            }
+
+            final PrintWriter writer = terminal.getWriter();
+            final PrintStream printStream = new PrintStream(new OutputStream() {
+                @Override
+                public void write(int b) {
+                    writer.write(b);
+                }
+            }, false, "UTF-8");
+            final boolean verbose = terminal.isPrintable(Terminal.Verbosity.VERBOSE);
+
+            final Directory indexDirectory = getDirectory(indexPath);
+
+            final Tuple<CleanStatus, String> indexCleanStatus;
+            final Tuple<CleanStatus, String> translogCleanStatus;
+            try (Directory indexDir = indexDirectory) {
+                // keep the index lock to block any runs of older versions of this tool
+                try (Lock writeIndexLock = indexDir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
+                    ////////// Index
+                    // that's only for 6.x branch for bwc with elasticsearch-translog
+                    if (removeCorruptedLuceneSegmentsAction != null) {
+                        terminal.println("");
+                        terminal.println("Opening Lucene index at " + indexPath);
+                        terminal.println("");
+                        try {
+                            indexCleanStatus = removeCorruptedLuceneSegmentsAction.getCleanStatus(shardPath, indexDir,
+                                writeIndexLock, printStream, verbose);
+                        } catch (Exception e) {
+                            terminal.println(e.getMessage());
+                            throw e;
+                        }
+
+                        terminal.println("");
+                        terminal.println(" >> Lucene index is " + indexCleanStatus.v1().getMessage() + " at " + indexPath);
+                        terminal.println("");
+                    } else {
+                        indexCleanStatus = Tuple.tuple(CleanStatus.CLEAN, null);
+                    }
+
+                    ////////// Translog
+                    // as translog relies on data stored in an index commit - we have to have non unrecoverable index to truncate translog
+                    if (indexCleanStatus.v1() != CleanStatus.UNRECOVERABLE) {
+                        terminal.println("");
+                        terminal.println("Opening translog at " + translogPath);
+                        terminal.println("");
+                        try {
+                            translogCleanStatus = truncateTranslogAction.getCleanStatus(shardPath, indexDir);
+                        } catch (Exception e) {
+                            terminal.println(e.getMessage());
+                            throw e;
+                        }
+
+                        terminal.println("");
+                        terminal.println(" >> Translog is " + translogCleanStatus.v1().getMessage() + " at " + translogPath);
+                        terminal.println("");
+                    } else {
+                        translogCleanStatus = Tuple.tuple(CleanStatus.UNRECOVERABLE, null);
+                    }
+
+                    ////////// Drop corrupted data
+                    final CleanStatus indexStatus = indexCleanStatus.v1();
+                    final CleanStatus translogStatus = translogCleanStatus.v1();
+
+                    if (indexStatus == CleanStatus.CLEAN && translogStatus == CleanStatus.CLEAN) {
+                        throw new ElasticsearchException("Shard does not seem to be corrupted at " + shardPath.getDataPath());
+                    }
+
+                    if (indexStatus == CleanStatus.UNRECOVERABLE) {
+                        if (indexCleanStatus.v2() != null) {
+                            terminal.println("Details: " + indexCleanStatus.v2());
+                        }
+
+                        terminal.println("You can allocate a new, empty, primary shard with the following command:");
+
+                        printRerouteCommand(shardPath, terminal, false);
+
+                        throw new ElasticsearchException("Index is unrecoverable");
+                    }
+
+
+                    terminal.println("-----------------------------------------------------------------------");
+                    if (indexStatus != CleanStatus.CLEAN) {
+                        loseDataDetailsBanner(terminal, indexCleanStatus);
+                    }
+                    if (translogStatus != CleanStatus.CLEAN) {
+                        loseDataDetailsBanner(terminal, translogCleanStatus);
+                    }
+                    terminal.println("            WARNING:              YOU MAY LOSE DATA.");
+                    terminal.println("-----------------------------------------------------------------------");
+
+
+                    confirm("Continue and remove corrupted data from the shard ?", terminal);
+
+                    if (indexStatus != CleanStatus.CLEAN) {
+                        removeCorruptedLuceneSegmentsAction.execute(terminal, shardPath, indexDir,
+                            writeIndexLock, printStream, verbose);
+                    }
+
+                    if (translogStatus != CleanStatus.CLEAN) {
+                        truncateTranslogAction.execute(terminal, shardPath, indexDir);
+                    }
+                } catch (LockObtainFailedException lofe) {
+                    final String msg = "Failed to lock shard's directory at [" + indexPath + "], is Elasticsearch still running?";
+                    terminal.println(msg);
+                    throw new ElasticsearchException(msg);
+                }
+
+                final CleanStatus indexStatus = indexCleanStatus.v1();
+                final CleanStatus translogStatus = translogCleanStatus.v1();
+
+                // newHistoryCommit obtains its own lock
+                addNewHistoryCommit(indexDir, terminal, translogStatus != CleanStatus.CLEAN);
+                newAllocationId(environment, shardPath, terminal);
+                if (indexStatus != CleanStatus.CLEAN) {
+                    dropCorruptMarkerFiles(terminal, indexPath, indexDir, indexStatus == CleanStatus.CLEAN_WITH_CORRUPTED_MARKER);
+                }
+            }
+        });
+    }
+
+    private Directory getDirectory(Path indexPath) {
+        Directory directory;
+        try {
+            directory = FSDirectory.open(indexPath, NativeFSLockFactory.INSTANCE);
+        } catch (Throwable t) {
+            throw new ElasticsearchException("ERROR: could not open directory \"" + indexPath + "\"; exiting");
+        }
+        return directory;
+    }
+
+    protected void addNewHistoryCommit(Directory indexDirectory, Terminal terminal, boolean updateLocalCheckpoint) throws IOException {
+        final String historyUUID = UUIDs.randomBase64UUID();
+
+        terminal.println("Marking index with the new history uuid : " + historyUUID);
+        // commit the new history id
+        final IndexWriterConfig iwc = new IndexWriterConfig(null)
+            // we don't want merges to happen here - we call maybe merge on the engine
+            // later once we stared it up otherwise we would need to wait for it here
+            // we also don't specify a codec here and merges should use the engines for this index
+            .setCommitOnClose(false)
+            .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
+            .setMergePolicy(NoMergePolicy.INSTANCE)
+            .setOpenMode(IndexWriterConfig.OpenMode.APPEND);
+        // IndexWriter acquires directory lock by its own
+        try (IndexWriter indexWriter = new IndexWriter(indexDirectory, iwc)) {
+            final Map<String, String> userData = new HashMap<>();
+            indexWriter.getLiveCommitData().forEach(e -> userData.put(e.getKey(), e.getValue()));
+
+            if (updateLocalCheckpoint) {
+                // In order to have a safe commit invariant, we have to assign the global checkpoint to the max_seqno of the last commit.
+                // We can only safely do it because we will generate a new history uuid this shard.
+                final SequenceNumbers.CommitInfo commitInfo = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userData.entrySet());
+                // Also advances the local checkpoint of the last commit to its max_seqno.
+                userData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(commitInfo.maxSeqNo));
+            }
+
+            // commit the new history id
+            userData.put(Engine.HISTORY_UUID_KEY, historyUUID);
+
+            indexWriter.setLiveCommitData(userData.entrySet());
+            indexWriter.commit();
+        }
+    }
+
+    protected void newAllocationId(Environment environment, ShardPath shardPath, Terminal terminal) throws IOException {
+        final Path shardStatePath = shardPath.getShardStatePath();
+        final ShardStateMetaData shardStateMetaData =
+            ShardStateMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, shardStatePath);
+
+        if (shardStateMetaData == null) {
+            throw new ElasticsearchException("No shard state meta data at " + shardStatePath);
+        }
+
+        final AllocationId newAllocationId = AllocationId.newInitializing();
+
+        terminal.println("Changing allocation id " + shardStateMetaData.allocationId.getId()
+            + " to " + newAllocationId.getId());
+
+        final ShardStateMetaData newShardStateMetaData =
+            new ShardStateMetaData(shardStateMetaData.primary, shardStateMetaData.indexUUID, newAllocationId);
+
+        ShardStateMetaData.FORMAT.write(newShardStateMetaData, shardStatePath);
+
+        terminal.println("");
+        terminal.println("You should run the following command to allocate this shard:");
+
+        printRerouteCommand(shardPath, terminal, true);
+    }
+
+    private void printRerouteCommand(ShardPath shardPath, Terminal terminal, boolean allocateStale) throws IOException {
+        final IndexMetaData indexMetaData =
+            IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry,
+                shardPath.getDataPath().getParent());
+
+        final Path nodePath = getNodePath(shardPath);
+        final NodeMetaData nodeMetaData =
+            NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodePath);
+
+        if (nodeMetaData == null) {
+            throw new ElasticsearchException("No node meta data at " + nodePath);
+        }
+
+        final String nodeId = nodeMetaData.nodeId();
+        final String index = indexMetaData.getIndex().getName();
+        final int id = shardPath.getShardId().id();
+        final AllocationCommands commands = new AllocationCommands(
+            allocateStale
+                ? new AllocateStalePrimaryAllocationCommand(index, id, nodeId, false)
+                : new AllocateEmptyPrimaryAllocationCommand(index, id, nodeId, false));
+
+        terminal.println("");
+        terminal.println("POST /_cluster/reroute'\n"
+            + Strings.toString(commands, true, true) + "'");
+        terminal.println("");
+        terminal.println("You must accept the possibility of data loss by changing parameter `accept_data_loss` to `true`.");
+        terminal.println("");
+    }
+
+    private Path getNodePath(ShardPath shardPath) {
+        final Path nodePath = shardPath.getDataPath().getParent().getParent().getParent();
+        if (Files.exists(nodePath) == false || Files.exists(nodePath.resolve(MetaDataStateFormat.STATE_DIR_NAME)) == false) {
+            throw new ElasticsearchException("Unable to resolve node path for " + shardPath);
+        }
+        return nodePath;
+    }
+
+    public enum CleanStatus {
+        CLEAN("clean"),
+        CLEAN_WITH_CORRUPTED_MARKER("marked corrupted, but no corruption detected"),
+        CORRUPTED("corrupted"),
+        UNRECOVERABLE("corrupted and unrecoverable");
+
+        private final String msg;
+
+        CleanStatus(String msg) {
+            this.msg = msg;
+        }
+
+        public String getMessage() {
+            return msg;
+        }
+    }
+
+}

server/src/main/java/org/elasticsearch/index/shard/ShardPath.java

@@ -112,16 +112,31 @@ public final class ShardPath {
      * <b>Note:</b> this method resolves custom data locations for the shard.
      */
     public static ShardPath loadShardPath(Logger logger, NodeEnvironment env, ShardId shardId, IndexSettings indexSettings) throws IOException {
-        final String indexUUID = indexSettings.getUUID();
         final Path[] paths = env.availableShardPaths(shardId);
+        final int nodeLockId = env.getNodeLockId();
+        final Path sharedDataPath = env.sharedDataPath();
+        return loadShardPath(logger, shardId, indexSettings, paths, nodeLockId, sharedDataPath);
+    }
+
+    /**
+     * This method walks through the nodes shard paths to find the data and state path for the given shard. If multiple
+     * directories with a valid shard state exist the one with the highest version will be used.
+     * <b>Note:</b> this method resolves custom data locations for the shard.
+     */
+    public static ShardPath loadShardPath(Logger logger, ShardId shardId, IndexSettings indexSettings, Path[] availableShardPaths,
+                                           int nodeLockId, Path sharedDataPath) throws IOException {
+        final String indexUUID = indexSettings.getUUID();
         Path loadedPath = null;
-        for (Path path : paths) {
+        for (Path path : availableShardPaths) {
             // EMPTY is safe here because we never call namedObject
             ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
             if (load != null) {
                 if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
-                    logger.warn("{} found shard on path: [{}] with a different index UUID - this shard seems to be leftover from a different index with the same name. Remove the leftover shard in order to reuse the path with the current index", shardId, path);
-                    throw new IllegalStateException(shardId + " index UUID in shard state was: " + load.indexUUID + " expected: " + indexUUID + " on shard path: " + path);
+                    logger.warn("{} found shard on path: [{}] with a different index UUID - this "
+                        + "shard seems to be leftover from a different index with the same name. "
+                        + "Remove the leftover shard in order to reuse the path with the current index", shardId, path);
+                    throw new IllegalStateException(shardId + " index UUID in shard state was: " + load.indexUUID
+                        + " expected: " + indexUUID + " on shard path: " + path);
                 }
                 if (loadedPath == null) {
                     loadedPath = path;
@@ -137,7 +152,7 @@ public final class ShardPath {
             final Path dataPath;
             final Path statePath = loadedPath;
             if (indexSettings.hasCustomDataPath()) {
-                dataPath = env.resolveCustomLocation(indexSettings, shardId);
+                dataPath = NodeEnvironment.resolveCustomLocation(indexSettings, shardId, sharedDataPath, nodeLockId);
             } else {
                 dataPath = statePath;
             }

server/src/main/java/org/elasticsearch/index/shard/ShardToolCli.java

@@ -16,22 +16,24 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+package org.elasticsearch.index.shard;
 
-package org.elasticsearch.index.analysis;
-
-import java.util.Map;
+import org.elasticsearch.cli.LoggingAwareMultiCommand;
+import org.elasticsearch.cli.Terminal;
 
 /**
- * Marks a {@link TokenFilterFactory} that refers to other filter factories.
- *
- * The analysis registry will call {@link #setReferences(Map)} with a map of all
- * available TokenFilterFactories after all factories have been registered
+ * Class encapsulating and dispatching commands from the {@code elasticsearch-shard} command line tool
  */
-public interface ReferringFilterFactory {
+public class ShardToolCli extends LoggingAwareMultiCommand {
 
-    /**
-     * Called with a map of all registered filter factories
-     */
-    void setReferences(Map<String, TokenFilterFactory> factories);
+    private ShardToolCli() {
+        super("A CLI tool to remove corrupted parts of unrecoverable shards");
+        subcommands.put("remove-corrupted-data", new RemoveCorruptedShardDataCommand());
+    }
+
+    public static void main(String[] args) throws Exception {
+        exit(new ShardToolCli().main(args, Terminal.DEFAULT));
+    }
 
 }
+

server/src/main/java/org/elasticsearch/index/similarity/NonNegativeScoresSimilarity.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.similarity;
+
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.similarities.Similarity;
+
+/**
+ * A {@link Similarity} that rejects negative scores. This class exists so that users get
+ * an error instead of silently corrupt top hits. It should be applied to any custom or
+ * scripted similarity.
+ */
+// public for testing
+public final class NonNegativeScoresSimilarity extends Similarity {
+
+    // Escape hatch
+    private static final String ES_ENFORCE_POSITIVE_SCORES = "es.enforce.positive.scores";
+    private static final boolean ENFORCE_POSITIVE_SCORES;
+    static {
+        String enforcePositiveScores = System.getProperty(ES_ENFORCE_POSITIVE_SCORES);
+        if (enforcePositiveScores == null) {
+            ENFORCE_POSITIVE_SCORES = true;
+        } else if ("false".equals(enforcePositiveScores)) {
+            ENFORCE_POSITIVE_SCORES = false;
+        } else {
+            throw new IllegalArgumentException(ES_ENFORCE_POSITIVE_SCORES + " may only be unset or set to [false], but got [" +
+                    enforcePositiveScores + "]");
+        }
+    }
+
+    private final Similarity in;
+
+    public NonNegativeScoresSimilarity(Similarity in) {
+        this.in = in;
+    }
+
+    public Similarity getDelegate() {
+        return in;
+    }
+
+    @Override
+    public long computeNorm(FieldInvertState state) {
+        return in.computeNorm(state);
+    }
+
+    @Override
+    public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+        final SimScorer inScorer = in.scorer(boost, collectionStats, termStats);
+        return new SimScorer() {
+
+            @Override
+            public float score(float freq, long norm) {
+                float score = inScorer.score(freq, norm);
+                if (score < 0f) {
+                    if (ENFORCE_POSITIVE_SCORES) {
+                        throw new IllegalArgumentException("Similarities must not produce negative scores, but got:\n" +
+                                inScorer.explain(Explanation.match(freq, "term frequency"), norm));
+                    } else {
+                        return 0f;
+                    }
+                }
+                return score;
+            }
+
+            @Override
+            public Explanation explain(Explanation freq, long norm) {
+                Explanation expl = inScorer.explain(freq, norm);
+                if (expl.isMatch() && expl.getValue().floatValue() < 0) {
+                    expl = Explanation.match(0f, "max of:",
+                            expl, Explanation.match(0f, "Minimum allowed score"));
+                }
+                return expl;
+            }
+        };
+    }
+}

server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java

@@ -19,15 +19,22 @@
 
 package org.elasticsearch.index.similarity;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.similarities.BM25Similarity;
 import org.apache.lucene.search.similarities.BooleanSimilarity;
 import org.apache.lucene.search.similarities.ClassicSimilarity;
 import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
 import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.Similarity.SimScorer;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexModule;
@@ -44,7 +51,7 @@ import java.util.function.Supplier;
 
 public final class SimilarityService extends AbstractIndexComponent {
 
-    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(SimilarityService.class));
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(SimilarityService.class));
     public static final String DEFAULT_SIMILARITY = "BM25";
     private static final String CLASSIC_SIMILARITY = "classic";
     private static final Map<String, Function<Version, Supplier<Similarity>>> DEFAULTS;
@@ -131,8 +138,14 @@ public final class SimilarityService extends AbstractIndexComponent {
             }
             TriFunction<Settings, Version, ScriptService, Similarity> defaultFactory = BUILT_IN.get(typeName);
             TriFunction<Settings, Version, ScriptService, Similarity> factory = similarities.getOrDefault(typeName, defaultFactory);
-            final Similarity similarity = factory.apply(providerSettings, indexSettings.getIndexVersionCreated(), scriptService);
-            providers.put(name, () -> similarity);
+            Similarity similarity = factory.apply(providerSettings, indexSettings.getIndexVersionCreated(), scriptService);
+            validateSimilarity(indexSettings.getIndexVersionCreated(), similarity);
+            if (BUILT_IN.containsKey(typeName) == false || "scripted".equals(typeName)) {
+                // We don't trust custom similarities
+                similarity = new NonNegativeScoresSimilarity(similarity);
+            }
+            final Similarity similarityF = similarity; // like similarity but final
+            providers.put(name, () -> similarityF);
         }
         for (Map.Entry<String, Function<Version, Supplier<Similarity>>> entry : DEFAULTS.entrySet()) {
             providers.put(entry.getKey(), entry.getValue().apply(indexSettings.getIndexVersionCreated()));
@@ -151,7 +164,7 @@ public final class SimilarityService extends AbstractIndexComponent {
                 defaultSimilarity;
     }
 
-    
+
     public SimilarityProvider getSimilarity(String name) {
         Supplier<Similarity> sim = similarities.get(name);
         if (sim == null) {
@@ -182,4 +195,80 @@ public final class SimilarityService extends AbstractIndexComponent {
             return (fieldType != null && fieldType.similarity() != null) ? fieldType.similarity().get() : defaultSimilarity;
         }
     }
+
+    static void validateSimilarity(Version indexCreatedVersion, Similarity similarity) {
+        validateScoresArePositive(indexCreatedVersion, similarity);
+        validateScoresDoNotDecreaseWithFreq(indexCreatedVersion, similarity);
+        validateScoresDoNotIncreaseWithNorm(indexCreatedVersion, similarity);
+    }
+
+    private static void validateScoresArePositive(Version indexCreatedVersion, Similarity similarity) {
+        CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000);
+        TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130);
+        SimScorer scorer = similarity.scorer(2f, collectionStats, termStats);
+        FieldInvertState state = new FieldInvertState(indexCreatedVersion.major, "some_field",
+                IndexOptions.DOCS_AND_FREQS, 20, 20, 0, 50, 10, 3); // length = 20, no overlap
+        final long norm = similarity.computeNorm(state);
+        for (int freq = 1; freq <= 10; ++freq) {
+            float score = scorer.score(freq, norm);
+            if (score < 0) {
+                fail(indexCreatedVersion, "Similarities should not return negative scores:\n" +
+                        scorer.explain(Explanation.match(freq, "term freq"), norm));
+            }
+        }
+    }
+
+    private static void validateScoresDoNotDecreaseWithFreq(Version indexCreatedVersion, Similarity similarity) {
+        CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000);
+        TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130);
+        SimScorer scorer = similarity.scorer(2f, collectionStats, termStats);
+        FieldInvertState state = new FieldInvertState(indexCreatedVersion.major, "some_field",
+                IndexOptions.DOCS_AND_FREQS, 20, 20, 0, 50, 10, 3); // length = 20, no overlap
+        final long norm = similarity.computeNorm(state);
+        float previousScore = 0;
+        for (int freq = 1; freq <= 10; ++freq) {
+            float score = scorer.score(freq, norm);
+            if (score < previousScore) {
+                fail(indexCreatedVersion, "Similarity scores should not decrease when term frequency increases:\n" +
+                        scorer.explain(Explanation.match(freq - 1, "term freq"), norm) + "\n" +
+                        scorer.explain(Explanation.match(freq, "term freq"), norm));
+            }
+            previousScore = score;
+        }
+    }
+
+    private static void validateScoresDoNotIncreaseWithNorm(Version indexCreatedVersion, Similarity similarity) {
+        CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000);
+        TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130);
+        SimScorer scorer = similarity.scorer(2f, collectionStats, termStats);
+
+        long previousNorm = 0;
+        float previousScore = Float.MAX_VALUE;
+        for (int length = 1; length <= 10; ++length) {
+            FieldInvertState state = new FieldInvertState(indexCreatedVersion.major, "some_field",
+                    IndexOptions.DOCS_AND_FREQS, length, length, 0, 50, 10, 3); // length = 20, no overlap
+            final long norm = similarity.computeNorm(state);
+            if (Long.compareUnsigned(previousNorm, norm) > 0) {
+                // esoteric similarity, skip this check
+                break;
+            }
+            float score = scorer.score(1, norm);
+            if (score > previousScore) {
+                fail(indexCreatedVersion, "Similarity scores should not increase when norm increases:\n" +
+                        scorer.explain(Explanation.match(1, "term freq"), norm - 1) + "\n" +
+                        scorer.explain(Explanation.match(1, "term freq"), norm));
+            }
+            previousScore = score;
+            previousNorm = norm;
+        }
+    }
+
+    private static void fail(Version indexCreatedVersion, String message) {
+        if (indexCreatedVersion.onOrAfter(Version.V_7_0_0_alpha1)) {
+            throw new IllegalArgumentException(message);
+        } else if (indexCreatedVersion.onOrAfter(Version.V_6_5_0)) {
+            DEPRECATION_LOGGER.deprecated(message);
+        }
+    }
+
 }

server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java

@@ -144,7 +144,6 @@ final class TranslogHeader {
         final long primaryTerm;
         if (version == VERSION_PRIMARY_TERM) {
             primaryTerm = in.readLong();
-            assert primaryTerm >= 0 : "Primary term must be non-negative [" + primaryTerm + "]; translog path [" + path + "]";
         } else {
             assert version == VERSION_CHECKPOINTS : "Unknown header version [" + version + "]";
             primaryTerm = UNKNOWN_PRIMARY_TERM;
@@ -153,6 +152,8 @@ final class TranslogHeader {
         if (version >= VERSION_PRIMARY_TERM) {
             Translog.verifyChecksum(in);
         }
+        assert primaryTerm >= 0 : "Primary term must be non-negative [" + primaryTerm + "]; translog path [" + path + "]";
+
         final int headerSizeInBytes = headerSizeInBytes(version, uuid.length);
         assert channel.position() == headerSizeInBytes :
             "Header is not fully read; header size [" + headerSizeInBytes + "], position [" + channel.position() + "]";

server/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java

@@ -21,15 +21,18 @@ package org.elasticsearch.index.translog;
 
 import org.elasticsearch.cli.LoggingAwareMultiCommand;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.index.shard.RemoveCorruptedShardDataCommand;
 
 /**
  * Class encapsulating and dispatching commands from the {@code elasticsearch-translog} command line tool
  */
+@Deprecated
 public class TranslogToolCli extends LoggingAwareMultiCommand {
 
     private TranslogToolCli() {
+        // that's only for 6.x branch for bwc with elasticsearch-translog
         super("A CLI tool for various Elasticsearch translog actions");
-        subcommands.put("truncate", new TruncateTranslogCommand());
+        subcommands.put("truncate", new RemoveCorruptedShardDataCommand(true));
     }
 
     public static void main(String[] args) throws Exception {

server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java

@@ -0,0 +1,245 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.translog;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.shard.RemoveCorruptedShardDataCommand;
+import org.elasticsearch.index.shard.ShardPath;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+public class TruncateTranslogAction {
+
+    protected static final Logger logger = Loggers.getLogger(TruncateTranslogAction.class);
+    private final NamedXContentRegistry namedXContentRegistry;
+
+    public TruncateTranslogAction(NamedXContentRegistry namedXContentRegistry) {
+        this.namedXContentRegistry = namedXContentRegistry;
+    }
+
+    public Tuple<RemoveCorruptedShardDataCommand.CleanStatus, String> getCleanStatus(ShardPath shardPath,
+                                                                                     Directory indexDirectory) throws IOException {
+        final Path indexPath = shardPath.resolveIndex();
+        final Path translogPath = shardPath.resolveTranslog();
+        final List<IndexCommit> commits;
+        try {
+            commits = DirectoryReader.listCommits(indexDirectory);
+        } catch (IndexNotFoundException infe) {
+            throw new ElasticsearchException("unable to find a valid shard at [" + indexPath + "]", infe);
+        }
+
+        // Retrieve the generation and UUID from the existing data
+        final Map<String, String> commitData = new HashMap<>(commits.get(commits.size() - 1).getUserData());
+        final String translogUUID = commitData.get(Translog.TRANSLOG_UUID_KEY);
+
+        if (translogUUID == null) {
+            throw new ElasticsearchException("shard must have a valid translog UUID but got: [null]");
+        }
+
+        final boolean clean = isTranslogClean(shardPath, translogUUID);
+
+        if (clean) {
+            return Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.CLEAN, null);
+        }
+
+        // Hold the lock open for the duration of the tool running
+        Set<Path> translogFiles;
+        try {
+            translogFiles = filesInDirectory(translogPath);
+        } catch (IOException e) {
+            throw new ElasticsearchException("failed to find existing translog files", e);
+        }
+        final String details = deletingFilesDetails(translogPath, translogFiles);
+
+        return Tuple.tuple(RemoveCorruptedShardDataCommand.CleanStatus.CORRUPTED, details);
+    }
+
+    public void execute(Terminal terminal, ShardPath shardPath, Directory indexDirectory) throws IOException {
+        final Path indexPath = shardPath.resolveIndex();
+        final Path translogPath = shardPath.resolveTranslog();
+
+        final String historyUUID = UUIDs.randomBase64UUID();
+        final Map<String, String> commitData;
+        // Hold the lock open for the duration of the tool running
+        Set<Path> translogFiles;
+        try {
+            terminal.println("Checking existing translog files");
+            translogFiles = filesInDirectory(translogPath);
+        } catch (IOException e) {
+            terminal.println("encountered IOException while listing directory, aborting...");
+            throw new ElasticsearchException("failed to find existing translog files", e);
+        }
+
+        List<IndexCommit> commits;
+        try {
+            terminal.println("Reading translog UUID information from Lucene commit from shard at [" + indexPath + "]");
+            commits = DirectoryReader.listCommits(indexDirectory);
+        } catch (IndexNotFoundException infe) {
+            throw new ElasticsearchException("unable to find a valid shard at [" + indexPath + "]", infe);
+        }
+
+        // Retrieve the generation and UUID from the existing data
+        commitData = commits.get(commits.size() - 1).getUserData();
+        final String translogGeneration = commitData.get(Translog.TRANSLOG_GENERATION_KEY);
+        final String translogUUID = commitData.get(Translog.TRANSLOG_UUID_KEY);
+        if (translogGeneration == null || translogUUID == null) {
+            throw new ElasticsearchException("shard must have a valid translog generation and UUID but got: [{}] and: [{}]",
+                translogGeneration, translogUUID);
+        }
+
+        final long globalCheckpoint = commitData.containsKey(SequenceNumbers.MAX_SEQ_NO)
+            ? Long.parseLong(commitData.get(SequenceNumbers.MAX_SEQ_NO))
+            : SequenceNumbers.UNASSIGNED_SEQ_NO;
+
+        terminal.println("Translog Generation: " + translogGeneration);
+        terminal.println("Translog UUID      : " + translogUUID);
+        terminal.println("History UUID       : " + historyUUID);
+
+        Path tempEmptyCheckpoint = translogPath.resolve("temp-" + Translog.CHECKPOINT_FILE_NAME);
+        Path realEmptyCheckpoint = translogPath.resolve(Translog.CHECKPOINT_FILE_NAME);
+        Path tempEmptyTranslog = translogPath.resolve("temp-" + Translog.TRANSLOG_FILE_PREFIX +
+            translogGeneration + Translog.TRANSLOG_FILE_SUFFIX);
+        Path realEmptyTranslog = translogPath.resolve(Translog.TRANSLOG_FILE_PREFIX +
+            translogGeneration + Translog.TRANSLOG_FILE_SUFFIX);
+
+        // Write empty checkpoint and translog to empty files
+        long gen = Long.parseLong(translogGeneration);
+        int translogLen = writeEmptyTranslog(tempEmptyTranslog, translogUUID);
+        writeEmptyCheckpoint(tempEmptyCheckpoint, translogLen, gen, globalCheckpoint);
+
+        terminal.println("Removing existing translog files");
+        IOUtils.rm(translogFiles.toArray(new Path[]{}));
+
+        terminal.println("Creating new empty checkpoint at [" + realEmptyCheckpoint + "]");
+        Files.move(tempEmptyCheckpoint, realEmptyCheckpoint, StandardCopyOption.ATOMIC_MOVE);
+        terminal.println("Creating new empty translog at [" + realEmptyTranslog + "]");
+        Files.move(tempEmptyTranslog, realEmptyTranslog, StandardCopyOption.ATOMIC_MOVE);
+
+        // Fsync the translog directory after rename
+        IOUtils.fsync(translogPath, true);
+    }
+
+    private boolean isTranslogClean(ShardPath shardPath, String translogUUID) throws IOException {
+        // perform clean check of translog instead of corrupted marker file
+        boolean clean = true;
+        try {
+            final Path translogPath = shardPath.resolveTranslog();
+            final long translogGlobalCheckpoint = Translog.readGlobalCheckpoint(translogPath, translogUUID);
+            final IndexMetaData indexMetaData =
+                IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, shardPath.getDataPath().getParent());
+            final IndexSettings indexSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
+            final TranslogConfig translogConfig = new TranslogConfig(shardPath.getShardId(), translogPath,
+                indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
+            long primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardPath.getShardId().id());
+            final TranslogDeletionPolicy translogDeletionPolicy =
+                new TranslogDeletionPolicy(indexSettings.getTranslogRetentionSize().getBytes(),
+                    indexSettings.getTranslogRetentionAge().getMillis());
+            try (Translog translog = new Translog(translogConfig, translogUUID,
+                translogDeletionPolicy, () -> translogGlobalCheckpoint, () -> primaryTerm);
+                 Translog.Snapshot snapshot = translog.newSnapshot()) {
+                while (snapshot.next() != null) {
+                    // just iterate over snapshot
+                }
+            }
+        } catch (TranslogCorruptedException e) {
+            clean = false;
+        }
+        return clean;
+    }
+
+    /** Write a checkpoint file to the given location with the given generation */
+    static void writeEmptyCheckpoint(Path filename, int translogLength, long translogGeneration, long globalCheckpoint) throws IOException {
+        Checkpoint emptyCheckpoint = Checkpoint.emptyTranslogCheckpoint(translogLength, translogGeneration,
+            globalCheckpoint, translogGeneration);
+        Checkpoint.write(FileChannel::open, filename, emptyCheckpoint,
+            StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW);
+        // fsync with metadata here to make sure.
+        IOUtils.fsync(filename, false);
+    }
+
+    /**
+     * Write a translog containing the given translog UUID to the given location. Returns the number of bytes written.
+     */
+    private static int writeEmptyTranslog(Path filename, String translogUUID) throws IOException {
+        try (FileChannel fc = FileChannel.open(filename, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)) {
+            TranslogHeader header = new TranslogHeader(translogUUID, TranslogHeader.UNKNOWN_PRIMARY_TERM);
+            header.write(fc);
+            return header.sizeInBytes();
+        }
+    }
+
+    /** Show a warning about deleting files, asking for a confirmation if {@code batchMode} is false */
+    private String deletingFilesDetails(Path translogPath, Set<Path> files) {
+        StringBuilder builder = new StringBuilder();
+
+        builder
+            .append("Documents inside of translog files will be lost.\n")
+            .append("  The following files will be DELETED at ")
+            .append(translogPath)
+            .append("\n\n");
+        for(Iterator<Path> it = files.iterator();it.hasNext();) {
+            builder.append("  --> ").append(it.next().getFileName());
+            if (it.hasNext()) {
+                builder.append("\n");
+            }
+        }
+        return builder.toString();
+    }
+
+    /** Return a Set of all files in a given directory */
+    public static Set<Path> filesInDirectory(Path directory) throws IOException {
+        Set<Path> files = new TreeSet<>();
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory)) {
+            for (Path file : stream) {
+                files.add(file);
+            }
+        }
+        return files;
+    }
+
+}

server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java

@@ -1,254 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.translog;
-
-import joptsimple.OptionParser;
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.NoMergePolicy;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.NativeFSLockFactory;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.core.internal.io.IOUtils;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cli.EnvironmentAwareCommand;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.seqno.SequenceNumbers;
-
-import java.io.IOException;
-import java.nio.channels.FileChannel;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.nio.file.StandardOpenOption;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class TruncateTranslogCommand extends EnvironmentAwareCommand {
-
-    private final OptionSpec<String> translogFolder;
-    private final OptionSpec<Void> batchMode;
-
-    public TruncateTranslogCommand() {
-        super("Truncates a translog to create a new, empty translog");
-        this.translogFolder = parser.acceptsAll(Arrays.asList("d", "dir"),
-                "Translog Directory location on disk")
-                .withRequiredArg()
-                .required();
-        this.batchMode = parser.acceptsAll(Arrays.asList("b", "batch"),
-                "Enable batch mode explicitly, automatic confirmation of warnings");
-    }
-
-    // Visible for testing
-    public OptionParser getParser() {
-        return this.parser;
-    }
-
-    @Override
-    protected void printAdditionalHelp(Terminal terminal) {
-        terminal.println("This tool truncates the translog and translog");
-        terminal.println("checkpoint files to create a new translog");
-    }
-
-    @SuppressForbidden(reason = "Necessary to use the path passed in")
-    private Path getTranslogPath(OptionSet options) {
-        return PathUtils.get(translogFolder.value(options), "", "");
-    }
-
-    @Override
-    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
-        boolean batch = options.has(batchMode);
-
-        Path translogPath = getTranslogPath(options);
-        Path idxLocation = translogPath.getParent().resolve("index");
-
-        if (Files.exists(translogPath) == false || Files.isDirectory(translogPath) == false) {
-            throw new ElasticsearchException("translog directory [" + translogPath + "], must exist and be a directory");
-        }
-
-        if (Files.exists(idxLocation) == false || Files.isDirectory(idxLocation) == false) {
-            throw new ElasticsearchException("unable to find a shard at [" + idxLocation + "], which must exist and be a directory");
-        }
-        try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE)) {
-            final String historyUUID = UUIDs.randomBase64UUID();
-            final Map<String, String> commitData;
-            // Hold the lock open for the duration of the tool running
-            try (Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
-                Set<Path> translogFiles;
-                try {
-                    terminal.println("Checking existing translog files");
-                    translogFiles = filesInDirectory(translogPath);
-                } catch (IOException e) {
-                    terminal.println("encountered IOException while listing directory, aborting...");
-                    throw new ElasticsearchException("failed to find existing translog files", e);
-                }
-
-                // Warn about ES being stopped and files being deleted
-                warnAboutDeletingFiles(terminal, translogFiles, batch);
-
-                List<IndexCommit> commits;
-                try {
-                    terminal.println("Reading translog UUID information from Lucene commit from shard at [" + idxLocation + "]");
-                    commits = DirectoryReader.listCommits(dir);
-                } catch (IndexNotFoundException infe) {
-                    throw new ElasticsearchException("unable to find a valid shard at [" + idxLocation + "]", infe);
-                }
-
-                // Retrieve the generation and UUID from the existing data
-                commitData = new HashMap<>(commits.get(commits.size() - 1).getUserData());
-                String translogGeneration = commitData.get(Translog.TRANSLOG_GENERATION_KEY);
-                String translogUUID = commitData.get(Translog.TRANSLOG_UUID_KEY);
-                final long globalCheckpoint;
-                // In order to have a safe commit invariant, we have to assign the global checkpoint to the max_seqno of the last commit.
-                // We can only safely do it because we will generate a new history uuid this shard.
-                if (commitData.containsKey(SequenceNumbers.MAX_SEQ_NO)) {
-                    globalCheckpoint = Long.parseLong(commitData.get(SequenceNumbers.MAX_SEQ_NO));
-                    // Also advances the local checkpoint of the last commit to its max_seqno.
-                    commitData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(globalCheckpoint));
-                } else {
-                    globalCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
-                }
-                if (translogGeneration == null || translogUUID == null) {
-                    throw new ElasticsearchException("shard must have a valid translog generation and UUID but got: [{}] and: [{}]",
-                        translogGeneration, translogUUID);
-                }
-                terminal.println("Translog Generation: " + translogGeneration);
-                terminal.println("Translog UUID      : " + translogUUID);
-                terminal.println("History UUID      : " + historyUUID);
-
-                Path tempEmptyCheckpoint = translogPath.resolve("temp-" + Translog.CHECKPOINT_FILE_NAME);
-                Path realEmptyCheckpoint = translogPath.resolve(Translog.CHECKPOINT_FILE_NAME);
-                Path tempEmptyTranslog = translogPath.resolve("temp-" + Translog.TRANSLOG_FILE_PREFIX +
-                    translogGeneration + Translog.TRANSLOG_FILE_SUFFIX);
-                Path realEmptyTranslog = translogPath.resolve(Translog.TRANSLOG_FILE_PREFIX +
-                    translogGeneration + Translog.TRANSLOG_FILE_SUFFIX);
-
-                // Write empty checkpoint and translog to empty files
-                long gen = Long.parseLong(translogGeneration);
-                int translogLen = writeEmptyTranslog(tempEmptyTranslog, translogUUID);
-                writeEmptyCheckpoint(tempEmptyCheckpoint, translogLen, gen, globalCheckpoint);
-
-                terminal.println("Removing existing translog files");
-                IOUtils.rm(translogFiles.toArray(new Path[]{}));
-
-                terminal.println("Creating new empty checkpoint at [" + realEmptyCheckpoint + "]");
-                Files.move(tempEmptyCheckpoint, realEmptyCheckpoint, StandardCopyOption.ATOMIC_MOVE);
-                terminal.println("Creating new empty translog at [" + realEmptyTranslog + "]");
-                Files.move(tempEmptyTranslog, realEmptyTranslog, StandardCopyOption.ATOMIC_MOVE);
-
-                // Fsync the translog directory after rename
-                IOUtils.fsync(translogPath, true);
-            }
-
-            terminal.println("Marking index with the new history uuid");
-            // commit the new histroy id
-            IndexWriterConfig iwc = new IndexWriterConfig(null)
-                .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
-                .setCommitOnClose(false)
-                // we don't want merges to happen here - we call maybe merge on the engine
-                // later once we stared it up otherwise we would need to wait for it here
-                // we also don't specify a codec here and merges should use the engines for this index
-                .setMergePolicy(NoMergePolicy.INSTANCE)
-                .setOpenMode(IndexWriterConfig.OpenMode.APPEND);
-            try (IndexWriter writer = new IndexWriter(dir, iwc)) {
-                Map<String, String> newCommitData = new HashMap<>(commitData);
-                newCommitData.put(Engine.HISTORY_UUID_KEY, historyUUID);
-                writer.setLiveCommitData(newCommitData.entrySet());
-                writer.commit();
-            }
-        } catch (LockObtainFailedException lofe) {
-            throw new ElasticsearchException("Failed to lock shard's directory at [" + idxLocation + "], is Elasticsearch still running?");
-        }
-
-        terminal.println("Done.");
-    }
-
-    /** Write a checkpoint file to the given location with the given generation */
-    static void writeEmptyCheckpoint(Path filename, int translogLength, long translogGeneration, long globalCheckpoint) throws IOException {
-        Checkpoint emptyCheckpoint = Checkpoint.emptyTranslogCheckpoint(translogLength, translogGeneration,
-            globalCheckpoint, translogGeneration);
-        Checkpoint.write(FileChannel::open, filename, emptyCheckpoint,
-            StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW);
-        // fsync with metadata here to make sure.
-        IOUtils.fsync(filename, false);
-    }
-
-    /**
-     * Write a translog containing the given translog UUID to the given location. Returns the number of bytes written.
-     */
-    public static int writeEmptyTranslog(Path filename, String translogUUID) throws IOException {
-        try (FileChannel fc = FileChannel.open(filename, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)) {
-            TranslogHeader header = new TranslogHeader(translogUUID, TranslogHeader.UNKNOWN_PRIMARY_TERM);
-            header.write(fc);
-            return header.sizeInBytes();
-        }
-    }
-
-    /** Show a warning about deleting files, asking for a confirmation if {@code batchMode} is false */
-    public static void warnAboutDeletingFiles(Terminal terminal, Set<Path> files, boolean batchMode) {
-        terminal.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
-        terminal.println("!   WARNING: Elasticsearch MUST be stopped before running this tool   !");
-        terminal.println("!                                                                     !");
-        terminal.println("!   WARNING:    Documents inside of translog files will be lost       !");
-        terminal.println("!                                                                     !");
-        terminal.println("!   WARNING:          The following files will be DELETED!            !");
-        terminal.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
-        for (Path file : files) {
-            terminal.println("--> " + file);
-        }
-        terminal.println("");
-        if (batchMode == false) {
-            String text = terminal.readText("Continue and DELETE files? [y/N] ");
-            if (!text.equalsIgnoreCase("y")) {
-                throw new ElasticsearchException("aborted by user");
-            }
-        }
-    }
-
-    /** Return a Set of all files in a given directory */
-    public static Set<Path> filesInDirectory(Path directory) throws IOException {
-        Set<Path> files = new HashSet<>();
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory)) {
-            for (Path file : stream) {
-                files.add(file);
-            }
-        }
-        return files;
-    }
-
-}

server/src/main/java/org/elasticsearch/search/profile/AbstractProfileBreakdown.java

@@ -49,6 +49,10 @@ public abstract class AbstractProfileBreakdown<T extends Enum<T>> {
         return timings[timing.ordinal()];
     }
 
+    public void setTimer(T timing, Timer timer) {
+        timings[timing.ordinal()] = timer;
+    }
+
     /** Convert this record to a map from timingType to times. */
     public Map<String, Long> toTimingMap() {
         Map<String, Long> map = new HashMap<>();

server/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java

@@ -19,7 +19,9 @@
 
 package org.elasticsearch.search.profile.query;
 
+import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
@@ -36,7 +38,10 @@ final class ProfileScorer extends Scorer {
 
     private final Scorer scorer;
     private ProfileWeight profileWeight;
+
     private final Timer scoreTimer, nextDocTimer, advanceTimer, matchTimer, shallowAdvanceTimer, computeMaxScoreTimer;
+    private final boolean isConstantScoreQuery;
+
 
     ProfileScorer(ProfileWeight w, Scorer scorer, QueryProfileBreakdown profile) throws IOException {
         super(w);
@@ -48,6 +53,26 @@ final class ProfileScorer extends Scorer {
         matchTimer = profile.getTimer(QueryTimingType.MATCH);
         shallowAdvanceTimer = profile.getTimer(QueryTimingType.SHALLOW_ADVANCE);
         computeMaxScoreTimer = profile.getTimer(QueryTimingType.COMPUTE_MAX_SCORE);
+        ProfileScorer profileScorer = null;
+        if (w.getQuery() instanceof ConstantScoreQuery && scorer instanceof ProfileScorer) {
+            //Case when we have a totalHits query and it is not cached
+            profileScorer = (ProfileScorer) scorer;
+        } else if (w.getQuery() instanceof ConstantScoreQuery && scorer.getChildren().size() == 1) {
+            //Case when we have a top N query. If the scorer has no children, it is because it is cached
+            //and in that case we do not do any special treatment
+            Scorable childScorer = scorer.getChildren().iterator().next().child;
+            if (childScorer instanceof ProfileScorer) {
+                profileScorer = (ProfileScorer) childScorer;
+            }
+        }
+        if (profileScorer != null) {
+            isConstantScoreQuery = true;
+            profile.setTimer(QueryTimingType.NEXT_DOC, profileScorer.nextDocTimer);
+            profile.setTimer(QueryTimingType.ADVANCE, profileScorer.advanceTimer);
+            profile.setTimer(QueryTimingType.MATCH, profileScorer.matchTimer);
+        } else {
+            isConstantScoreQuery = false;
+        }
     }
 
     @Override
@@ -77,6 +102,9 @@ final class ProfileScorer extends Scorer {
 
     @Override
     public DocIdSetIterator iterator() {
+        if (isConstantScoreQuery) {
+            return scorer.iterator();
+        }
         final DocIdSetIterator in = scorer.iterator();
         return new DocIdSetIterator() {
 
@@ -114,6 +142,9 @@ final class ProfileScorer extends Scorer {
 
     @Override
     public TwoPhaseIterator twoPhaseIterator() {
+        if (isConstantScoreQuery) {
+            return scorer.twoPhaseIterator();
+        }
         final TwoPhaseIterator in = scorer.twoPhaseIterator();
         if (in == null) {
             return null;

server/src/test/java/org/elasticsearch/index/IndexModuleTests.java

@@ -59,6 +59,7 @@ import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.similarity.NonNegativeScoresSimilarity;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.indices.IndicesModule;
@@ -77,6 +78,7 @@ import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.test.engine.MockEngineFactory;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -295,10 +297,13 @@ public class IndexModuleTests extends ESTestCase {
 
         IndexService indexService = newIndexService(module);
         SimilarityService similarityService = indexService.similarityService();
-        assertNotNull(similarityService.getSimilarity("my_similarity"));
-        assertTrue(similarityService.getSimilarity("my_similarity").get() instanceof TestSimilarity);
+        Similarity similarity = similarityService.getSimilarity("my_similarity").get();
+        assertNotNull(similarity);
+        assertThat(similarity, Matchers.instanceOf(NonNegativeScoresSimilarity.class));
+        similarity = ((NonNegativeScoresSimilarity) similarity).getDelegate();
+        assertThat(similarity, Matchers.instanceOf(TestSimilarity.class));
         assertEquals("my_similarity", similarityService.getSimilarity("my_similarity").name());
-        assertEquals("there is a key", ((TestSimilarity) similarityService.getSimilarity("my_similarity").get()).key);
+        assertEquals("there is a key", ((TestSimilarity) similarity).key);
         indexService.close("simon says", false);
     }
 

server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java

@@ -23,7 +23,6 @@ import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
@@ -2439,7 +2438,7 @@ public class IndexShardTests extends IndexShardTestCase {
         closeShards(sourceShard, targetShard);
     }
 
-    public void testDocStats() throws IOException, InterruptedException {
+    public void testDocStats() throws Exception {
         IndexShard indexShard = null;
         try {
             indexShard = newStartedShard(
@@ -2456,7 +2455,14 @@ public class IndexShardTests extends IndexShardTestCase {
                 indexShard.flush(new FlushRequest());
             }
             {
+                IndexShard shard = indexShard;
+                assertBusy(() -> {
+                    ThreadPool threadPool = shard.getThreadPool();
+                    assertThat(threadPool.relativeTimeInMillis(), greaterThan(shard.getLastSearcherAccess()));
+                });
+                long prevAccessTime = shard.getLastSearcherAccess();
                 final DocsStats docsStats = indexShard.docStats();
+                assertThat("searcher was not marked as accessed", shard.getLastSearcherAccess(), greaterThan(prevAccessTime));
                 assertThat(docsStats.getCount(), equalTo(numDocs));
                 try (Engine.Searcher searcher = indexShard.acquireSearcher("test")) {
                     assertTrue(searcher.reader().numDocs() <= docsStats.getCount());
@@ -2641,7 +2647,8 @@ public class IndexShardTests extends IndexShardTestCase {
 
         final ShardPath shardPath = indexShard.shardPath();
 
-        final Path indexPath = corruptIndexFile(shardPath);
+        final Path indexPath = shardPath.getDataPath().resolve(ShardPath.INDEX_FOLDER_NAME);
+        CorruptionUtils.corruptIndex(random(), indexPath, false);
 
         final AtomicInteger corruptedMarkerCount = new AtomicInteger();
         final SimpleFileVisitor<Path> corruptedVisitor = new SimpleFileVisitor<Path>() {
@@ -2750,22 +2757,6 @@ public class IndexShardTests extends IndexShardTestCase {
         assertThat("store still has a single corrupt marker", corruptedMarkerCount.get(), equalTo(1));
     }
 
-    private Path corruptIndexFile(ShardPath shardPath) throws IOException {
-        final Path indexPath = shardPath.getDataPath().resolve(ShardPath.INDEX_FOLDER_NAME);
-        final Path[] filesToCorrupt =
-            Files.walk(indexPath)
-                .filter(p -> {
-                    final String name = p.getFileName().toString();
-                    return Files.isRegularFile(p)
-                        && name.startsWith("extra") == false // Skip files added by Lucene's ExtrasFS
-                        && IndexWriter.WRITE_LOCK_NAME.equals(name) == false
-                        && name.startsWith("segments_") == false && name.endsWith(".si") == false;
-                })
-                .toArray(Path[]::new);
-        CorruptionUtils.corruptFile(random(), filesToCorrupt);
-        return indexPath;
-    }
-
     /**
      * Simulates a scenario that happens when we are async fetching snapshot metadata from GatewayService
      * and checking index concurrently. This should always be possible without any exception.
@@ -3428,4 +3419,9 @@ public class IndexShardTests extends IndexShardTestCase {
         assertThat(shard.translogStats().estimatedNumberOfOperations(), equalTo(translogStats.estimatedNumberOfOperations()));
         closeShard(shard, false);
     }
+
+    @Override
+    public Settings threadPoolSettings() {
+        return Settings.builder().put(super.threadPoolSettings()).put("thread_pool.estimated_time_interval", "5ms").build();
+    }
 }

server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java

@@ -0,0 +1,652 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.shard;
+
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.NativeFSLockFactory;
+import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.MergePolicyConfig;
+import org.elasticsearch.index.MockEngineFactoryPlugin;
+import org.elasticsearch.index.seqno.SeqNoStats;
+import org.elasticsearch.index.translog.TestTranslog;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.monitor.fs.FsInfo;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.CorruptionUtils;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.engine.MockEngineSupport;
+import org.elasticsearch.test.transport.MockTransportService;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.startsWith;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0)
+public class RemoveCorruptedShardDataCommandIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(MockTransportService.TestPlugin.class, MockEngineFactoryPlugin.class, InternalSettingsPlugin.class);
+    }
+
+    public void testCorruptIndex() throws Exception {
+        final String node = internalCluster().startNode();
+
+        final String indexName = "index42";
+        assertAcked(prepareCreate(indexName).setSettings(Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")
+            .put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true)
+            .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), "checksum")
+        ));
+
+        // index some docs in several segments
+        int numDocs = 0;
+        for (int k = 0, attempts = randomIntBetween(5, 10); k < attempts; k++) {
+            final int numExtraDocs = between(10, 100);
+            IndexRequestBuilder[] builders = new IndexRequestBuilder[numExtraDocs];
+            for (int i = 0; i < builders.length; i++) {
+                builders[i] = client().prepareIndex(indexName, "type").setSource("foo", "bar");
+            }
+
+            numDocs += numExtraDocs;
+
+            indexRandom(false, false, false, Arrays.asList(builders));
+            flush(indexName);
+        }
+
+        logger.info("--> indexed {} docs", numDocs);
+
+        final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
+        final MockTerminal terminal = new MockTerminal();
+        final OptionParser parser = command.getParser();
+
+        final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
+        final OptionSet options = parser.parse("-index", indexName, "-shard-id", "0");
+
+        // Try running it before the node is stopped (and shard is closed)
+        try {
+            command.execute(terminal, options, environment);
+            fail("expected the command to fail as node is locked");
+        } catch (Exception e) {
+            assertThat(e.getMessage(),
+                allOf(containsString("Failed to lock node's directory"),
+                    containsString("is Elasticsearch still running ?")));
+        }
+
+        final Set<Path> indexDirs = getDirs(indexName, ShardPath.INDEX_FOLDER_NAME);
+        assertThat(indexDirs, hasSize(1));
+
+        internalCluster().restartNode(node, new InternalTestCluster.RestartCallback() {
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                // Try running it before the shard is corrupted, it should flip out because there is no corruption file marker
+                try {
+                    command.execute(terminal, options, environment);
+                    fail("expected the command to fail as there is no corruption file marker");
+                } catch (Exception e) {
+                    assertThat(e.getMessage(), startsWith("Shard does not seem to be corrupted at"));
+                }
+
+                CorruptionUtils.corruptIndex(random(), indexDirs.iterator().next(), false);
+                return super.onNodeStopped(nodeName);
+            }
+        });
+
+        // shard should be failed due to a corrupted index
+        assertBusy(() -> {
+            final ClusterAllocationExplanation explanation =
+                client().admin().cluster().prepareAllocationExplain()
+                    .setIndex(indexName).setShard(0).setPrimary(true)
+                    .get().getExplanation();
+
+            final ShardAllocationDecision shardAllocationDecision = explanation.getShardAllocationDecision();
+            assertThat(shardAllocationDecision.isDecisionTaken(), equalTo(true));
+            assertThat(shardAllocationDecision.getAllocateDecision().getAllocationDecision(),
+                equalTo(AllocationDecision.NO_VALID_SHARD_COPY));
+        });
+
+        internalCluster().restartNode(node, new InternalTestCluster.RestartCallback() {
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                terminal.addTextInput("y");
+                command.execute(terminal, options, environment);
+
+                return super.onNodeStopped(nodeName);
+            }
+        });
+
+        waitNoPendingTasksOnAll();
+
+        String nodeId = null;
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final DiscoveryNodes nodes = state.nodes();
+        for (ObjectObjectCursor<String, DiscoveryNode> cursor : nodes.getNodes()) {
+            final String name = cursor.value.getName();
+            if (name.equals(node)) {
+                nodeId = cursor.key;
+                break;
+            }
+        }
+        assertThat(nodeId, notNullValue());
+
+        logger.info("--> output:\n{}", terminal.getOutput());
+
+        assertThat(terminal.getOutput(), containsString("allocate_stale_primary"));
+        assertThat(terminal.getOutput(), containsString("\"node\" : \"" + nodeId + "\""));
+
+        // there is only _stale_ primary (due to new allocation id)
+        assertBusy(() -> {
+            final ClusterAllocationExplanation explanation =
+                client().admin().cluster().prepareAllocationExplain()
+                    .setIndex(indexName).setShard(0).setPrimary(true)
+                    .get().getExplanation();
+
+            final ShardAllocationDecision shardAllocationDecision = explanation.getShardAllocationDecision();
+            assertThat(shardAllocationDecision.isDecisionTaken(), equalTo(true));
+            assertThat(shardAllocationDecision.getAllocateDecision().getAllocationDecision(),
+                equalTo(AllocationDecision.NO_VALID_SHARD_COPY));
+        });
+
+        client().admin().cluster().prepareReroute()
+            .add(new AllocateStalePrimaryAllocationCommand(indexName, 0, nodeId, true))
+            .get();
+
+        assertBusy(() -> {
+            final ClusterAllocationExplanation explanation =
+                client().admin().cluster().prepareAllocationExplain()
+                    .setIndex(indexName).setShard(0).setPrimary(true)
+                    .get().getExplanation();
+
+            assertThat(explanation.getCurrentNode(), notNullValue());
+            assertThat(explanation.getShardState(), equalTo(ShardRoutingState.STARTED));
+        });
+
+        final Pattern pattern = Pattern.compile("Corrupted Lucene index segments found -\\s+(?<docs>\\d+) documents will be lost.");
+        final Matcher matcher = pattern.matcher(terminal.getOutput());
+        assertThat(matcher.find(), equalTo(true));
+        final int expectedNumDocs = numDocs - Integer.parseInt(matcher.group("docs"));
+
+        ensureGreen(indexName);
+
+        assertHitCount(client().prepareSearch(indexName).setQuery(matchAllQuery()).get(), expectedNumDocs);
+    }
+
+    public void testCorruptTranslogTruncation() throws Exception {
+        internalCluster().startNodes(2, Settings.EMPTY);
+
+        final String node1 = internalCluster().getNodeNames()[0];
+        final String node2 = internalCluster().getNodeNames()[1];
+
+        final String indexName = "test";
+        assertAcked(prepareCreate(indexName).setSettings(Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")
+            .put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true) // never flush - always recover from translog
+            .put("index.routing.allocation.exclude._name", node2)
+        ));
+        ensureYellow();
+
+        assertAcked(client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder()
+            .put("index.routing.allocation.exclude._name", (String)null)
+        ));
+        ensureGreen();
+
+        // Index some documents
+        int numDocsToKeep = randomIntBetween(10, 100);
+        logger.info("--> indexing [{}] docs to be kept", numDocsToKeep);
+        IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocsToKeep];
+        for (int i = 0; i < builders.length; i++) {
+            builders[i] = client().prepareIndex(indexName, "type").setSource("foo", "bar");
+        }
+        indexRandom(false, false, false, Arrays.asList(builders));
+        flush(indexName);
+
+        disableTranslogFlush(indexName);
+        // having no extra docs is an interesting case for seq no based recoveries - test it more often
+        int numDocsToTruncate = randomBoolean() ? 0 : randomIntBetween(0, 100);
+        logger.info("--> indexing [{}] more doc to be truncated", numDocsToTruncate);
+        builders = new IndexRequestBuilder[numDocsToTruncate];
+        for (int i = 0; i < builders.length; i++) {
+            builders[i] = client().prepareIndex(indexName, "type").setSource("foo", "bar");
+        }
+        indexRandom(false, false, false, Arrays.asList(builders));
+        Set<Path> translogDirs = getDirs(indexName, ShardPath.TRANSLOG_FOLDER_NAME);
+
+        // that's only for 6.x branch for bwc with elasticsearch-translog
+        final boolean translogOnly = randomBoolean();
+        final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand(translogOnly);
+        final MockTerminal terminal = new MockTerminal();
+        final OptionParser parser = command.getParser();
+
+        if (randomBoolean() && numDocsToTruncate > 0) {
+            // flush the replica, so it will have more docs than what the primary will have
+            Index index = resolveIndex(indexName);
+            IndexShard replica = internalCluster().getInstance(IndicesService.class, node2).getShardOrNull(new ShardId(index, 0));
+            replica.flush(new FlushRequest());
+            logger.info("--> performed extra flushing on replica");
+        }
+
+        // shut down the replica node to be tested later
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node2));
+
+        // Corrupt the translog file(s)
+        logger.info("--> corrupting translog");
+        corruptRandomTranslogFiles(indexName);
+
+        // Restart the single node
+        logger.info("--> restarting node");
+        internalCluster().restartRandomDataNode();
+
+        // all shards should be failed due to a corrupted translog
+        assertBusy(() -> {
+            final ClusterAllocationExplanation explanation =
+                client().admin().cluster().prepareAllocationExplain()
+                    .setIndex(indexName).setShard(0).setPrimary(true)
+                    .get().getExplanation();
+
+            final UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+            assertThat(unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
+        });
+
+        // have to shut down primary node - otherwise node lock is present
+        final InternalTestCluster.RestartCallback callback =
+            new InternalTestCluster.RestartCallback() {
+                @Override
+                public Settings onNodeStopped(String nodeName) throws Exception {
+                    // and we can actually truncate the translog
+                    for (Path translogDir : translogDirs) {
+                        final Path idxLocation = translogDir.getParent().resolve(ShardPath.INDEX_FOLDER_NAME);
+                        assertBusy(() -> {
+                            logger.info("--> checking that lock has been released for {}", idxLocation);
+                            try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE);
+                                 Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
+                                // Great, do nothing, we just wanted to obtain the lock
+                            } catch (LockObtainFailedException lofe) {
+                                logger.info("--> failed acquiring lock for {}", idxLocation);
+                                fail("still waiting for lock release at [" + idxLocation + "]");
+                            } catch (IOException ioe) {
+                                fail("Got an IOException: " + ioe);
+                            }
+                        });
+
+                        final Settings defaultSettings = internalCluster().getDefaultSettings();
+                        final Environment environment = TestEnvironment.newEnvironment(defaultSettings);
+
+                        terminal.addTextInput("y");
+                        OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString());
+                        logger.info("--> running command for [{}]", translogDir.toAbsolutePath());
+                        command.execute(terminal, options, environment);
+                        logger.info("--> output:\n{}", terminal.getOutput());
+                    }
+
+                    return super.onNodeStopped(nodeName);
+                }
+            };
+        internalCluster().restartNode(node1, callback);
+
+        String primaryNodeId = null;
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final DiscoveryNodes nodes = state.nodes();
+        for (ObjectObjectCursor<String, DiscoveryNode> cursor : nodes.getNodes()) {
+            final String name = cursor.value.getName();
+            if (name.equals(node1)) {
+                primaryNodeId = cursor.key;
+                break;
+            }
+        }
+        assertThat(primaryNodeId, notNullValue());
+
+        assertThat(terminal.getOutput(), containsString("allocate_stale_primary"));
+        assertThat(terminal.getOutput(), containsString("\"node\" : \"" + primaryNodeId + "\""));
+
+        // there is only _stale_ primary (due to new allocation id)
+        assertBusy(() -> {
+            final ClusterAllocationExplanation explanation =
+                client().admin().cluster().prepareAllocationExplain()
+                    .setIndex(indexName).setShard(0).setPrimary(true)
+                    .get().getExplanation();
+
+            final ShardAllocationDecision shardAllocationDecision = explanation.getShardAllocationDecision();
+            assertThat(shardAllocationDecision.isDecisionTaken(), equalTo(true));
+            assertThat(shardAllocationDecision.getAllocateDecision().getAllocationDecision(),
+                equalTo(AllocationDecision.NO_VALID_SHARD_COPY));
+        });
+
+        client().admin().cluster().prepareReroute()
+            .add(new AllocateStalePrimaryAllocationCommand(indexName, 0, primaryNodeId, true))
+            .get();
+
+        assertBusy(() -> {
+            final ClusterAllocationExplanation explanation =
+                client().admin().cluster().prepareAllocationExplain()
+                    .setIndex(indexName).setShard(0).setPrimary(true)
+                    .get().getExplanation();
+
+            assertThat(explanation.getCurrentNode(), notNullValue());
+            assertThat(explanation.getShardState(), equalTo(ShardRoutingState.STARTED));
+        });
+
+        ensureYellow(indexName);
+
+        // Run a search and make sure it succeeds
+        assertHitCount(client().prepareSearch(indexName).setQuery(matchAllQuery()).get(), numDocsToKeep);
+
+        logger.info("--> starting the replica node to test recovery");
+        internalCluster().startNode();
+        ensureGreen(indexName);
+        for (String node : internalCluster().nodesInclude(indexName)) {
+            SearchRequestBuilder q = client().prepareSearch(indexName).setPreference("_only_nodes:" + node).setQuery(matchAllQuery());
+            assertHitCount(q.get(), numDocsToKeep);
+        }
+        final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).setActiveOnly(false).get();
+        final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get(indexName).stream()
+            .filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get();
+        assertThat(replicaRecoveryState.getIndex().toString(), replicaRecoveryState.getIndex().recoveredFileCount(), greaterThan(0));
+        // Ensure that the global checkpoint and local checkpoint are restored from the max seqno of the last commit.
+        final SeqNoStats seqNoStats = getSeqNoStats(indexName, 0);
+        assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
+        assertThat(seqNoStats.getLocalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
+    }
+
+    public void testCorruptTranslogTruncationOfReplica() throws Exception {
+        internalCluster().startNodes(2, Settings.EMPTY);
+
+        final String node1 = internalCluster().getNodeNames()[0];
+        final String node2 = internalCluster().getNodeNames()[1];
+        logger.info("--> nodes name: {}, {}", node1, node2);
+
+        final String indexName = "test";
+        assertAcked(prepareCreate(indexName).setSettings(Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")
+            .put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true) // never flush - always recover from translog
+            .put("index.routing.allocation.exclude._name", node2)
+        ));
+        ensureYellow();
+
+        assertAcked(client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder()
+            .put("index.routing.allocation.exclude._name", (String)null)
+        ));
+        ensureGreen();
+
+        // Index some documents
+        int numDocsToKeep = randomIntBetween(0, 100);
+        logger.info("--> indexing [{}] docs to be kept", numDocsToKeep);
+        IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocsToKeep];
+        for (int i = 0; i < builders.length; i++) {
+            builders[i] = client().prepareIndex(indexName, "type").setSource("foo", "bar");
+        }
+        indexRandom(false, false, false, Arrays.asList(builders));
+        flush(indexName);
+        disableTranslogFlush(indexName);
+        // having no extra docs is an interesting case for seq no based recoveries - test it more often
+        int numDocsToTruncate = randomBoolean() ? 0 : randomIntBetween(0, 100);
+        logger.info("--> indexing [{}] more docs to be truncated", numDocsToTruncate);
+        builders = new IndexRequestBuilder[numDocsToTruncate];
+        for (int i = 0; i < builders.length; i++) {
+            builders[i] = client().prepareIndex(indexName, "type").setSource("foo", "bar");
+        }
+        indexRandom(false, false, false, Arrays.asList(builders));
+        final int totalDocs = numDocsToKeep + numDocsToTruncate;
+
+        // sample the replica node translog dirs
+        final ShardId shardId = new ShardId(resolveIndex(indexName), 0);
+        final Set<Path> translogDirs = getDirs(node2, shardId, ShardPath.TRANSLOG_FOLDER_NAME);
+
+        // stop the cluster nodes. we don't use full restart so the node start up order will be the same
+        // and shard roles will be maintained
+        internalCluster().stopRandomDataNode();
+        internalCluster().stopRandomDataNode();
+
+        // Corrupt the translog file(s)
+        logger.info("--> corrupting translog");
+        TestTranslog.corruptRandomTranslogFile(logger, random(), translogDirs);
+
+        // Restart the single node
+        logger.info("--> starting node");
+        internalCluster().startNode();
+
+        ensureYellow();
+
+        // Run a search and make sure it succeeds
+        assertHitCount(client().prepareSearch(indexName).setQuery(matchAllQuery()).get(), totalDocs);
+
+        final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
+        final MockTerminal terminal = new MockTerminal();
+        final OptionParser parser = command.getParser();
+
+        final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
+
+        internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                logger.info("--> node {} stopped", nodeName);
+                for (Path translogDir : translogDirs) {
+                    final Path idxLocation = translogDir.getParent().resolve(ShardPath.INDEX_FOLDER_NAME);
+                    assertBusy(() -> {
+                        logger.info("--> checking that lock has been released for {}", idxLocation);
+                        try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE);
+                             Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
+                            // Great, do nothing, we just wanted to obtain the lock
+                        }  catch (LockObtainFailedException lofe) {
+                            logger.info("--> failed acquiring lock for {}", idxLocation);
+                            fail("still waiting for lock release at [" + idxLocation + "]");
+                        } catch (IOException ioe) {
+                            fail("Got an IOException: " + ioe);
+                        }
+                    });
+
+                    terminal.addTextInput("y");
+                    OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString());
+                    logger.info("--> running command for [{}]", translogDir.toAbsolutePath());
+                    command.execute(terminal, options, environment);
+                    logger.info("--> output:\n{}", terminal.getOutput());
+                }
+
+                return super.onNodeStopped(nodeName);
+            }
+        });
+
+        logger.info("--> starting the replica node to test recovery");
+        internalCluster().startNode();
+        ensureGreen(indexName);
+        for (String node : internalCluster().nodesInclude(indexName)) {
+            assertHitCount(client().prepareSearch(indexName)
+                .setPreference("_only_nodes:" + node).setQuery(matchAllQuery()).get(), totalDocs);
+        }
+
+        final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).setActiveOnly(false).get();
+        final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get(indexName).stream()
+            .filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get();
+        // the replica translog was disabled so it doesn't know what hte global checkpoint is and thus can't do ops based recovery
+        assertThat(replicaRecoveryState.getIndex().toString(), replicaRecoveryState.getIndex().recoveredFileCount(), greaterThan(0));
+        // Ensure that the global checkpoint and local checkpoint are restored from the max seqno of the last commit.
+        final SeqNoStats seqNoStats = getSeqNoStats(indexName, 0);
+        assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
+        assertThat(seqNoStats.getLocalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
+    }
+
+    public void testResolvePath() throws Exception {
+        final int numOfNodes = randomIntBetween(1, 5);
+        final List<String> nodeNames = internalCluster().startNodes(numOfNodes, Settings.EMPTY);
+
+        final String indexName = "test" + randomInt(100);
+        assertAcked(prepareCreate(indexName).setSettings(Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numOfNodes - 1)
+        ));
+        flush(indexName);
+
+        ensureGreen(indexName);
+
+        final Map<String, String> nodeNameToNodeId = new HashMap<>();
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final DiscoveryNodes nodes = state.nodes();
+        for (ObjectObjectCursor<String, DiscoveryNode> cursor : nodes.getNodes()) {
+            nodeNameToNodeId.put(cursor.value.getName(), cursor.key);
+        }
+
+        final GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{indexName}, false);
+        final List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
+        final ShardRouting shardRouting = iterators.iterator().next().nextOrNull();
+        assertThat(shardRouting, notNullValue());
+        final ShardId shardId = shardRouting.shardId();
+
+        final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
+        final OptionParser parser = command.getParser();
+
+        final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
+
+        final Map<String, Path> indexPathByNodeName = new HashMap<>();
+        for (String nodeName : nodeNames) {
+            final String nodeId = nodeNameToNodeId.get(nodeName);
+            final Set<Path> indexDirs = getDirs(nodeId, shardId, ShardPath.INDEX_FOLDER_NAME);
+            assertThat(indexDirs, hasSize(1));
+            indexPathByNodeName.put(nodeName, indexDirs.iterator().next());
+
+            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeName));
+            logger.info(" -- stopped {}", nodeName);
+        }
+
+        for (String nodeName : nodeNames) {
+            final Path indexPath = indexPathByNodeName.get(nodeName);
+            final OptionSet options = parser.parse("--dir", indexPath.toAbsolutePath().toString());
+            command.findAndProcessShardPath(options, environment,
+                shardPath -> assertThat(shardPath.resolveIndex(), equalTo(indexPath)));
+        }
+    }
+
+    private Set<Path> getDirs(String indexName, String dirSuffix) {
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{indexName}, false);
+        List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
+        ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
+        ShardRouting shardRouting = shardIterator.nextOrNull();
+        assertNotNull(shardRouting);
+        assertTrue(shardRouting.primary());
+        assertTrue(shardRouting.assignedToNode());
+        String nodeId = shardRouting.currentNodeId();
+        ShardId shardId = shardRouting.shardId();
+        return getDirs(nodeId, shardId, dirSuffix);
+    }
+
+    private Set<Path> getDirs(String nodeId, ShardId shardId, String dirSuffix) {
+        final NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
+        final Set<Path> translogDirs = new TreeSet<>();
+        final NodeStats nodeStats = nodeStatses.getNodes().get(0);
+        for (FsInfo.Path fsPath : nodeStats.getFs()) {
+            final String path = fsPath.getPath();
+            final Path p = PathUtils.get(path)
+                .resolve(NodeEnvironment.INDICES_FOLDER)
+                .resolve(shardId.getIndex().getUUID())
+                .resolve(Integer.toString(shardId.getId()))
+                .resolve(dirSuffix);
+            if (Files.isDirectory(p)) {
+                translogDirs.add(p);
+            }
+        }
+        return translogDirs;
+    }
+
+    private void corruptRandomTranslogFiles(String indexName) throws IOException {
+        Set<Path> translogDirs = getDirs(indexName, ShardPath.TRANSLOG_FOLDER_NAME);
+        TestTranslog.corruptRandomTranslogFile(logger, random(), translogDirs);
+    }
+
+    /** Disables translog flushing for the specified index */
+    private static void disableTranslogFlush(String index) {
+        Settings settings = Settings.builder()
+            .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB))
+            .build();
+        client().admin().indices().prepareUpdateSettings(index).setSettings(settings).get();
+    }
+
+    private SeqNoStats getSeqNoStats(String index, int shardId) {
+        final ShardStats[] shardStats = client().admin().indices()
+            .prepareStats(index).get()
+            .getIndices().get(index).getShards();
+        return shardStats[shardId].getSeqNoStats();
+    }
+}

server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java

@@ -0,0 +1,409 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.shard;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingHelper;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.MergePolicyConfig;
+import org.elasticsearch.index.engine.EngineException;
+import org.elasticsearch.index.engine.InternalEngineFactory;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.translog.TestTranslog;
+import org.elasticsearch.index.translog.TranslogCorruptedException;
+import org.elasticsearch.test.CorruptionUtils;
+import org.elasticsearch.test.DummyShardLock;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
+
+public class RemoveCorruptedShardDataCommandTests extends IndexShardTestCase {
+
+    private ShardId shardId;
+    private ShardRouting routing;
+    private Path dataDir;
+    private Environment environment;
+    private Settings settings;
+    private ShardPath shardPath;
+    private IndexMetaData indexMetaData;
+    private IndexShard indexShard;
+    private Path translogPath;
+    private Path indexPath;
+
+    @Before
+    public void setup() throws IOException {
+        shardId = new ShardId("index0", "_na_", 0);
+        final String nodeId = randomAlphaOfLength(10);
+        routing = TestShardRouting.newShardRouting(shardId, nodeId, true, ShardRoutingState.INITIALIZING,
+            RecoverySource.EmptyStoreRecoverySource.INSTANCE);
+
+        dataDir = createTempDir();
+
+        environment =
+            TestEnvironment.newEnvironment(Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), dataDir)
+                .putList(Environment.PATH_DATA_SETTING.getKey(), dataDir.toAbsolutePath().toString()).build());
+
+        // create same directory structure as prod does
+        final Path path = NodeEnvironment.resolveNodePath(dataDir, 0);
+        Files.createDirectories(path);
+        settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+
+        final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(path);
+        shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
+        final IndexMetaData.Builder metaData = IndexMetaData.builder(routing.getIndexName())
+            .settings(settings)
+            .primaryTerm(0, randomIntBetween(1, 100))
+            .putMapping("_doc", "{ \"properties\": {} }");
+        indexMetaData = metaData.build();
+
+        indexShard = newStartedShard(p ->
+                newShard(routing, shardPath, indexMetaData, null, null,
+                    new InternalEngineFactory(), () -> {
+                    }, EMPTY_EVENT_LISTENER),
+            true);
+
+        translogPath = shardPath.resolveTranslog();
+        indexPath = shardPath.resolveIndex();
+    }
+
+    public void testShardLock() throws Exception {
+        indexDocs(indexShard, true);
+
+        final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
+        final MockTerminal t = new MockTerminal();
+        final OptionParser parser = command.getParser();
+
+        // Try running it before the shard is closed, it should flip out because it can't acquire the lock
+        try {
+            final OptionSet options = parser.parse("-d", indexPath.toString());
+            command.execute(t, options, environment);
+            fail("expected the command to fail not being able to acquire the lock");
+        } catch (Exception e) {
+            assertThat(e.getMessage(), containsString("Failed to lock shard's directory"));
+        }
+
+        // close shard
+        closeShards(indexShard);
+
+        // Try running it before the shard is corrupted
+        try {
+            final OptionSet options = parser.parse("-d", indexPath.toString());
+            command.execute(t, options, environment);
+            fail("expected the command to fail not being able to find a corrupt file marker");
+        } catch (ElasticsearchException e) {
+            assertThat(e.getMessage(), startsWith("Shard does not seem to be corrupted at"));
+            assertThat(t.getOutput(), containsString("Lucene index is clean at"));
+        }
+    }
+
+    public void testCorruptedIndex() throws Exception {
+        final int numDocs = indexDocs(indexShard, true);
+
+        // close shard
+        closeShards(indexShard);
+
+        final boolean corruptSegments = randomBoolean();
+        CorruptionUtils.corruptIndex(random(), indexPath, corruptSegments);
+
+        // test corrupted shard
+        final IndexShard corruptedShard = reopenIndexShard(true);
+        allowShardFailures();
+        expectThrows(IndexShardRecoveryException.class, () -> newStartedShard(p -> corruptedShard, true));
+        closeShards(corruptedShard);
+
+        final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
+        final MockTerminal t = new MockTerminal();
+        final OptionParser parser = command.getParser();
+
+        // run command with dry-run
+        t.addTextInput("n"); // mean dry run
+        final OptionSet options = parser.parse("-d", indexPath.toString());
+        t.setVerbosity(Terminal.Verbosity.VERBOSE);
+        try {
+            command.execute(t, options, environment);
+            fail();
+        } catch (ElasticsearchException e) {
+            if (corruptSegments) {
+                assertThat(e.getMessage(), is("Index is unrecoverable"));
+            } else {
+                assertThat(e.getMessage(), containsString("aborted by user"));
+            }
+        }
+
+        logger.info("--> output:\n{}", t.getOutput());
+
+        if (corruptSegments == false) {
+
+            // run command without dry-run
+            t.addTextInput("y");
+            command.execute(t, options, environment);
+
+            final String output = t.getOutput();
+            logger.info("--> output:\n{}", output);
+
+            // reopen shard
+            failOnShardFailures();
+            final IndexShard newShard = newStartedShard(p -> reopenIndexShard(false), true);
+
+            final Set<String> shardDocUIDs = getShardDocUIDs(newShard);
+
+            final Pattern pattern = Pattern.compile("Corrupted Lucene index segments found -\\s+(?<docs>\\d+) documents will be lost.");
+            final Matcher matcher = pattern.matcher(output);
+            assertThat(matcher.find(), equalTo(true));
+            final int expectedNumDocs = numDocs - Integer.parseInt(matcher.group("docs"));
+
+            assertThat(shardDocUIDs.size(), equalTo(expectedNumDocs));
+
+            closeShards(newShard);
+        }
+    }
+
+    public void testCorruptedTranslog() throws Exception {
+        final int numDocsToKeep = indexDocs(indexShard, false);
+
+        // close shard
+        closeShards(indexShard);
+
+        TestTranslog.corruptRandomTranslogFile(logger, random(), Arrays.asList(translogPath));
+
+        // test corrupted shard
+        final IndexShard corruptedShard = reopenIndexShard(true);
+
+        allowShardFailures();
+        // it has to fail on start up due to index.shard.check_on_startup = checksum
+        final Exception exception = expectThrows(Exception.class, () -> newStartedShard(p -> corruptedShard, true));
+        final Throwable cause = exception.getCause() instanceof EngineException ? exception.getCause().getCause() : exception.getCause();
+        assertThat(cause, instanceOf(TranslogCorruptedException.class));
+
+        closeShards(corruptedShard);
+
+        final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
+        final MockTerminal t = new MockTerminal();
+        final OptionParser parser = command.getParser();
+
+        final OptionSet options = parser.parse("-d", translogPath.toString());
+        // run command with dry-run
+        t.addTextInput("n"); // mean dry run
+        t.setVerbosity(Terminal.Verbosity.VERBOSE);
+        try {
+            command.execute(t, options, environment);
+            fail();
+        } catch (ElasticsearchException e) {
+            assertThat(e.getMessage(), containsString("aborted by user"));
+            assertThat(t.getOutput(), containsString("Continue and remove corrupted data from the shard ?"));
+        }
+
+        logger.info("--> output:\n{}", t.getOutput());
+
+        // run command without dry-run
+        t.reset();
+        t.addTextInput("y");
+        command.execute(t, options, environment);
+
+        final String output = t.getOutput();
+        logger.info("--> output:\n{}", output);
+
+        // reopen shard
+        failOnShardFailures();
+        final IndexShard newShard = newStartedShard(p -> reopenIndexShard(false), true);
+
+        final Set<String> shardDocUIDs = getShardDocUIDs(newShard);
+
+        assertThat(shardDocUIDs.size(), equalTo(numDocsToKeep));
+
+        closeShards(newShard);
+    }
+
+    public void testCorruptedBothIndexAndTranslog() throws Exception {
+        // index some docs in several segments
+        final int numDocsToKeep = indexDocs(indexShard, false);
+
+        // close shard
+        closeShards(indexShard);
+
+        CorruptionUtils.corruptIndex(random(), indexPath, false);
+
+        // test corrupted shard
+        final IndexShard corruptedShard = reopenIndexShard(true);
+        allowShardFailures();
+        expectThrows(IndexShardRecoveryException.class, () -> newStartedShard(p -> corruptedShard, true));
+        closeShards(corruptedShard);
+
+        TestTranslog.corruptRandomTranslogFile(logger, random(), Arrays.asList(translogPath));
+
+        final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
+        final MockTerminal t = new MockTerminal();
+        final OptionParser parser = command.getParser();
+
+        final OptionSet options = parser.parse("-d", translogPath.toString());
+        // run command with dry-run
+        t.addTextInput("n"); // mean dry run
+        t.addTextInput("n"); // mean dry run
+        t.setVerbosity(Terminal.Verbosity.VERBOSE);
+        try {
+            command.execute(t, options, environment);
+            fail();
+        } catch (ElasticsearchException e) {
+            assertThat(e.getMessage(), containsString("aborted by user"));
+            assertThat(t.getOutput(), containsString("Continue and remove corrupted data from the shard ?"));
+        }
+
+        logger.info("--> output:\n{}", t.getOutput());
+
+        // run command without dry-run
+        t.reset();
+        t.addTextInput("y");
+        command.execute(t, options, environment);
+
+        final String output = t.getOutput();
+        logger.info("--> output:\n{}", output);
+
+        // reopen shard
+        failOnShardFailures();
+        final IndexShard newShard = newStartedShard(p -> reopenIndexShard(false), true);
+
+        final Set<String> shardDocUIDs = getShardDocUIDs(newShard);
+
+        final Pattern pattern = Pattern.compile("Corrupted Lucene index segments found -\\s+(?<docs>\\d+) documents will be lost.");
+        final Matcher matcher = pattern.matcher(output);
+        assertThat(matcher.find(), equalTo(true));
+        final int expectedNumDocs = numDocsToKeep - Integer.parseInt(matcher.group("docs"));
+
+        assertThat(shardDocUIDs.size(), equalTo(expectedNumDocs));
+
+        closeShards(newShard);
+    }
+
+    public void testResolveIndexDirectory() throws Exception {
+        // index a single doc to have files on a disk
+        indexDoc(indexShard, "_doc", "0", "{}");
+        flushShard(indexShard, true);
+        writeIndexState();
+
+        // close shard
+        closeShards(indexShard);
+
+        final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
+        final OptionParser parser = command.getParser();
+
+        // `--index index_name --shard-id 0` has to be resolved to indexPath
+        final OptionSet options = parser.parse("--index", shardId.getIndex().getName(),
+            "--shard-id", Integer.toString(shardId.id()));
+
+        command.findAndProcessShardPath(options, environment,
+            shardPath -> assertThat(shardPath.resolveIndex(), equalTo(indexPath)));
+
+        final OptionSet options2 = parser.parse("--dir", indexPath.toAbsolutePath().toString());
+        command.findAndProcessShardPath(options2, environment,
+            shardPath -> assertThat(shardPath.resolveIndex(), equalTo(indexPath)));
+    }
+
+    private IndexShard reopenIndexShard(boolean corrupted) throws IOException {
+        // open shard with the same location
+        final ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(indexShard.routingEntry(),
+            RecoverySource.ExistingStoreRecoverySource.INSTANCE
+        );
+
+        final IndexMetaData metaData = IndexMetaData.builder(indexMetaData)
+            .settings(Settings.builder()
+                .put(indexShard.indexSettings().getSettings())
+                .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), "checksum"))
+            .build();
+
+        CheckedFunction<IndexSettings, Store, IOException> storeProvider =
+            corrupted == false ? null :
+                indexSettings -> {
+                    final ShardId shardId = shardPath.getShardId();
+                    final BaseDirectoryWrapper baseDirectoryWrapper = newFSDirectory(shardPath.resolveIndex());
+                    // index is corrupted - don't even try to check index on close - it fails
+                    baseDirectoryWrapper.setCheckIndexOnClose(false);
+                    return new Store(shardId, indexSettings, baseDirectoryWrapper, new DummyShardLock(shardId));
+        };
+
+        return newShard(shardRouting, shardPath, metaData, storeProvider, null,
+            indexShard.engineFactory, indexShard.getGlobalCheckpointSyncer(), EMPTY_EVENT_LISTENER);
+    }
+
+    private int indexDocs(IndexShard indexShard, boolean flushLast) throws IOException {
+        // index some docs in several segments
+        int numDocs = 0;
+        int numDocsToKeep = 0;
+        for (int i = 0, attempts = randomIntBetween(5, 10); i < attempts; i++) {
+            final int numExtraDocs = between(10, 100);
+            for (long j = 0; j < numExtraDocs; j++) {
+                indexDoc(indexShard, "_doc", Long.toString(numDocs + j), "{}");
+            }
+            numDocs += numExtraDocs;
+
+            if (flushLast || i < attempts - 1) {
+                numDocsToKeep += numExtraDocs;
+                flushShard(indexShard, true);
+            }
+        }
+
+        logger.info("--> indexed {} docs, {} to keep", numDocs, numDocsToKeep);
+
+        writeIndexState();
+        return numDocsToKeep;
+    }
+
+    private void writeIndexState() throws IOException {
+        // create _state of IndexMetaData
+        try(NodeEnvironment nodeEnvironment = new NodeEnvironment(environment.settings(), environment, nId -> {})) {
+            final Path[] paths = nodeEnvironment.indexPaths(indexMetaData.getIndex());
+            IndexMetaData.FORMAT.write(indexMetaData, paths);
+            logger.info("--> index metadata persisted to {} ", Arrays.toString(paths));
+        }
+    }
+
+}

server/src/test/java/org/elasticsearch/index/similarity/NonNegativeScoresSimilarityTests.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.similarity;
+
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.Similarity.SimScorer;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+public class NonNegativeScoresSimilarityTests extends ESTestCase {
+
+    public void testBasics() {
+        Similarity negativeScoresSim = new Similarity() {
+
+            @Override
+            public long computeNorm(FieldInvertState state) {
+                return state.getLength();
+            }
+
+            @Override
+            public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+                return new SimScorer() {
+                    @Override
+                    public float score(float freq, long norm) {
+                        return freq - 5;
+                    }
+                };
+            }
+        };
+        Similarity assertingSimilarity = new NonNegativeScoresSimilarity(negativeScoresSim);
+        SimScorer scorer = assertingSimilarity.scorer(1f, null);
+        assertEquals(2f, scorer.score(7f, 1L), 0f);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> scorer.score(2f, 1L));
+        assertThat(e.getMessage(), Matchers.containsString("Similarities must not produce negative scores"));
+    }
+
+}

server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java

@@ -18,12 +18,18 @@
  */
 package org.elasticsearch.index.similarity;
 
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.similarities.BM25Similarity;
 import org.apache.lucene.search.similarities.BooleanSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.hamcrest.Matchers;
 
 import java.util.Collections;
 
@@ -56,4 +62,76 @@ public class SimilarityServiceTests extends ESTestCase {
         SimilarityService service = new SimilarityService(indexSettings, null, Collections.emptyMap());
         assertTrue(service.getDefaultSimilarity() instanceof BooleanSimilarity);
     }
+
+    public void testSimilarityValidation() {
+        Similarity negativeScoresSim = new Similarity() {
+
+            @Override
+            public long computeNorm(FieldInvertState state) {
+                return state.getLength();
+            }
+
+            @Override
+            public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+                return new SimScorer() {
+
+                    @Override
+                    public float score(float freq, long norm) {
+                        return -1;
+                    }
+
+                };
+            }
+        };
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> SimilarityService.validateSimilarity(Version.V_7_0_0_alpha1, negativeScoresSim));
+        assertThat(e.getMessage(), Matchers.containsString("Similarities should not return negative scores"));
+
+        Similarity decreasingScoresWithFreqSim = new Similarity() {
+
+            @Override
+            public long computeNorm(FieldInvertState state) {
+                return state.getLength();
+            }
+
+            @Override
+            public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+                return new SimScorer() {
+
+                    @Override
+                    public float score(float freq, long norm) {
+                        return 1 / (freq + norm);
+                    }
+
+                };
+            }
+        };
+        e = expectThrows(IllegalArgumentException.class,
+                () -> SimilarityService.validateSimilarity(Version.V_7_0_0_alpha1, decreasingScoresWithFreqSim));
+        assertThat(e.getMessage(), Matchers.containsString("Similarity scores should not decrease when term frequency increases"));
+
+        Similarity increasingScoresWithNormSim = new Similarity() {
+
+            @Override
+            public long computeNorm(FieldInvertState state) {
+                return state.getLength();
+            }
+
+            @Override
+            public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+                return new SimScorer() {
+
+                    @Override
+                    public float score(float freq, long norm) {
+                        return freq + norm;
+                    }
+
+                };
+            }
+        };
+        e = expectThrows(IllegalArgumentException.class,
+                () -> SimilarityService.validateSimilarity(Version.V_7_0_0_alpha1, increasingScoresWithNormSim));
+        assertThat(e.getMessage(), Matchers.containsString("Similarity scores should not increase when norm increases"));
+    }
+
 }

server/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java

@@ -125,7 +125,7 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
             }
         }
         Path translogDir = RandomPicks.randomFrom(random(), translogDirs);
-        TestTranslog.corruptRandomTranslogFile(logger, random(), translogDir, TestTranslog.minTranslogGenUsedInRecovery(translogDir));
+        TestTranslog.corruptRandomTranslogFile(logger, random(), Arrays.asList(translogDir));
     }
 
     /** Disables translog flushing for the specified index */

server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java

@@ -34,6 +34,7 @@ import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.Collection;
 import java.util.List;
 import java.util.Random;
 import java.util.Set;
@@ -52,13 +53,19 @@ import static org.hamcrest.core.IsNot.not;
 public class TestTranslog {
     static final Pattern TRANSLOG_FILE_PATTERN = Pattern.compile("translog-(\\d+)\\.tlog");
 
+    public static void corruptRandomTranslogFile(Logger logger, Random random, Collection<Path> translogDirs) throws IOException {
+        for (Path translogDir : translogDirs) {
+            final long minTranslogGen = minTranslogGenUsedInRecovery(translogDir);
+            corruptRandomTranslogFile(logger, random, translogDir, minTranslogGen);
+        }
+    }
+
     /**
      * Corrupts random translog file (translog-N.tlog) from the given translog directory.
      *
      * @return a translog file which has been corrupted.
      */
-    public static Path corruptRandomTranslogFile(Logger logger, Random random, Path translogDir, long minGeneration) throws
-            IOException {
+    public static Path corruptRandomTranslogFile(Logger logger, Random random, Path translogDir, long minGeneration) throws IOException {
         Set<Path> candidates = new TreeSet<>(); // TreeSet makes sure iteration order is deterministic
         logger.info("--> Translog dir [{}], minUsedTranslogGen [{}]", translogDir, minGeneration);
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(translogDir)) {

server/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java

@@ -1,382 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.translog;
-
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import joptsimple.OptionParser;
-import joptsimple.OptionSet;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.NativeFSLockFactory;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
-import org.elasticsearch.action.admin.indices.flush.FlushRequest;
-import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
-import org.elasticsearch.action.admin.indices.stats.ShardStats;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.cli.MockTerminal;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
-import org.elasticsearch.cluster.routing.ShardIterator;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.MockEngineFactoryPlugin;
-import org.elasticsearch.index.seqno.SeqNoStats;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.monitor.fs.FsInfo;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalTestCluster;
-import org.elasticsearch.test.engine.MockEngineSupport;
-import org.elasticsearch.test.transport.MockTransportService;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
-
-import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0)
-public class TruncateTranslogIT extends ESIntegTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(MockTransportService.TestPlugin.class, MockEngineFactoryPlugin.class);
-    }
-
-    public void testCorruptTranslogTruncation() throws Exception {
-        internalCluster().startNodes(2, Settings.EMPTY);
-
-        final String replicaNode = internalCluster().getNodeNames()[1];
-
-            assertAcked(prepareCreate("test").setSettings(Settings.builder()
-            .put("index.number_of_shards", 1)
-            .put("index.number_of_replicas", 1)
-            .put("index.refresh_interval", "-1")
-            .put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true) // never flush - always recover from translog
-            .put("index.routing.allocation.exclude._name", replicaNode)
-        ));
-        ensureYellow();
-
-        assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
-            .put("index.routing.allocation.exclude._name", (String)null)
-        ));
-        ensureGreen();
-
-        // Index some documents
-        int numDocsToKeep = randomIntBetween(0, 100);
-        logger.info("--> indexing [{}] docs to be kept", numDocsToKeep);
-        IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocsToKeep];
-        for (int i = 0; i < builders.length; i++) {
-            builders[i] = client().prepareIndex("test", "type").setSource("foo", "bar");
-        }
-        indexRandom(false, false, false, Arrays.asList(builders));
-        flush("test");
-        disableTranslogFlush("test");
-        // having no extra docs is an interesting case for seq no based recoveries - test it more often
-        int numDocsToTruncate = randomBoolean() ? 0 : randomIntBetween(0, 100);
-        logger.info("--> indexing [{}] more doc to be truncated", numDocsToTruncate);
-        builders = new IndexRequestBuilder[numDocsToTruncate];
-        for (int i = 0; i < builders.length; i++) {
-            builders[i] = client().prepareIndex("test", "type").setSource("foo", "bar");
-        }
-        indexRandom(false, false, false, Arrays.asList(builders));
-        Set<Path> translogDirs = getTranslogDirs("test");
-
-        TruncateTranslogCommand ttc = new TruncateTranslogCommand();
-        MockTerminal t = new MockTerminal();
-        OptionParser parser = ttc.getParser();
-
-        for (Path translogDir : translogDirs) {
-            OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString(), "-b");
-            // Try running it before the shard is closed, it should flip out because it can't acquire the lock
-            try {
-                logger.info("--> running truncate while index is open on [{}]", translogDir.toAbsolutePath());
-                ttc.execute(t, options, null /* TODO: env should be real here, and ttc should actually use it... */);
-                fail("expected the truncate command to fail not being able to acquire the lock");
-            } catch (Exception e) {
-                assertThat(e.getMessage(), containsString("Failed to lock shard's directory"));
-            }
-        }
-
-        if (randomBoolean() && numDocsToTruncate > 0) {
-            // flush the replica, so it will have more docs than what the primary will have
-            Index index = resolveIndex("test");
-            IndexShard replica = internalCluster().getInstance(IndicesService.class, replicaNode).getShardOrNull(new ShardId(index, 0));
-            replica.flush(new FlushRequest());
-            logger.info("--> performed extra flushing on replica");
-        }
-
-        // shut down the replica node to be tested later
-        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode));
-
-        // Corrupt the translog file
-        logger.info("--> corrupting translog");
-        corruptRandomTranslogFile("test");
-
-        // Restart the single node
-        logger.info("--> restarting node");
-        internalCluster().restartRandomDataNode();
-        client().admin().cluster().prepareHealth().setWaitForYellowStatus()
-            .setTimeout(new TimeValue(1000, TimeUnit.MILLISECONDS))
-            .setWaitForEvents(Priority.LANGUID)
-            .get();
-
-        try {
-            client().prepareSearch("test").setQuery(matchAllQuery()).get();
-            fail("all shards should be failed due to a corrupted translog");
-        } catch (SearchPhaseExecutionException e) {
-            // Good, all shards should be failed because there is only a
-            // single shard and its translog is corrupt
-        }
-
-        // Close the index so we can actually truncate the translog
-        logger.info("--> closing 'test' index");
-        client().admin().indices().prepareClose("test").get();
-
-        for (Path translogDir : translogDirs) {
-            final Path idxLocation = translogDir.getParent().resolve("index");
-            assertBusy(() -> {
-                logger.info("--> checking that lock has been released for {}", idxLocation);
-                try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE);
-                     Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
-                    // Great, do nothing, we just wanted to obtain the lock
-                }  catch (LockObtainFailedException lofe) {
-                    logger.info("--> failed acquiring lock for {}", idxLocation);
-                    fail("still waiting for lock release at [" + idxLocation + "]");
-                } catch (IOException ioe) {
-                    fail("Got an IOException: " + ioe);
-                }
-            });
-
-            OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString(), "-b");
-            logger.info("--> running truncate translog command for [{}]", translogDir.toAbsolutePath());
-            ttc.execute(t, options, null /* TODO: env should be real here, and ttc should actually use it... */);
-            logger.info("--> output:\n{}", t.getOutput());
-        }
-
-        // Re-open index
-        logger.info("--> opening 'test' index");
-        client().admin().indices().prepareOpen("test").get();
-        ensureYellow("test");
-
-        // Run a search and make sure it succeeds
-        assertHitCount(client().prepareSearch("test").setQuery(matchAllQuery()).get(), numDocsToKeep);
-
-        logger.info("--> starting the replica node to test recovery");
-        internalCluster().startNode();
-        ensureGreen("test");
-        for (String node : internalCluster().nodesInclude("test")) {
-            SearchRequestBuilder q = client().prepareSearch("test").setPreference("_only_nodes:" + node).setQuery(matchAllQuery());
-            assertHitCount(q.get(), numDocsToKeep);
-        }
-        final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").setActiveOnly(false).get();
-        final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get("test").stream()
-            .filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get();
-        assertThat(replicaRecoveryState.getIndex().toString(), replicaRecoveryState.getIndex().recoveredFileCount(), greaterThan(0));
-        // Ensure that the global checkpoint and local checkpoint are restored from the max seqno of the last commit.
-        final SeqNoStats seqNoStats = getSeqNoStats("test", 0);
-        assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
-        assertThat(seqNoStats.getLocalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
-    }
-
-    public void testCorruptTranslogTruncationOfReplica() throws Exception {
-        internalCluster().startNodes(2, Settings.EMPTY);
-
-        final String primaryNode = internalCluster().getNodeNames()[0];
-        final String replicaNode = internalCluster().getNodeNames()[1];
-
-        assertAcked(prepareCreate("test").setSettings(Settings.builder()
-            .put("index.number_of_shards", 1)
-            .put("index.number_of_replicas", 1)
-            .put("index.refresh_interval", "-1")
-            .put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true) // never flush - always recover from translog
-            .put("index.routing.allocation.exclude._name", replicaNode)
-        ));
-        ensureYellow();
-
-        assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
-            .put("index.routing.allocation.exclude._name", (String)null)
-        ));
-        ensureGreen();
-
-        // Index some documents
-        int numDocsToKeep = randomIntBetween(0, 100);
-        logger.info("--> indexing [{}] docs to be kept", numDocsToKeep);
-        IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocsToKeep];
-        for (int i = 0; i < builders.length; i++) {
-            builders[i] = client().prepareIndex("test", "type").setSource("foo", "bar");
-        }
-        indexRandom(false, false, false, Arrays.asList(builders));
-        flush("test");
-        disableTranslogFlush("test");
-        // having no extra docs is an interesting case for seq no based recoveries - test it more often
-        int numDocsToTruncate = randomBoolean() ? 0 : randomIntBetween(0, 100);
-        logger.info("--> indexing [{}] more docs to be truncated", numDocsToTruncate);
-        builders = new IndexRequestBuilder[numDocsToTruncate];
-        for (int i = 0; i < builders.length; i++) {
-            builders[i] = client().prepareIndex("test", "type").setSource("foo", "bar");
-        }
-        indexRandom(false, false, false, Arrays.asList(builders));
-        final int totalDocs = numDocsToKeep + numDocsToTruncate;
-
-
-        // sample the replica node translog dirs
-        final ShardId shardId = new ShardId(resolveIndex("test"), 0);
-        Set<Path> translogDirs = getTranslogDirs(replicaNode, shardId);
-        Path tdir = randomFrom(translogDirs);
-
-        // stop the cluster nodes. we don't use full restart so the node start up order will be the same
-        // and shard roles will be maintained
-        internalCluster().stopRandomDataNode();
-        internalCluster().stopRandomDataNode();
-
-        // Corrupt the translog file
-        logger.info("--> corrupting translog");
-        TestTranslog.corruptRandomTranslogFile(logger, random(), tdir, TestTranslog.minTranslogGenUsedInRecovery(tdir));
-
-        // Restart the single node
-        logger.info("--> starting node");
-        internalCluster().startNode();
-
-        ensureYellow();
-
-        // Run a search and make sure it succeeds
-        assertHitCount(client().prepareSearch("test").setQuery(matchAllQuery()).get(), totalDocs);
-
-        TruncateTranslogCommand ttc = new TruncateTranslogCommand();
-        MockTerminal t = new MockTerminal();
-        OptionParser parser = ttc.getParser();
-
-        for (Path translogDir : translogDirs) {
-            final Path idxLocation = translogDir.getParent().resolve("index");
-            assertBusy(() -> {
-                logger.info("--> checking that lock has been released for {}", idxLocation);
-                try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE);
-                     Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
-                    // Great, do nothing, we just wanted to obtain the lock
-                }  catch (LockObtainFailedException lofe) {
-                    logger.info("--> failed acquiring lock for {}", idxLocation);
-                    fail("still waiting for lock release at [" + idxLocation + "]");
-                } catch (IOException ioe) {
-                    fail("Got an IOException: " + ioe);
-                }
-            });
-
-            OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString(), "-b");
-            logger.info("--> running truncate translog command for [{}]", translogDir.toAbsolutePath());
-            ttc.execute(t, options, null /* TODO: env should be real here, and ttc should actually use it... */);
-            logger.info("--> output:\n{}", t.getOutput());
-        }
-
-        logger.info("--> starting the replica node to test recovery");
-        internalCluster().startNode();
-        ensureGreen("test");
-        for (String node : internalCluster().nodesInclude("test")) {
-            assertHitCount(client().prepareSearch("test").setPreference("_only_nodes:" + node).setQuery(matchAllQuery()).get(), totalDocs);
-        }
-
-        final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").setActiveOnly(false).get();
-        final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get("test").stream()
-            .filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get();
-        // the replica translog was disabled so it doesn't know what hte global checkpoint is and thus can't do ops based recovery
-        assertThat(replicaRecoveryState.getIndex().toString(), replicaRecoveryState.getIndex().recoveredFileCount(), greaterThan(0));
-        // Ensure that the global checkpoint and local checkpoint are restored from the max seqno of the last commit.
-        final SeqNoStats seqNoStats = getSeqNoStats("test", 0);
-        assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
-        assertThat(seqNoStats.getLocalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
-    }
-
-    private Set<Path> getTranslogDirs(String indexName) throws IOException {
-        ClusterState state = client().admin().cluster().prepareState().get().getState();
-        GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{indexName}, false);
-        List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
-        ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
-        ShardRouting shardRouting = shardIterator.nextOrNull();
-        assertNotNull(shardRouting);
-        assertTrue(shardRouting.primary());
-        assertTrue(shardRouting.assignedToNode());
-        String nodeId = shardRouting.currentNodeId();
-        ShardId shardId = shardRouting.shardId();
-        return getTranslogDirs(nodeId, shardId);
-    }
-
-    private Set<Path> getTranslogDirs(String nodeId, ShardId shardId) {
-        NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
-        Set<Path> translogDirs = new TreeSet<>(); // treeset makes sure iteration order is deterministic
-        for (FsInfo.Path fsPath : nodeStatses.getNodes().get(0).getFs()) {
-            String path = fsPath.getPath();
-            final String relativeDataLocationPath =  "indices/"+ shardId.getIndex().getUUID() +"/" + Integer.toString(shardId.getId())
-                + "/translog";
-            Path translogPath = PathUtils.get(path).resolve(relativeDataLocationPath);
-            if (Files.isDirectory(translogPath)) {
-                translogDirs.add(translogPath);
-            }
-        }
-        return translogDirs;
-    }
-
-    private void corruptRandomTranslogFile(String indexName) throws IOException {
-        Set<Path> translogDirs = getTranslogDirs(indexName);
-        Path translogDir = randomFrom(translogDirs);
-        TestTranslog.corruptRandomTranslogFile(logger, random(), translogDir, TestTranslog.minTranslogGenUsedInRecovery(translogDir));
-    }
-
-    /** Disables translog flushing for the specified index */
-    private static void disableTranslogFlush(String index) {
-        Settings settings = Settings.builder()
-                .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB))
-                .build();
-        client().admin().indices().prepareUpdateSettings(index).setSettings(settings).get();
-    }
-
-    private SeqNoStats getSeqNoStats(String index, int shardId) {
-        final ShardStats[] shardStats = client().admin().indices()
-            .prepareStats(index).get()
-            .getIndices().get(index).getShards();
-        return shardStats[shardId].getSeqNoStats();
-    }
-}

server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.indices;
 
 import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
@@ -56,6 +57,7 @@ import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.index.similarity.NonNegativeScoresSimilarity;
 import org.elasticsearch.indices.IndicesService.ShardDeletionCheckResult;
 import org.elasticsearch.plugins.EnginePlugin;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -448,8 +450,10 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
             .build();
         MapperService mapperService = indicesService.createIndexMapperService(indexMetaData);
         assertNotNull(mapperService.documentMapperParser().parserContext("type").typeParser("fake-mapper"));
-        assertThat(mapperService.documentMapperParser().parserContext("type").getSimilarity("test").get(),
-            instanceOf(BM25Similarity.class));
+        Similarity sim = mapperService.documentMapperParser().parserContext("type").getSimilarity("test").get();
+        assertThat(sim, instanceOf(NonNegativeScoresSimilarity.class));
+        sim = ((NonNegativeScoresSimilarity) sim).getDelegate();
+        assertThat(sim, instanceOf(BM25Similarity.class));
     }
 
     public void testStatsByShardDoesNotDieFromExpectedExceptions() {

server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java

@@ -28,10 +28,12 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.RandomApproximationQuery;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
@@ -118,6 +120,209 @@ public class QueryProfilerTests extends ESTestCase {
         assertThat(rewriteTime, greaterThan(0L));
     }
 
+    public void testConstantScoreQuery() throws IOException {
+        QueryProfiler profiler = new QueryProfiler();
+        searcher.setProfiler(profiler);
+        Query query = new ConstantScoreQuery(new TermQuery(new Term("foo", "bar")));
+        searcher.search(query, 1);
+        List<ProfileResult> results = profiler.getTree();
+        assertEquals(1, results.size());
+        Map<String, Long> breakdownConstantScoreQuery = results.get(0).getTimeBreakdown();
+        assertEquals(1, results.get(0).getProfiledChildren().size());
+        Map<String, Long> breakdownTermQuery = results.get(0).getProfiledChildren().get(0).getTimeBreakdown();
+
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
+
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
+
+        assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
+
+        assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
+
+        assertEquals(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(),
+            breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue());
+
+        long rewriteTime = profiler.getRewriteTime();
+        assertThat(rewriteTime, greaterThan(0L));
+    }
+
+    public void testConstantScoreTotalHitsBeingCachedQuery() throws IOException {
+        Query query =  new ConstantScoreQuery(new TermQuery(new Term("foo", "bar")));
+        //clean cache and make sure queries will be cached
+        searcher.setQueryCache(IndexSearcher.getDefaultQueryCache());
+        searcher.setQueryCachingPolicy(ALWAYS_CACHE_POLICY);
+
+        QueryProfiler profiler = new QueryProfiler();
+        searcher.setProfiler(profiler);
+        TotalHitCountCollector collector  = new TotalHitCountCollector();
+        searcher.search(query, collector);
+
+        List<ProfileResult> results = profiler.getTree();
+        assertEquals(1, results.size());
+        Map<String, Long> breakdownConstantScoreQuery = results.get(0).getTimeBreakdown();
+        assertEquals(1, results.get(0).getProfiledChildren().size());
+        Map<String, Long> breakdownTermQuery = results.get(0).getProfiledChildren().get(0).getTimeBreakdown();
+        //In this case scorers for constant score query and term query are disconnected.
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
+
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
+
+        assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
+
+        assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
+
+        long rewriteTime = profiler.getRewriteTime();
+        assertThat(rewriteTime, greaterThan(0L));
+    }
+
+    public void testConstantScoreTotalHitsNotCachedQuery() throws IOException {
+        Query query =  new ConstantScoreQuery(new TermQuery(new Term("foo", "bar")));
+
+        //clean cache and make sure queries will not be cached
+        searcher.setQueryCache(IndexSearcher.getDefaultQueryCache());
+        searcher.setQueryCachingPolicy(NEVER_CACHE_POLICY);
+
+        QueryProfiler profiler = new QueryProfiler();
+        searcher.setProfiler(profiler);
+        TotalHitCountCollector collector  = new TotalHitCountCollector();
+        searcher.search(query, collector);
+
+        List<ProfileResult> results = profiler.getTree();
+        assertEquals(1, results.size());
+        Map<String, Long> breakdownConstantScoreQuery = results.get(0).getTimeBreakdown();
+        assertEquals(1, results.get(0).getProfiledChildren().size());
+        Map<String, Long> breakdownTermQuery = results.get(0).getProfiledChildren().get(0).getTimeBreakdown();
+        //Timing from the scorer of term query are inherited by constant score query scorer.
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
+
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
+
+        assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
+
+        assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
+
+
+        assertEquals(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(),
+                breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue());
+
+
+        long rewriteTime = profiler.getRewriteTime();
+        assertThat(rewriteTime, greaterThan(0L));
+    }
+
+    public void testConstantScoreTotalHitsCachedQuery() throws IOException {
+        Query query =  new ConstantScoreQuery(new TermQuery(new Term("foo", "bar")));
+
+        //clean cache and make sure queries will be cached
+        searcher.setQueryCache(IndexSearcher.getDefaultQueryCache());
+        searcher.setQueryCachingPolicy(ALWAYS_CACHE_POLICY);
+        //Put query on cache
+        TotalHitCountCollector collector  = new TotalHitCountCollector();
+        searcher.search(query, collector);
+
+        QueryProfiler profiler = new QueryProfiler();
+        searcher.setProfiler(profiler);
+        collector  = new TotalHitCountCollector();
+        searcher.search(query, collector);
+
+        List<ProfileResult> results = profiler.getTree();
+        assertEquals(1, results.size());
+        Map<String, Long> breakdownConstantScoreQuery = results.get(0).getTimeBreakdown();
+        assertEquals(1, results.get(0).getProfiledChildren().size());
+        Map<String, Long> breakdownTermQuery = results.get(0).getProfiledChildren().get(0).getTimeBreakdown();
+
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
+
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownConstantScoreQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
+
+        assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
+
+        assertThat(breakdownTermQuery.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count").longValue(), greaterThan(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.BUILD_SCORER.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.NEXT_DOC.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.ADVANCE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.SCORE.toString() + "_count").longValue(), equalTo(0L));
+        assertThat(breakdownTermQuery.get(QueryTimingType.MATCH.toString() + "_count").longValue(), equalTo(0L));
+
+        long rewriteTime = profiler.getRewriteTime();
+        assertThat(rewriteTime, greaterThan(0L));
+    }
+
+
     public void testNoScoring() throws IOException {
         QueryProfiler profiler = new QueryProfiler();
         searcher.setProfiler(profiler);
@@ -276,4 +481,29 @@ public class QueryProfilerTests extends ESTestCase {
         reader.close();
         dir.close();
     }
+
+    private static final QueryCachingPolicy ALWAYS_CACHE_POLICY = new QueryCachingPolicy() {
+
+        @Override
+        public void onUse(Query query) {}
+
+        @Override
+        public boolean shouldCache(Query query) throws IOException {
+            return true;
+        }
+
+    };
+
+    private static final QueryCachingPolicy NEVER_CACHE_POLICY = new QueryCachingPolicy() {
+
+        @Override
+        public void onUse(Query query) {}
+
+        @Override
+        public boolean shouldCache(Query query) throws IOException {
+            return false;
+        }
+
+    };
+
 }

server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.transport;
 
-import java.util.function.Supplier;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
@@ -56,6 +55,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiFunction;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -578,7 +578,7 @@ public class RemoteClusterServiceTests extends ESTestCase {
                         remoteClusterService.collectSearchShards(IndicesOptions.lenientExpandOpen(), "index_not_found",
                                 null, remoteIndicesByCluster,
                                 new LatchedActionListener<>(ActionListener.wrap(response::set, failure::set), latch));
-                        assertTrue(latch.await(1, TimeUnit.SECONDS));
+                        assertTrue(latch.await(2, TimeUnit.SECONDS));
                         assertNull(response.get());
                         assertNotNull(failure.get());
                         assertThat(failure.get(), instanceOf(RemoteTransportException.class));

test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java

@@ -133,7 +133,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
         super.setUp();
         threadPool = new TestThreadPool(getClass().getName(), threadPoolSettings());
         primaryTerm = randomIntBetween(1, 100); // use random but fixed term for creating shards
-        failOnShardFailures.set(true);
+        failOnShardFailures();
     }
 
     @Override
@@ -154,6 +154,10 @@ public abstract class IndexShardTestCase extends ESTestCase {
         failOnShardFailures.set(false);
     }
 
+    protected void failOnShardFailures() {
+        failOnShardFailures.set(true);
+    }
+
     public Settings threadPoolSettings() {
         return Settings.EMPTY;
     }
@@ -233,7 +237,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
             .settings(indexSettings)
             .primaryTerm(0, primaryTerm)
             .putMapping("_doc", "{ \"properties\": {} }");
-        return newShard(shardRouting, metaData.build(), engineFactory, listeners);
+        return newShard(shardRouting, metaData.build(), null, engineFactory, () -> {}, listeners);
     }
 
     /**
@@ -279,7 +283,6 @@ public abstract class IndexShardTestCase extends ESTestCase {
         return newShard(shardRouting, indexMetaData, searcherWrapper, new InternalEngineFactory(), globalCheckpointSyncer);
     }
 
-
     /**
      * creates a new initializing shard. The shard will will be put in its proper path under the
      * current node id the shard is assigned to.

test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java

@@ -21,6 +21,7 @@ package org.elasticsearch.test;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
@@ -47,6 +48,23 @@ public final class CorruptionUtils {
     private static Logger logger = ESLoggerFactory.getLogger("test");
     private CorruptionUtils() {}
 
+    public static void corruptIndex(Random random, Path indexPath, boolean corruptSegments) throws IOException {
+        // corrupt files
+        final Path[] filesToCorrupt =
+            Files.walk(indexPath)
+                .filter(p -> {
+                    final String name = p.getFileName().toString();
+                    boolean segmentFile = name.startsWith("segments_") || name.endsWith(".si");
+                        return Files.isRegularFile(p)
+                            && name.startsWith("extra") == false // Skip files added by Lucene's ExtrasFS
+                            && IndexWriter.WRITE_LOCK_NAME.equals(name) == false
+                            && (corruptSegments ? segmentFile : segmentFile == false);
+                    }
+                )
+                .toArray(Path[]::new);
+        corruptFile(random, filesToCorrupt);
+    }
+
     /**
      * Corrupts a random file at a random position
      */

test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java

@@ -182,6 +182,7 @@ public class MockTcpTransport extends TcpTransport {
         executor.submit(() -> {
             try {
                 socket.connect(address);
+                socket.setSoLinger(false, 0);
                 channel.loopRead(executor);
                 connectListener.onResponse(null);
             } catch (Exception ex) {

test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java

@@ -193,6 +193,7 @@ public class MockNioTransport extends TcpTransport {
             BytesChannelContext context = new BytesChannelContext(nioChannel, selector, (e) -> exceptionCaught(nioChannel, e),
                 readWriteHandler, new InboundChannelBuffer(pageSupplier));
             nioChannel.setContext(context);
+            nioChannel.setSoLinger(0);
             return nioChannel;
         }
 

x-pack/plugin/ccr/qa/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java

@@ -199,13 +199,13 @@ public class FollowIndexSecurityIT extends ESRestTestCase {
 
     private static void followIndex(String leaderIndex, String followIndex) throws IOException {
         final Request request = new Request("POST", "/" + followIndex + "/_ccr/follow");
-        request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"idle_shard_retry_delay\": \"10ms\"}");
+        request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"poll_timeout\": \"10ms\"}");
         assertOK(client().performRequest(request));
     }
 
     private static void createAndFollowIndex(String leaderIndex, String followIndex) throws IOException {
         final Request request = new Request("POST", "/" + followIndex + "/_ccr/create_and_follow");
-        request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"idle_shard_retry_delay\": \"10ms\"}");
+        request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"poll_timeout\": \"10ms\"}");
         assertOK(client().performRequest(request));
     }
 

x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java

@@ -141,13 +141,13 @@ public class FollowIndexIT extends ESRestTestCase {
 
     private static void followIndex(String leaderIndex, String followIndex) throws IOException {
         final Request request = new Request("POST", "/" + followIndex + "/_ccr/follow");
-        request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"idle_shard_retry_delay\": \"10ms\"}");
+        request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"poll_timeout\": \"10ms\"}");
         assertOK(client().performRequest(request));
     }
 
     private static void createAndFollowIndex(String leaderIndex, String followIndex) throws IOException {
         final Request request = new Request("POST", "/" + followIndex + "/_ccr/create_and_follow");
-        request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"idle_shard_retry_delay\": \"10ms\"}");
+        request.setJsonEntity("{\"leader_index\": \"" + leaderIndex + "\", \"poll_timeout\": \"10ms\"}");
         assertOK(client().performRequest(request));
     }
 

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java

@@ -297,12 +297,16 @@ public class AutoFollowCoordinator implements ClusterStateApplier {
 
             String leaderIndexNameWithClusterAliasPrefix = clusterAlias.equals("_local_") ? leaderIndexName :
                 clusterAlias + ":" + leaderIndexName;
-            FollowIndexAction.Request request =
-                new FollowIndexAction.Request(leaderIndexNameWithClusterAliasPrefix, followIndexName,
-                    pattern.getMaxBatchOperationCount(), pattern.getMaxConcurrentReadBatches(),
-                    pattern.getMaxOperationSizeInBytes(), pattern.getMaxConcurrentWriteBatches(),
-                    pattern.getMaxWriteBufferSize(), pattern.getMaxRetryDelay(),
-                    pattern.getIdleShardRetryDelay());
+            FollowIndexAction.Request request = new FollowIndexAction.Request();
+            request.setLeaderIndex(leaderIndexNameWithClusterAliasPrefix);
+            request.setFollowerIndex(followIndexName);
+            request.setMaxBatchOperationCount(pattern.getMaxBatchOperationCount());
+            request.setMaxConcurrentReadBatches(pattern.getMaxConcurrentReadBatches());
+            request.setMaxOperationSizeInBytes(pattern.getMaxOperationSizeInBytes());
+            request.setMaxConcurrentWriteBatches(pattern.getMaxConcurrentWriteBatches());
+            request.setMaxWriteBufferSize(pattern.getMaxWriteBufferSize());
+            request.setMaxRetryDelay(pattern.getMaxRetryDelay());
+            request.setPollTimeout(pattern.getPollTimeout());
 
             // Execute if the create and follow api call succeeds:
             Runnable successHandler = () -> {

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java

@@ -32,7 +32,6 @@ import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ccr.action.FollowIndexAction;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -64,8 +63,8 @@ public class ShardChangesAction extends Action<ShardChangesAction.Response> {
         private int maxOperationCount;
         private ShardId shardId;
         private String expectedHistoryUUID;
-        private TimeValue pollTimeout = FollowIndexAction.DEFAULT_POLL_TIMEOUT;
-        private long maxOperationSizeInBytes = FollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES;
+        private TimeValue pollTimeout = TransportFollowIndexAction.DEFAULT_POLL_TIMEOUT;
+        private long maxOperationSizeInBytes = TransportFollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES;
 
         public Request(ShardId shardId, String expectedHistoryUUID) {
             super(shardId.getIndexName());

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowIndexAction.java

@@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexingSlowLog;
@@ -55,6 +56,14 @@ import java.util.stream.Collectors;
 
 public class TransportFollowIndexAction extends HandledTransportAction<FollowIndexAction.Request, AcknowledgedResponse> {
 
+    static final long DEFAULT_MAX_BATCH_SIZE_IN_BYTES = Long.MAX_VALUE;
+    private static final TimeValue DEFAULT_MAX_RETRY_DELAY = new TimeValue(500);
+    private static final int DEFAULT_MAX_CONCURRENT_WRITE_BATCHES = 1;
+    private static final int DEFAULT_MAX_WRITE_BUFFER_SIZE = 10240;
+    private static final int DEFAULT_MAX_BATCH_OPERATION_COUNT = 1024;
+    private static final int DEFAULT_MAX_CONCURRENT_READ_BATCHES = 1;
+    static final TimeValue DEFAULT_POLL_TIMEOUT = TimeValue.timeValueMinutes(1);
+
     private final Client client;
     private final ThreadPool threadPool;
     private final ClusterService clusterService;
@@ -179,19 +188,8 @@ public class TransportFollowIndexAction extends HandledTransportAction<FollowInd
             String[] recordedLeaderShardHistoryUUIDs = extractIndexShardHistoryUUIDs(ccrIndexMetadata);
             String recordedLeaderShardHistoryUUID = recordedLeaderShardHistoryUUIDs[shardId];
 
-            ShardFollowTask shardFollowTask = new ShardFollowTask(
-                    clusterNameAlias,
-                    new ShardId(followIndexMetadata.getIndex(), shardId),
-                    new ShardId(leaderIndexMetadata.getIndex(), shardId),
-                    request.getMaxBatchOperationCount(),
-                    request.getMaxConcurrentReadBatches(),
-                    request.getMaxOperationSizeInBytes(),
-                    request.getMaxConcurrentWriteBatches(),
-                    request.getMaxWriteBufferSize(),
-                    request.getMaxRetryDelay(),
-                    request.getPollTimeout(),
-                    recordedLeaderShardHistoryUUID,
-                    filteredHeaders);
+            final ShardFollowTask shardFollowTask =  createShardFollowTask(shardId, clusterNameAlias, request,
+                leaderIndexMetadata, followIndexMetadata, recordedLeaderShardHistoryUUID, filteredHeaders);
             persistentTasksService.sendStartRequest(taskId, ShardFollowTask.NAME, shardFollowTask,
                     new ActionListener<PersistentTasksCustomMetaData.PersistentTask<ShardFollowTask>>() {
                         @Override
@@ -299,6 +297,69 @@ public class TransportFollowIndexAction extends HandledTransportAction<FollowInd
         followerMapperService.merge(leaderIndex, MapperService.MergeReason.MAPPING_RECOVERY);
     }
 
+    private static ShardFollowTask createShardFollowTask(
+        int shardId,
+        String clusterAliasName,
+        FollowIndexAction.Request request,
+        IndexMetaData leaderIndexMetadata,
+        IndexMetaData followIndexMetadata,
+        String recordedLeaderShardHistoryUUID,
+        Map<String, String> filteredHeaders
+    ) {
+        int maxBatchOperationCount;
+        if (request.getMaxBatchOperationCount() != null) {
+            maxBatchOperationCount = request.getMaxBatchOperationCount();
+        } else {
+            maxBatchOperationCount = DEFAULT_MAX_BATCH_OPERATION_COUNT;
+        }
+
+        int maxConcurrentReadBatches;
+        if (request.getMaxConcurrentReadBatches() != null){
+            maxConcurrentReadBatches = request.getMaxConcurrentReadBatches();
+        } else {
+            maxConcurrentReadBatches = DEFAULT_MAX_CONCURRENT_READ_BATCHES;
+        }
+
+        long maxOperationSizeInBytes;
+        if (request.getMaxOperationSizeInBytes() != null) {
+            maxOperationSizeInBytes = request.getMaxOperationSizeInBytes();
+        } else {
+            maxOperationSizeInBytes = DEFAULT_MAX_BATCH_SIZE_IN_BYTES;
+        }
+
+        int maxConcurrentWriteBatches;
+        if (request.getMaxConcurrentWriteBatches() != null) {
+            maxConcurrentWriteBatches = request.getMaxConcurrentWriteBatches();
+        } else {
+            maxConcurrentWriteBatches = DEFAULT_MAX_CONCURRENT_WRITE_BATCHES;
+        }
+
+        int maxWriteBufferSize;
+        if (request.getMaxWriteBufferSize() != null) {
+            maxWriteBufferSize = request.getMaxWriteBufferSize();
+        } else {
+            maxWriteBufferSize = DEFAULT_MAX_WRITE_BUFFER_SIZE;
+        }
+
+        TimeValue maxRetryDelay = request.getMaxRetryDelay() == null ? DEFAULT_MAX_RETRY_DELAY : request.getMaxRetryDelay();
+        TimeValue pollTimeout = request.getPollTimeout() == null ? DEFAULT_POLL_TIMEOUT : request.getPollTimeout();
+
+        return new ShardFollowTask(
+            clusterAliasName,
+            new ShardId(followIndexMetadata.getIndex(), shardId),
+            new ShardId(leaderIndexMetadata.getIndex(), shardId),
+            maxBatchOperationCount,
+            maxConcurrentReadBatches,
+            maxOperationSizeInBytes,
+            maxConcurrentWriteBatches,
+            maxWriteBufferSize,
+            maxRetryDelay,
+            pollTimeout,
+            recordedLeaderShardHistoryUUID,
+            filteredHeaders
+        );
+    }
+
     private static String[] extractIndexShardHistoryUUIDs(Map<String, String> ccrIndexMetaData) {
         String historyUUIDs = ccrIndexMetaData.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS);
         return historyUUIDs.split(",");

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java

@@ -156,7 +156,7 @@ public class TransportPutAutoFollowPatternAction extends
             request.getMaxConcurrentWriteBatches(),
             request.getMaxWriteBufferSize(),
             request.getMaxRetryDelay(),
-            request.getIdleShardRetryDelay(),
+            request.getPollTimeout(),
             filteredHeaders);
         patterns.put(request.getLeaderClusterAlias(), autoFollowPattern);
         ClusterState.Builder newState = ClusterState.builder(localState);

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java

@@ -192,16 +192,12 @@ public class CcrLicenseIT extends ESSingleNodeTestCase {
     }
 
     private FollowIndexAction.Request getFollowRequest() {
-        return new FollowIndexAction.Request(
-                "leader",
-                "follower",
-                FollowIndexAction.DEFAULT_MAX_BATCH_OPERATION_COUNT,
-                FollowIndexAction.DEFAULT_MAX_CONCURRENT_READ_BATCHES,
-                FollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES,
-                FollowIndexAction.DEFAULT_MAX_CONCURRENT_WRITE_BATCHES,
-                FollowIndexAction.DEFAULT_MAX_WRITE_BUFFER_SIZE,
-                TimeValue.timeValueMillis(10),
-                TimeValue.timeValueMillis(10));
+        FollowIndexAction.Request request = new FollowIndexAction.Request();
+        request.setLeaderIndex("leader");
+        request.setFollowerIndex("follower");
+        request.setMaxRetryDelay(TimeValue.timeValueMillis(10));
+        request.setPollTimeout(TimeValue.timeValueMillis(10));
+        return request;
     }
 
 }

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/ShardChangesIT.java

@@ -319,9 +319,11 @@ public class ShardChangesIT extends ESIntegTestCase {
         long numDocsIndexed = Math.min(3000 * 2, randomLongBetween(maxReadSize, maxReadSize * 10));
         atLeastDocsIndexed("index1", numDocsIndexed / 3);
 
-        final FollowIndexAction.Request followRequest = new FollowIndexAction.Request("index1", "index2", maxReadSize,
-            randomIntBetween(2, 10), Long.MAX_VALUE, randomIntBetween(2, 10),
-            randomIntBetween(1024, 10240), TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
+        FollowIndexAction.Request followRequest = createFollowRequest("index1", "index2");
+        followRequest.setMaxBatchOperationCount(maxReadSize);
+        followRequest.setMaxConcurrentReadBatches(randomIntBetween(2, 10));
+        followRequest.setMaxConcurrentWriteBatches(randomIntBetween(2, 10));
+        followRequest.setMaxWriteBufferSize(randomIntBetween(1024, 10240));
         CreateAndFollowIndexAction.Request createAndFollowRequest = new CreateAndFollowIndexAction.Request(followRequest);
         client().execute(CreateAndFollowIndexAction.INSTANCE, createAndFollowRequest).get();
 
@@ -358,9 +360,10 @@ public class ShardChangesIT extends ESIntegTestCase {
         });
         thread.start();
 
-        final FollowIndexAction.Request followRequest = new FollowIndexAction.Request("index1", "index2", randomIntBetween(32, 2048),
-            randomIntBetween(2, 10), Long.MAX_VALUE, randomIntBetween(2, 10),
-            FollowIndexAction.DEFAULT_MAX_WRITE_BUFFER_SIZE, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
+        FollowIndexAction.Request followRequest = createFollowRequest("index1", "index2");
+        followRequest.setMaxBatchOperationCount(randomIntBetween(32, 2048));
+        followRequest.setMaxConcurrentReadBatches(randomIntBetween(2, 10));
+        followRequest.setMaxConcurrentWriteBatches(randomIntBetween(2, 10));
         client().execute(CreateAndFollowIndexAction.INSTANCE, new CreateAndFollowIndexAction.Request(followRequest)).get();
 
         long maxNumDocsReplicated = Math.min(1000, randomLongBetween(followRequest.getMaxBatchOperationCount(),
@@ -447,7 +450,7 @@ public class ShardChangesIT extends ESIntegTestCase {
                 .actionGet());
     }
 
-    public void testFollowIndex_lowMaxTranslogBytes() throws Exception {
+    public void testFollowIndexMaxOperationSizeInBytes() throws Exception {
         final String leaderIndexSettings = getIndexSettings(1, between(0, 1),
             singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true"));
         assertAcked(client().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON));
@@ -460,8 +463,8 @@ public class ShardChangesIT extends ESIntegTestCase {
             client().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get();
         }
 
-        final FollowIndexAction.Request followRequest = new FollowIndexAction.Request("index1", "index2", 1024, 1, 1024L,
-            1, 10240, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
+        FollowIndexAction.Request followRequest = createFollowRequest("index1", "index2");
+        followRequest.setMaxOperationSizeInBytes(1L);
         final CreateAndFollowIndexAction.Request createAndFollowRequest = new CreateAndFollowIndexAction.Request(followRequest);
         client().execute(CreateAndFollowIndexAction.INSTANCE, createAndFollowRequest).get();
 
@@ -489,25 +492,21 @@ public class ShardChangesIT extends ESIntegTestCase {
         assertAcked(client().admin().indices().prepareCreate("index3").setSource(leaderIndexSettings, XContentType.JSON));
         ensureGreen("index3");
 
-        FollowIndexAction.Request followRequest = new FollowIndexAction.Request("index1", "index2", 1024, 1, 1024L,
-            1, 10240, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
+        FollowIndexAction.Request followRequest = createFollowRequest("index1", "index2");
         CreateAndFollowIndexAction.Request createAndFollowRequest = new CreateAndFollowIndexAction.Request(followRequest);
         client().execute(CreateAndFollowIndexAction.INSTANCE, createAndFollowRequest).get();
 
-        followRequest = new FollowIndexAction.Request("index3", "index4", 1024, 1, 1024L,
-            1, 10240, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
+        followRequest = createFollowRequest("index3", "index4");
         createAndFollowRequest = new CreateAndFollowIndexAction.Request(followRequest);
         client().execute(CreateAndFollowIndexAction.INSTANCE, createAndFollowRequest).get();
         unfollowIndex("index2", "index4");
 
-        FollowIndexAction.Request wrongRequest1 = new FollowIndexAction.Request("index1", "index4", 1024, 1, 1024L,
-            1, 10240, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
+        FollowIndexAction.Request wrongRequest1 = createFollowRequest("index1", "index4");
         Exception e = expectThrows(IllegalArgumentException.class,
             () -> client().execute(FollowIndexAction.INSTANCE, wrongRequest1).actionGet());
         assertThat(e.getMessage(), containsString("follow index [index4] should reference"));
 
-        FollowIndexAction.Request wrongRequest2 = new FollowIndexAction.Request("index3", "index2", 1024, 1, 1024L,
-            1, 10240, TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(10));
+        FollowIndexAction.Request wrongRequest2 = createFollowRequest("index3", "index2");
         e = expectThrows(IllegalArgumentException.class, () -> client().execute(FollowIndexAction.INSTANCE, wrongRequest2).actionGet());
         assertThat(e.getMessage(), containsString("follow index [index2] should reference"));
     }
@@ -716,10 +715,12 @@ public class ShardChangesIT extends ESIntegTestCase {
         }, 60, TimeUnit.SECONDS);
     }
 
-    public static FollowIndexAction.Request createFollowRequest(String leaderIndex, String followIndex) {
-        return new FollowIndexAction.Request(leaderIndex, followIndex, FollowIndexAction.DEFAULT_MAX_BATCH_OPERATION_COUNT,
-            FollowIndexAction.DEFAULT_MAX_CONCURRENT_READ_BATCHES, FollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES,
-            FollowIndexAction.DEFAULT_MAX_CONCURRENT_WRITE_BATCHES, FollowIndexAction.DEFAULT_MAX_WRITE_BUFFER_SIZE,
-            TimeValue.timeValueMillis(10), TimeValue.timeValueMillis(10));
+    public static FollowIndexAction.Request createFollowRequest(String leaderIndex, String followerIndex) {
+        FollowIndexAction.Request request = new FollowIndexAction.Request();
+        request.setLeaderIndex(leaderIndex);
+        request.setFollowerIndex(followerIndex);
+        request.setMaxRetryDelay(TimeValue.timeValueMillis(10));
+        request.setPollTimeout(TimeValue.timeValueMillis(10));
+        return request;
     }
 }

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowTests.java

@@ -136,7 +136,7 @@ public class AutoFollowTests extends ESSingleNodeTestCase {
             request.setMaxRetryDelay(TimeValue.timeValueMillis(500));
         }
         if (randomBoolean()) {
-            request.setIdleShardRetryDelay(TimeValue.timeValueMillis(500));
+            request.setPollTimeout(TimeValue.timeValueMillis(500));
         }
         assertTrue(client().execute(PutAutoFollowPatternAction.INSTANCE, request).actionGet().isAcknowledged());
 
@@ -167,8 +167,8 @@ public class AutoFollowTests extends ESSingleNodeTestCase {
             if (request.getMaxRetryDelay() != null) {
                 assertThat(shardFollowTask.getMaxRetryDelay(), equalTo(request.getMaxRetryDelay()));
             }
-            if (request.getIdleShardRetryDelay() != null) {
-                assertThat(shardFollowTask.getPollTimeout(), equalTo(request.getIdleShardRetryDelay()));
+            if (request.getPollTimeout() != null) {
+                assertThat(shardFollowTask.getPollTimeout(), equalTo(request.getPollTimeout()));
             }
         });
     }

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowIndexRequestTests.java

@@ -40,24 +40,49 @@ public class FollowIndexRequestTests extends AbstractStreamableXContentTestCase<
     }
 
     static FollowIndexAction.Request createTestRequest() {
-        return new FollowIndexAction.Request(randomAlphaOfLength(4), randomAlphaOfLength(4), randomIntBetween(1, Integer.MAX_VALUE),
-            randomIntBetween(1, Integer.MAX_VALUE), randomNonNegativeLong(), randomIntBetween(1, Integer.MAX_VALUE),
-            randomIntBetween(1, Integer.MAX_VALUE), TimeValue.timeValueMillis(500), TimeValue.timeValueMillis(500));
+        FollowIndexAction.Request request = new FollowIndexAction.Request();
+        request.setLeaderIndex(randomAlphaOfLength(4));
+        request.setFollowerIndex(randomAlphaOfLength(4));
+        if (randomBoolean()) {
+            request.setMaxBatchOperationCount(randomIntBetween(1, Integer.MAX_VALUE));
+        }
+        if (randomBoolean()) {
+            request.setMaxConcurrentReadBatches(randomIntBetween(1, Integer.MAX_VALUE));
+        }
+        if (randomBoolean()) {
+            request.setMaxConcurrentWriteBatches(randomIntBetween(1, Integer.MAX_VALUE));
+        }
+        if (randomBoolean()) {
+            request.setMaxOperationSizeInBytes(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            request.setMaxWriteBufferSize(randomIntBetween(1, Integer.MAX_VALUE));
+        }
+        if (randomBoolean()) {
+            request.setMaxRetryDelay(TimeValue.timeValueMillis(500));
+        }
+        if (randomBoolean()) {
+            request.setPollTimeout(TimeValue.timeValueMillis(500));
+        }
+        return request;
     }
 
     public void testValidate() {
-        FollowIndexAction.Request request = new FollowIndexAction.Request("index1", "index2", null, null, null, null,
-            null, TimeValue.ZERO, null);
+        FollowIndexAction.Request request = new FollowIndexAction.Request();
+        request.setLeaderIndex("index1");
+        request.setFollowerIndex("index2");
+        request.setMaxRetryDelay(TimeValue.ZERO);
+
         ActionRequestValidationException validationException = request.validate();
         assertThat(validationException, notNullValue());
         assertThat(validationException.getMessage(), containsString("[max_retry_delay] must be positive but was [0ms]"));
 
-        request = new FollowIndexAction.Request("index1", "index2", null, null, null, null, null, TimeValue.timeValueMinutes(10), null);
+        request.setMaxRetryDelay(TimeValue.timeValueMinutes(10));
         validationException = request.validate();
         assertThat(validationException, notNullValue());
         assertThat(validationException.getMessage(), containsString("[max_retry_delay] must be less than [5m] but was [10m]"));
 
-        request = new FollowIndexAction.Request("index1", "index2", null, null, null, null, null, TimeValue.timeValueMinutes(1), null);
+        request.setMaxRetryDelay(TimeValue.timeValueMinutes(1));
         validationException = request.validate();
         assertThat(validationException, nullValue());
     }

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java

@@ -45,7 +45,7 @@ public class PutAutoFollowPatternRequestTests extends AbstractStreamableXContent
             request.setFollowIndexNamePattern(randomAlphaOfLength(4));
         }
         if (randomBoolean()) {
-            request.setIdleShardRetryDelay(TimeValue.timeValueMillis(500));
+            request.setPollTimeout(TimeValue.timeValueMillis(500));
         }
         if (randomBoolean()) {
             request.setMaxRetryDelay(TimeValue.timeValueMillis(500));

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskRandomTests.java

@@ -15,7 +15,6 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsResponse;
-import org.elasticsearch.xpack.core.ccr.action.FollowIndexAction;
 import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
 
 import java.nio.charset.StandardCharsets;
@@ -81,7 +80,7 @@ public class ShardFollowNodeTaskRandomTests extends ESTestCase {
             new ShardId("leader_index", "", 0),
             testRun.maxOperationCount,
             concurrency,
-            FollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES,
+            TransportFollowIndexAction.DEFAULT_MAX_BATCH_SIZE_IN_BYTES,
             concurrency,
             10240,
             TimeValue.timeValueMillis(10),

x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java

@@ -81,31 +81,21 @@ public class SourceOnlySnapshot {
         String segmentFileName;
         try (Lock writeLock = targetDirectory.obtainLock(IndexWriter.WRITE_LOCK_NAME);
              StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(commit)) {
-            SegmentInfos segmentInfos = reader.getSegmentInfos();
+            SegmentInfos segmentInfos = reader.getSegmentInfos().clone();
+            DirectoryReader wrappedReader = wrapReader(reader);
             List<SegmentCommitInfo> newInfos = new ArrayList<>();
-            for (LeafReaderContext ctx : reader.leaves()) {
+            for (LeafReaderContext ctx : wrappedReader.leaves()) {
                 LeafReader leafReader = ctx.reader();
-                SegmentCommitInfo info = reader.getSegmentInfos().info(ctx.ord);
-                assert info.info.equals(Lucene.segmentReader(ctx.reader()).getSegmentInfo().info);
-                /* We could do this totally different without wrapping this dummy directory reader if FilterCodecReader would have a
-                 * getDelegate method. This is fixed in LUCENE-8502 but we need to wait for it to come in 7.5.1 or 7.6.
-                 * The reason here is that the ctx.ord is not guaranteed to be equivalent to the SegmentCommitInfo ord in the SegmentInfo
-                 * object since we might drop fully deleted segments. if that happens we are using the wrong reader for the SI and
-                 * might almost certainly expose deleted documents.
-                 */
-                DirectoryReader wrappedReader = wrapReader(new DummyDirectoryReader(reader.directory(), leafReader));
-                if (wrappedReader.leaves().isEmpty() == false) {
-                    leafReader = wrappedReader.leaves().get(0).reader();
-                    LiveDocs liveDocs = getLiveDocs(leafReader);
-                    if (leafReader.numDocs() != 0) { // fully deleted segments don't need to be processed
-                        SegmentCommitInfo newInfo = syncSegment(info, liveDocs, leafReader.getFieldInfos(), existingSegments, createdFiles);
-                        newInfos.add(newInfo);
-                    }
+                SegmentCommitInfo info = Lucene.segmentReader(leafReader).getSegmentInfo();
+                LiveDocs liveDocs = getLiveDocs(leafReader);
+                if (leafReader.numDocs() != 0) { // fully deleted segments don't need to be processed
+                    SegmentCommitInfo newInfo = syncSegment(info, liveDocs, leafReader.getFieldInfos(), existingSegments, createdFiles);
+                    newInfos.add(newInfo);
                 }
             }
             segmentInfos.clear();
             segmentInfos.addAll(newInfos);
-            segmentInfos.setNextWriteGeneration(Math.max(segmentInfos.getGeneration(), generation)+1);
+            segmentInfos.setNextWriteGeneration(Math.max(segmentInfos.getGeneration(), generation) + 1);
             String pendingSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS,
                 "", segmentInfos.getGeneration());
             try (IndexOutput segnOutput = targetDirectory.createOutput(pendingSegmentFileName, IOContext.DEFAULT)) {
@@ -207,9 +197,9 @@ public class SourceOnlySnapshot {
             newInfo = new SegmentCommitInfo(newSegmentInfo, 0, 0, -1, -1, -1);
             List<FieldInfo> fieldInfoCopy = new ArrayList<>(fieldInfos.size());
             for (FieldInfo fieldInfo : fieldInfos) {
-                    fieldInfoCopy.add(new FieldInfo(fieldInfo.name, fieldInfo.number,
-                        false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, fieldInfo.attributes(), 0, 0,
-                        fieldInfo.isSoftDeletesField()));
+                fieldInfoCopy.add(new FieldInfo(fieldInfo.name, fieldInfo.number,
+                    false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, fieldInfo.attributes(), 0, 0,
+                    fieldInfo.isSoftDeletesField()));
             }
             FieldInfos newFieldInfos = new FieldInfos(fieldInfoCopy.toArray(new FieldInfo[0]));
             codec.fieldInfosFormat().write(trackingDir, newSegmentInfo, segmentSuffix, newFieldInfos, IOContext.DEFAULT);
@@ -250,7 +240,7 @@ public class SourceOnlySnapshot {
 
     private boolean assertLiveDocs(Bits liveDocs, int deletes) {
         int actualDeletes = 0;
-        for (int i = 0; i < liveDocs.length(); i++ ) {
+        for (int i = 0; i < liveDocs.length(); i++) {
             if (liveDocs.get(i) == false) {
                 actualDeletes++;
             }
@@ -268,51 +258,4 @@ public class SourceOnlySnapshot {
             this.bits = bits;
         }
     }
-
-    private static class DummyDirectoryReader extends DirectoryReader {
-
-        protected DummyDirectoryReader(Directory directory, LeafReader... segmentReaders) throws IOException {
-            super(directory, segmentReaders);
-        }
-
-        @Override
-        protected DirectoryReader doOpenIfChanged() throws IOException {
-            return null;
-        }
-
-        @Override
-        protected DirectoryReader doOpenIfChanged(IndexCommit commit) throws IOException {
-            return null;
-        }
-
-        @Override
-        protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws IOException {
-            return null;
-        }
-
-        @Override
-        public long getVersion() {
-            return 0;
-        }
-
-        @Override
-        public boolean isCurrent() throws IOException {
-            return false;
-        }
-
-        @Override
-        public IndexCommit getIndexCommit() throws IOException {
-            return null;
-        }
-
-        @Override
-        protected void doClose() throws IOException {
-
-        }
-
-        @Override
-        public CacheHelper getReaderCacheHelper() {
-            return null;
-        }
-    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java

@@ -171,7 +171,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
         public static final ParseField MAX_CONCURRENT_WRITE_BATCHES = new ParseField("max_concurrent_write_batches");
         public static final ParseField MAX_WRITE_BUFFER_SIZE = new ParseField("max_write_buffer_size");
         public static final ParseField MAX_RETRY_DELAY = new ParseField("max_retry_delay");
-        public static final ParseField IDLE_SHARD_RETRY_DELAY = new ParseField("idle_shard_retry_delay");
+        public static final ParseField POLL_TIMEOUT = new ParseField("poll_timeout");
         private static final ParseField HEADERS = new ParseField("headers");
 
         @SuppressWarnings("unchecked")
@@ -193,8 +193,8 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
                 (p, c) -> TimeValue.parseTimeValue(p.text(), MAX_RETRY_DELAY.getPreferredName()),
                 MAX_RETRY_DELAY, ObjectParser.ValueType.STRING);
             PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
-                (p, c) -> TimeValue.parseTimeValue(p.text(), IDLE_SHARD_RETRY_DELAY.getPreferredName()),
-                IDLE_SHARD_RETRY_DELAY, ObjectParser.ValueType.STRING);
+                (p, c) -> TimeValue.parseTimeValue(p.text(), POLL_TIMEOUT.getPreferredName()),
+                POLL_TIMEOUT, ObjectParser.ValueType.STRING);
             PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.mapStrings(), HEADERS);
         }
 
@@ -206,7 +206,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
         private final Integer maxConcurrentWriteBatches;
         private final Integer maxWriteBufferSize;
         private final TimeValue maxRetryDelay;
-        private final TimeValue idleShardRetryDelay;
+        private final TimeValue pollTimeout;
         private final Map<String, String> headers;
 
         public AutoFollowPattern(List<String> leaderIndexPatterns,
@@ -217,7 +217,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
                                  Integer maxConcurrentWriteBatches,
                                  Integer maxWriteBufferSize,
                                  TimeValue maxRetryDelay,
-                                 TimeValue idleShardRetryDelay,
+                                 TimeValue pollTimeout,
                                  Map<String, String> headers) {
             this.leaderIndexPatterns = leaderIndexPatterns;
             this.followIndexPattern = followIndexPattern;
@@ -227,7 +227,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
             this.maxConcurrentWriteBatches = maxConcurrentWriteBatches;
             this.maxWriteBufferSize = maxWriteBufferSize;
             this.maxRetryDelay = maxRetryDelay;
-            this.idleShardRetryDelay = idleShardRetryDelay;
+            this.pollTimeout = pollTimeout;
             this.headers = headers != null ? Collections.unmodifiableMap(headers) : Collections.emptyMap();
         }
 
@@ -240,7 +240,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
             maxConcurrentWriteBatches = in.readOptionalVInt();
             maxWriteBufferSize = in.readOptionalVInt();
             maxRetryDelay = in.readOptionalTimeValue();
-            idleShardRetryDelay = in.readOptionalTimeValue();
+            pollTimeout = in.readOptionalTimeValue();
             this.headers = Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString));
         }
 
@@ -284,8 +284,8 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
             return maxRetryDelay;
         }
 
-        public TimeValue getIdleShardRetryDelay() {
-            return idleShardRetryDelay;
+        public TimeValue getPollTimeout() {
+            return pollTimeout;
         }
 
         public Map<String, String> getHeaders() {
@@ -302,7 +302,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
             out.writeOptionalVInt(maxConcurrentWriteBatches);
             out.writeOptionalVInt(maxWriteBufferSize);
             out.writeOptionalTimeValue(maxRetryDelay);
-            out.writeOptionalTimeValue(idleShardRetryDelay);
+            out.writeOptionalTimeValue(pollTimeout);
             out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString);
         }
 
@@ -330,8 +330,8 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
             if (maxRetryDelay != null) {
                 builder.field(MAX_RETRY_DELAY.getPreferredName(), maxRetryDelay);
             }
-            if (idleShardRetryDelay != null) {
-                builder.field(IDLE_SHARD_RETRY_DELAY.getPreferredName(), idleShardRetryDelay);
+            if (pollTimeout != null) {
+                builder.field(POLL_TIMEOUT.getPreferredName(), pollTimeout);
             }
             builder.field(HEADERS.getPreferredName(), headers);
             return builder;
@@ -355,7 +355,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
                 Objects.equals(maxConcurrentWriteBatches, that.maxConcurrentWriteBatches) &&
                 Objects.equals(maxWriteBufferSize, that.maxWriteBufferSize) &&
                 Objects.equals(maxRetryDelay, that.maxRetryDelay) &&
-                Objects.equals(idleShardRetryDelay, that.idleShardRetryDelay) &&
+                Objects.equals(pollTimeout, that.pollTimeout) &&
                 Objects.equals(headers, that.headers);
         }
 
@@ -370,7 +370,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable<MetaData.Custom> i
                 maxConcurrentWriteBatches,
                 maxWriteBufferSize,
                 maxRetryDelay,
-                idleShardRetryDelay,
+                pollTimeout,
                 headers
             );
         }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowIndexAction.java

@@ -14,7 +14,6 @@ import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -30,14 +29,7 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
     public static final FollowIndexAction INSTANCE = new FollowIndexAction();
     public static final String NAME = "cluster:admin/xpack/ccr/follow_index";
 
-    public static final int DEFAULT_MAX_WRITE_BUFFER_SIZE = 10240;
-    public static final int DEFAULT_MAX_BATCH_OPERATION_COUNT = 1024;
-    public static final int DEFAULT_MAX_CONCURRENT_READ_BATCHES = 1;
-    public static final int DEFAULT_MAX_CONCURRENT_WRITE_BATCHES = 1;
-    public static final long DEFAULT_MAX_BATCH_SIZE_IN_BYTES = Long.MAX_VALUE;
-    static final TimeValue DEFAULT_MAX_RETRY_DELAY = new TimeValue(500);
-    static final TimeValue MAX_RETRY_DELAY = TimeValue.timeValueMinutes(5);
-    public static final TimeValue DEFAULT_POLL_TIMEOUT = TimeValue.timeValueMinutes(1);
+    public static final TimeValue MAX_RETRY_DELAY = TimeValue.timeValueMinutes(5);
 
     private FollowIndexAction() {
         super(NAME);
@@ -59,30 +51,23 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
         private static final ParseField MAX_WRITE_BUFFER_SIZE = new ParseField("max_write_buffer_size");
         private static final ParseField MAX_RETRY_DELAY_FIELD = new ParseField("max_retry_delay");
         private static final ParseField POLL_TIMEOUT = new ParseField("poll_timeout");
-        private static final ConstructingObjectParser<Request, String> PARSER = new ConstructingObjectParser<>(NAME, true,
-            (args, followerIndex) -> {
-                if (args[1] != null) {
-                    followerIndex = (String) args[1];
-                }
-                return new Request((String) args[0], followerIndex, (Integer) args[2], (Integer) args[3], (Long) args[4],
-                    (Integer) args[5], (Integer) args[6], (TimeValue) args[7], (TimeValue) args[8]);
-        });
+        private static final ObjectParser<Request, String> PARSER = new ObjectParser<>(NAME, Request::new);
 
         static {
-            PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), LEADER_INDEX_FIELD);
-            PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FOLLOWER_INDEX_FIELD);
-            PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_BATCH_OPERATION_COUNT);
-            PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_CONCURRENT_READ_BATCHES);
-            PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MAX_BATCH_SIZE_IN_BYTES);
-            PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_CONCURRENT_WRITE_BATCHES);
-            PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_WRITE_BUFFER_SIZE);
+            PARSER.declareString(Request::setLeaderIndex, LEADER_INDEX_FIELD);
+            PARSER.declareString(Request::setFollowerIndex, FOLLOWER_INDEX_FIELD);
+            PARSER.declareInt(Request::setMaxBatchOperationCount, MAX_BATCH_OPERATION_COUNT);
+            PARSER.declareInt(Request::setMaxConcurrentReadBatches, MAX_CONCURRENT_READ_BATCHES);
+            PARSER.declareLong(Request::setMaxOperationSizeInBytes, MAX_BATCH_SIZE_IN_BYTES);
+            PARSER.declareInt(Request::setMaxConcurrentWriteBatches, MAX_CONCURRENT_WRITE_BATCHES);
+            PARSER.declareInt(Request::setMaxWriteBufferSize, MAX_WRITE_BUFFER_SIZE);
             PARSER.declareField(
-                    ConstructingObjectParser.optionalConstructorArg(),
+                    Request::setMaxRetryDelay,
                     (p, c) -> TimeValue.parseTimeValue(p.text(), MAX_RETRY_DELAY_FIELD.getPreferredName()),
                 MAX_RETRY_DELAY_FIELD,
                     ObjectParser.ValueType.STRING);
             PARSER.declareField(
-                    ConstructingObjectParser.optionalConstructorArg(),
+                    Request::setPollTimeout,
                     (p, c) -> TimeValue.parseTimeValue(p.text(), POLL_TIMEOUT.getPreferredName()),
                     POLL_TIMEOUT,
                     ObjectParser.ValueType.STRING);
@@ -108,6 +93,9 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
             return leaderIndex;
         }
 
+        public void setLeaderIndex(String leaderIndex) {
+            this.leaderIndex = leaderIndex;
+        }
 
         private String followerIndex;
 
@@ -115,38 +103,66 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
             return followerIndex;
         }
 
-        private int maxBatchOperationCount;
+        public void setFollowerIndex(String followerIndex) {
+            this.followerIndex = followerIndex;
+        }
 
-        public int getMaxBatchOperationCount() {
+        private Integer maxBatchOperationCount;
+
+        public Integer getMaxBatchOperationCount() {
             return maxBatchOperationCount;
         }
 
-        private int maxConcurrentReadBatches;
+        public void setMaxBatchOperationCount(Integer maxBatchOperationCount) {
+            this.maxBatchOperationCount = maxBatchOperationCount;
+        }
 
-        public int getMaxConcurrentReadBatches() {
+        private Integer maxConcurrentReadBatches;
+
+        public Integer getMaxConcurrentReadBatches() {
             return maxConcurrentReadBatches;
         }
 
-        private long maxOperationSizeInBytes;
+        public void setMaxConcurrentReadBatches(Integer maxConcurrentReadBatches) {
+            this.maxConcurrentReadBatches = maxConcurrentReadBatches;
+        }
 
-        public long getMaxOperationSizeInBytes() {
+        private Long maxOperationSizeInBytes;
+
+        public Long getMaxOperationSizeInBytes() {
             return maxOperationSizeInBytes;
         }
 
-        private int maxConcurrentWriteBatches;
+        public void setMaxOperationSizeInBytes(Long maxOperationSizeInBytes) {
+            this.maxOperationSizeInBytes = maxOperationSizeInBytes;
+        }
 
-        public int getMaxConcurrentWriteBatches() {
+        private Integer maxConcurrentWriteBatches;
+
+        public Integer getMaxConcurrentWriteBatches() {
             return maxConcurrentWriteBatches;
         }
 
-        private int maxWriteBufferSize;
+        public void setMaxConcurrentWriteBatches(Integer maxConcurrentWriteBatches) {
+            this.maxConcurrentWriteBatches = maxConcurrentWriteBatches;
+        }
 
-        public int getMaxWriteBufferSize() {
+        private Integer maxWriteBufferSize;
+
+        public Integer getMaxWriteBufferSize() {
             return maxWriteBufferSize;
         }
 
+        public void setMaxWriteBufferSize(Integer maxWriteBufferSize) {
+            this.maxWriteBufferSize = maxWriteBufferSize;
+        }
+
         private TimeValue maxRetryDelay;
 
+        public void setMaxRetryDelay(TimeValue maxRetryDelay) {
+            this.maxRetryDelay = maxRetryDelay;
+        }
+
         public TimeValue getMaxRetryDelay() {
             return maxRetryDelay;
         }
@@ -157,88 +173,50 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
             return pollTimeout;
         }
 
-        public Request(
-            final String leaderIndex,
-            final String followerIndex,
-            final Integer maxBatchOperationCount,
-            final Integer maxConcurrentReadBatches,
-            final Long maxOperationSizeInBytes,
-            final Integer maxConcurrentWriteBatches,
-            final Integer maxWriteBufferSize,
-            final TimeValue maxRetryDelay,
-            final TimeValue pollTimeout) {
-
-            if (leaderIndex == null) {
-                throw new IllegalArgumentException(LEADER_INDEX_FIELD.getPreferredName() + " is missing");
-            }
-
-            if (followerIndex == null) {
-                throw new IllegalArgumentException(FOLLOWER_INDEX_FIELD.getPreferredName() + " is missing");
-            }
-
-            final int actualMaxBatchOperationCount =
-                    maxBatchOperationCount == null ? DEFAULT_MAX_BATCH_OPERATION_COUNT : maxBatchOperationCount;
-            if (actualMaxBatchOperationCount < 1) {
-                throw new IllegalArgumentException(MAX_BATCH_OPERATION_COUNT.getPreferredName() + " must be larger than 0");
-            }
-
-            final int actualMaxConcurrentReadBatches =
-                    maxConcurrentReadBatches == null ? DEFAULT_MAX_CONCURRENT_READ_BATCHES : maxConcurrentReadBatches;
-            if (actualMaxConcurrentReadBatches < 1) {
-                throw new IllegalArgumentException(MAX_CONCURRENT_READ_BATCHES.getPreferredName() + " must be larger than 0");
-            }
-
-            final long actualMaxOperationSizeInBytes =
-                    maxOperationSizeInBytes == null ? DEFAULT_MAX_BATCH_SIZE_IN_BYTES : maxOperationSizeInBytes;
-            if (actualMaxOperationSizeInBytes <= 0) {
-                throw new IllegalArgumentException(MAX_BATCH_SIZE_IN_BYTES.getPreferredName() + " must be larger than 0");
-            }
-
-            final int actualMaxConcurrentWriteBatches =
-                    maxConcurrentWriteBatches == null ? DEFAULT_MAX_CONCURRENT_WRITE_BATCHES : maxConcurrentWriteBatches;
-            if (actualMaxConcurrentWriteBatches < 1) {
-                throw new IllegalArgumentException(MAX_CONCURRENT_WRITE_BATCHES.getPreferredName() + " must be larger than 0");
-            }
-
-            final int actualMaxWriteBufferSize = maxWriteBufferSize == null ? DEFAULT_MAX_WRITE_BUFFER_SIZE : maxWriteBufferSize;
-            if (actualMaxWriteBufferSize < 1) {
-                throw new IllegalArgumentException(MAX_WRITE_BUFFER_SIZE.getPreferredName() + " must be larger than 0");
-            }
-
-            final TimeValue actualRetryTimeout = maxRetryDelay == null ? DEFAULT_MAX_RETRY_DELAY : maxRetryDelay;
-            final TimeValue actualPollTimeout = pollTimeout == null ? DEFAULT_POLL_TIMEOUT : pollTimeout;
-
-            this.leaderIndex = leaderIndex;
-            this.followerIndex = followerIndex;
-            this.maxBatchOperationCount = actualMaxBatchOperationCount;
-            this.maxConcurrentReadBatches = actualMaxConcurrentReadBatches;
-            this.maxOperationSizeInBytes = actualMaxOperationSizeInBytes;
-            this.maxConcurrentWriteBatches = actualMaxConcurrentWriteBatches;
-            this.maxWriteBufferSize = actualMaxWriteBufferSize;
-            this.maxRetryDelay = actualRetryTimeout;
-            this.pollTimeout = actualPollTimeout;
+        public void setPollTimeout(TimeValue pollTimeout) {
+            this.pollTimeout = pollTimeout;
         }
 
         public Request() {
-
         }
 
         @Override
         public ActionRequestValidationException validate() {
-            ActionRequestValidationException validationException = null;
+            ActionRequestValidationException e = null;
 
-            if (maxRetryDelay.millis() <= 0) {
+            if (leaderIndex == null) {
+                e = addValidationError(LEADER_INDEX_FIELD.getPreferredName() + " is missing", e);
+            }
+            if (followerIndex == null) {
+                e = addValidationError(FOLLOWER_INDEX_FIELD.getPreferredName() + " is missing", e);
+            }
+            if (maxBatchOperationCount != null && maxBatchOperationCount < 1) {
+                e = addValidationError(MAX_BATCH_OPERATION_COUNT.getPreferredName() + " must be larger than 0", e);
+            }
+            if (maxConcurrentReadBatches != null && maxConcurrentReadBatches < 1) {
+                e = addValidationError(MAX_CONCURRENT_READ_BATCHES.getPreferredName() + " must be larger than 0", e);
+            }
+            if (maxOperationSizeInBytes != null && maxOperationSizeInBytes <= 0) {
+                e = addValidationError(MAX_BATCH_SIZE_IN_BYTES.getPreferredName() + " must be larger than 0", e);
+            }
+            if (maxConcurrentWriteBatches != null && maxConcurrentWriteBatches < 1) {
+                e = addValidationError(MAX_CONCURRENT_WRITE_BATCHES.getPreferredName() + " must be larger than 0", e);
+            }
+            if (maxWriteBufferSize != null && maxWriteBufferSize < 1) {
+                e = addValidationError(MAX_WRITE_BUFFER_SIZE.getPreferredName() + " must be larger than 0", e);
+            }
+            if (maxRetryDelay != null && maxRetryDelay.millis() <= 0) {
                 String message = "[" + MAX_RETRY_DELAY_FIELD.getPreferredName() + "] must be positive but was [" +
                     maxRetryDelay.getStringRep() + "]";
-                validationException = addValidationError(message, validationException);
+                e = addValidationError(message, e);
             }
-            if (maxRetryDelay.millis() > FollowIndexAction.MAX_RETRY_DELAY.millis()) {
+            if (maxRetryDelay != null && maxRetryDelay.millis() > FollowIndexAction.MAX_RETRY_DELAY.millis()) {
                 String message = "[" + MAX_RETRY_DELAY_FIELD.getPreferredName() + "] must be less than [" + MAX_RETRY_DELAY +
                     "] but was [" + maxRetryDelay.getStringRep() + "]";
-                validationException = addValidationError(message, validationException);
+                e = addValidationError(message, e);
             }
 
-            return validationException;
+            return e;
         }
 
         @Override
@@ -246,11 +224,11 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
             super.readFrom(in);
             leaderIndex = in.readString();
             followerIndex = in.readString();
-            maxBatchOperationCount = in.readVInt();
-            maxConcurrentReadBatches = in.readVInt();
-            maxOperationSizeInBytes = in.readVLong();
-            maxConcurrentWriteBatches = in.readVInt();
-            maxWriteBufferSize = in.readVInt();
+            maxBatchOperationCount = in.readOptionalVInt();
+            maxConcurrentReadBatches = in.readOptionalVInt();
+            maxOperationSizeInBytes = in.readOptionalLong();
+            maxConcurrentWriteBatches = in.readOptionalVInt();
+            maxWriteBufferSize = in.readOptionalVInt();
             maxRetryDelay = in.readOptionalTimeValue();
             pollTimeout = in.readOptionalTimeValue();
         }
@@ -260,11 +238,11 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
             super.writeTo(out);
             out.writeString(leaderIndex);
             out.writeString(followerIndex);
-            out.writeVInt(maxBatchOperationCount);
-            out.writeVInt(maxConcurrentReadBatches);
-            out.writeVLong(maxOperationSizeInBytes);
-            out.writeVInt(maxConcurrentWriteBatches);
-            out.writeVInt(maxWriteBufferSize);
+            out.writeOptionalVInt(maxBatchOperationCount);
+            out.writeOptionalVInt(maxConcurrentReadBatches);
+            out.writeOptionalLong(maxOperationSizeInBytes);
+            out.writeOptionalVInt(maxConcurrentWriteBatches);
+            out.writeOptionalVInt(maxWriteBufferSize);
             out.writeOptionalTimeValue(maxRetryDelay);
             out.writeOptionalTimeValue(pollTimeout);
         }
@@ -275,13 +253,27 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
             {
                 builder.field(LEADER_INDEX_FIELD.getPreferredName(), leaderIndex);
                 builder.field(FOLLOWER_INDEX_FIELD.getPreferredName(), followerIndex);
-                builder.field(MAX_BATCH_OPERATION_COUNT.getPreferredName(), maxBatchOperationCount);
-                builder.field(MAX_BATCH_SIZE_IN_BYTES.getPreferredName(), maxOperationSizeInBytes);
-                builder.field(MAX_WRITE_BUFFER_SIZE.getPreferredName(), maxWriteBufferSize);
-                builder.field(MAX_CONCURRENT_READ_BATCHES.getPreferredName(), maxConcurrentReadBatches);
-                builder.field(MAX_CONCURRENT_WRITE_BATCHES.getPreferredName(), maxConcurrentWriteBatches);
-                builder.field(MAX_RETRY_DELAY_FIELD.getPreferredName(), maxRetryDelay.getStringRep());
-                builder.field(POLL_TIMEOUT.getPreferredName(), pollTimeout.getStringRep());
+                if (maxBatchOperationCount != null) {
+                    builder.field(MAX_BATCH_OPERATION_COUNT.getPreferredName(), maxBatchOperationCount);
+                }
+                if (maxOperationSizeInBytes != null) {
+                    builder.field(MAX_BATCH_SIZE_IN_BYTES.getPreferredName(), maxOperationSizeInBytes);
+                }
+                if (maxWriteBufferSize != null) {
+                    builder.field(MAX_WRITE_BUFFER_SIZE.getPreferredName(), maxWriteBufferSize);
+                }
+                if (maxConcurrentReadBatches != null) {
+                    builder.field(MAX_CONCURRENT_READ_BATCHES.getPreferredName(), maxConcurrentReadBatches);
+                }
+                if (maxConcurrentWriteBatches != null) {
+                    builder.field(MAX_CONCURRENT_WRITE_BATCHES.getPreferredName(), maxConcurrentWriteBatches);
+                }
+                if (maxRetryDelay != null) {
+                    builder.field(MAX_RETRY_DELAY_FIELD.getPreferredName(), maxRetryDelay.getStringRep());
+                }
+                if (pollTimeout != null) {
+                    builder.field(POLL_TIMEOUT.getPreferredName(), pollTimeout.getStringRep());
+                }
             }
             builder.endObject();
             return builder;
@@ -292,11 +284,11 @@ public final class FollowIndexAction extends Action<AcknowledgedResponse> {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             Request request = (Request) o;
-            return maxBatchOperationCount == request.maxBatchOperationCount &&
-                maxConcurrentReadBatches == request.maxConcurrentReadBatches &&
-                maxOperationSizeInBytes == request.maxOperationSizeInBytes &&
-                maxConcurrentWriteBatches == request.maxConcurrentWriteBatches &&
-                maxWriteBufferSize == request.maxWriteBufferSize &&
+            return Objects.equals(maxBatchOperationCount, request.maxBatchOperationCount) &&
+                Objects.equals(maxConcurrentReadBatches, request.maxConcurrentReadBatches) &&
+                Objects.equals(maxOperationSizeInBytes, request.maxOperationSizeInBytes) &&
+                Objects.equals(maxConcurrentWriteBatches, request.maxConcurrentWriteBatches) &&
+                Objects.equals(maxWriteBufferSize, request.maxWriteBufferSize) &&
                 Objects.equals(maxRetryDelay, request.maxRetryDelay) &&
                 Objects.equals(pollTimeout, request.pollTimeout) &&
                 Objects.equals(leaderIndex, request.leaderIndex) &&

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java

@@ -59,9 +59,9 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
             PARSER.declareField(Request::setMaxRetryDelay,
                 (p, c) -> TimeValue.parseTimeValue(p.text(), AutoFollowPattern.MAX_RETRY_DELAY.getPreferredName()),
                 AutoFollowPattern.MAX_RETRY_DELAY, ObjectParser.ValueType.STRING);
-            PARSER.declareField(Request::setIdleShardRetryDelay,
-                (p, c) -> TimeValue.parseTimeValue(p.text(), AutoFollowPattern.IDLE_SHARD_RETRY_DELAY.getPreferredName()),
-                AutoFollowPattern.IDLE_SHARD_RETRY_DELAY, ObjectParser.ValueType.STRING);
+            PARSER.declareField(Request::setPollTimeout,
+                (p, c) -> TimeValue.parseTimeValue(p.text(), AutoFollowPattern.POLL_TIMEOUT.getPreferredName()),
+                AutoFollowPattern.POLL_TIMEOUT, ObjectParser.ValueType.STRING);
         }
 
         public static Request fromXContent(XContentParser parser, String remoteClusterAlias) throws IOException {
@@ -88,7 +88,7 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
         private Integer maxConcurrentWriteBatches;
         private Integer maxWriteBufferSize;
         private TimeValue maxRetryDelay;
-        private TimeValue idleShardRetryDelay;
+        private TimeValue pollTimeout;
 
         @Override
         public ActionRequestValidationException validate() {
@@ -189,12 +189,12 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
             this.maxRetryDelay = maxRetryDelay;
         }
 
-        public TimeValue getIdleShardRetryDelay() {
-            return idleShardRetryDelay;
+        public TimeValue getPollTimeout() {
+            return pollTimeout;
         }
 
-        public void setIdleShardRetryDelay(TimeValue idleShardRetryDelay) {
-            this.idleShardRetryDelay = idleShardRetryDelay;
+        public void setPollTimeout(TimeValue pollTimeout) {
+            this.pollTimeout = pollTimeout;
         }
 
         @Override
@@ -209,7 +209,7 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
             maxConcurrentWriteBatches = in.readOptionalVInt();
             maxWriteBufferSize = in.readOptionalVInt();
             maxRetryDelay = in.readOptionalTimeValue();
-            idleShardRetryDelay = in.readOptionalTimeValue();
+            pollTimeout = in.readOptionalTimeValue();
         }
 
         @Override
@@ -224,7 +224,7 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
             out.writeOptionalVInt(maxConcurrentWriteBatches);
             out.writeOptionalVInt(maxWriteBufferSize);
             out.writeOptionalTimeValue(maxRetryDelay);
-            out.writeOptionalTimeValue(idleShardRetryDelay);
+            out.writeOptionalTimeValue(pollTimeout);
         }
 
         @Override
@@ -254,8 +254,8 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
                 if (maxRetryDelay != null) {
                     builder.field(AutoFollowPattern.MAX_RETRY_DELAY.getPreferredName(), maxRetryDelay.getStringRep());
                 }
-                if (idleShardRetryDelay != null) {
-                    builder.field(AutoFollowPattern.IDLE_SHARD_RETRY_DELAY.getPreferredName(), idleShardRetryDelay.getStringRep());
+                if (pollTimeout != null) {
+                    builder.field(AutoFollowPattern.POLL_TIMEOUT.getPreferredName(), pollTimeout.getStringRep());
                 }
             }
             builder.endObject();
@@ -276,7 +276,7 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
                 Objects.equals(maxConcurrentWriteBatches, request.maxConcurrentWriteBatches) &&
                 Objects.equals(maxWriteBufferSize, request.maxWriteBufferSize) &&
                 Objects.equals(maxRetryDelay, request.maxRetryDelay) &&
-                Objects.equals(idleShardRetryDelay, request.idleShardRetryDelay);
+                Objects.equals(pollTimeout, request.pollTimeout);
         }
 
         @Override
@@ -291,7 +291,7 @@ public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
                 maxConcurrentWriteBatches,
                 maxWriteBufferSize,
                 maxRetryDelay,
-                idleShardRetryDelay
+                pollTimeout
             );
         }
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/JobStorageDeletionTask.java

@@ -11,8 +11,12 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.Client;
@@ -23,10 +27,12 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.IdsQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction;
@@ -69,43 +75,103 @@ public class JobStorageDeletionTask extends Task {
         final String indexName = AnomalyDetectorsIndex.getPhysicalIndexFromState(state, jobId);
         final String indexPattern = indexName + "-*";
 
-        ActionListener<Boolean> deleteAliasHandler = ActionListener.wrap(finishedHandler, failureHandler);
+        final ActionListener<AcknowledgedResponse> completionHandler = ActionListener.wrap(
+            response -> finishedHandler.accept(response.isAcknowledged()),
+            failureHandler);
 
-        // Step 5. DBQ state done, delete the aliases
+        // Step 7. If we did not drop the index and after DBQ state done, we delete the aliases
         ActionListener<BulkByScrollResponse> dbqHandler = ActionListener.wrap(
                 bulkByScrollResponse -> {
-                    if (bulkByScrollResponse.isTimedOut()) {
-                        logger.warn("[{}] DeleteByQuery for indices [{}, {}] timed out.", jobId, indexName, indexPattern);
-                    }
-                    if (!bulkByScrollResponse.getBulkFailures().isEmpty()) {
-                        logger.warn("[{}] {} failures and {} conflicts encountered while running DeleteByQuery on indices [{}, {}].",
+                    if (bulkByScrollResponse == null) { // no action was taken by DBQ, assume Index was deleted
+                        completionHandler.onResponse(new AcknowledgedResponse(true));
+                    } else {
+                        if (bulkByScrollResponse.isTimedOut()) {
+                            logger.warn("[{}] DeleteByQuery for indices [{}, {}] timed out.", jobId, indexName, indexPattern);
+                        }
+                        if (!bulkByScrollResponse.getBulkFailures().isEmpty()) {
+                            logger.warn("[{}] {} failures and {} conflicts encountered while running DeleteByQuery on indices [{}, {}].",
                                 jobId, bulkByScrollResponse.getBulkFailures().size(), bulkByScrollResponse.getVersionConflicts(),
                                 indexName, indexPattern);
-                        for (BulkItemResponse.Failure failure : bulkByScrollResponse.getBulkFailures()) {
-                            logger.warn("DBQ failure: " + failure);
+                            for (BulkItemResponse.Failure failure : bulkByScrollResponse.getBulkFailures()) {
+                                logger.warn("DBQ failure: " + failure);
+                            }
                         }
+                        deleteAliases(jobId, client, completionHandler);
                     }
-                    deleteAliases(jobId, client, deleteAliasHandler);
                 },
                 failureHandler);
 
-        // Step 4. Delete categorizer state done, DeleteByQuery on the index, matching all docs with the right job_id
-        ActionListener<Boolean> deleteCategorizerStateHandler = ActionListener.wrap(
+        // Step 6. If we did not delete the index, we run a delete by query
+        ActionListener<Boolean> deleteByQueryExecutor = ActionListener.wrap(
                 response -> {
-                    logger.info("Running DBQ on [" + indexName + "," + indexPattern + "] for job [" + jobId + "]");
-                    DeleteByQueryRequest request = new DeleteByQueryRequest(indexName, indexPattern);
-                    ConstantScoreQueryBuilder query =
+                    if (response) {
+                        logger.info("Running DBQ on [" + indexName + "," + indexPattern + "] for job [" + jobId + "]");
+                        DeleteByQueryRequest request = new DeleteByQueryRequest(indexName, indexPattern);
+                        ConstantScoreQueryBuilder query =
                             new ConstantScoreQueryBuilder(new TermQueryBuilder(Job.ID.getPreferredName(), jobId));
-                    request.setQuery(query);
-                    request.setIndicesOptions(MlIndicesUtils.addIgnoreUnavailable(IndicesOptions.lenientExpandOpen()));
-                    request.setSlices(5);
-                    request.setAbortOnVersionConflict(false);
-                    request.setRefresh(true);
+                        request.setQuery(query);
+                        request.setIndicesOptions(MlIndicesUtils.addIgnoreUnavailable(IndicesOptions.lenientExpandOpen()));
+                        request.setSlices(5);
+                        request.setAbortOnVersionConflict(false);
+                        request.setRefresh(true);
 
-                    executeAsyncWithOrigin(client, ML_ORIGIN, DeleteByQueryAction.INSTANCE, request, dbqHandler);
+                        executeAsyncWithOrigin(client, ML_ORIGIN, DeleteByQueryAction.INSTANCE, request, dbqHandler);
+                    } else { // We did not execute DBQ, no need to delete aliases or check the response
+                        dbqHandler.onResponse(null);
+                    }
                 },
                 failureHandler);
 
+        // Step 5. If we have any hits, that means we are NOT the only job on this index, and should not delete it
+        // if we do not have any hits, we can drop the index and then skip the DBQ and alias deletion
+        ActionListener<SearchResponse> customIndexSearchHandler = ActionListener.wrap(
+            searchResponse -> {
+                if (searchResponse == null || searchResponse.getHits().totalHits > 0) {
+                    deleteByQueryExecutor.onResponse(true); // We need to run DBQ and alias deletion
+                } else {
+                    logger.info("Running DELETE Index on [" + indexName + "] for job [" + jobId + "]");
+                    DeleteIndexRequest request = new DeleteIndexRequest(indexName);
+                    request.indicesOptions(IndicesOptions.lenientExpandOpen());
+                    // If we have deleted the index, then we don't need to delete the aliases or run the DBQ
+                    executeAsyncWithOrigin(
+                        client.threadPool().getThreadContext(),
+                        ML_ORIGIN,
+                        request,
+                        ActionListener.<AcknowledgedResponse>wrap(
+                            response -> deleteByQueryExecutor.onResponse(false), // skip DBQ && Alias
+                            failureHandler),
+                        client.admin().indices()::delete);
+                }
+            },
+            failure -> {
+                if (failure.getClass() == IndexNotFoundException.class) { // assume the index is already deleted
+                    deleteByQueryExecutor.onResponse(false); // skip DBQ && Alias
+                } else {
+                    failureHandler.accept(failure);
+                }
+            }
+        );
+
+        // Step 4. Determine if we are on a shared index by looking at `.ml-anomalies-shared` or the custom index's aliases
+        ActionListener<Boolean> deleteCategorizerStateHandler = ActionListener.wrap(
+            response -> {
+                if (indexName.equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX +
+                    AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT)) {
+                    customIndexSearchHandler.onResponse(null); //don't bother searching the index any further, we are on the default shared
+                } else {
+                    SearchSourceBuilder source = new SearchSourceBuilder()
+                        .size(1)
+                        .query(QueryBuilders.boolQuery().filter(
+                            QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId))));
+
+                    SearchRequest searchRequest = new SearchRequest(indexName);
+                    searchRequest.source(source);
+                    executeAsyncWithOrigin(client, ML_ORIGIN, SearchAction.INSTANCE, searchRequest, customIndexSearchHandler);
+                }
+            },
+            failureHandler
+        );
+
         // Step 3. Delete quantiles done, delete the categorizer state
         ActionListener<Boolean> deleteQuantilesHandler = ActionListener.wrap(
                 response -> deleteCategorizerState(jobId, client, 1, deleteCategorizerStateHandler),
@@ -189,7 +255,7 @@ public class JobStorageDeletionTask extends Task {
                 }));
     }
 
-    private void deleteAliases(String jobId, Client client, ActionListener<Boolean> finishedHandler) {
+    private void deleteAliases(String jobId, Client client, ActionListener<AcknowledgedResponse> finishedHandler) {
         final String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);
         final String writeAliasName = AnomalyDetectorsIndex.resultsWriteAlias(jobId);
 
@@ -204,12 +270,13 @@ public class JobStorageDeletionTask extends Task {
                             if (removeRequest == null) {
                                 // don't error if the job's aliases have already been deleted - carry on and delete the
                                 // rest of the job's data
-                                finishedHandler.onResponse(true);
+                                finishedHandler.onResponse(new AcknowledgedResponse(true));
                                 return;
                             }
                             executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, removeRequest,
-                                    ActionListener.<AcknowledgedResponse>wrap(removeResponse -> finishedHandler.onResponse(true),
-                                            finishedHandler::onFailure),
+                                ActionListener.<AcknowledgedResponse>wrap(
+                                    finishedHandler::onResponse,
+                                    finishedHandler::onFailure),
                                     client.admin().indices()::aliases);
                         },
                         finishedHandler::onFailure), client.admin().indices()::getAliases);

x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponseTests.java

@@ -13,6 +13,8 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractXContentTestCase;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Predicate;
@@ -88,12 +90,29 @@ public class GraphExploreResponseTests extends AbstractXContentTestCase< GraphEx
         assertThat(newInstance.getTook(), equalTo(expectedInstance.getTook()));
         assertThat(newInstance.isTimedOut(), equalTo(expectedInstance.isTimedOut()));
         
+        Comparator<Connection> connComparator = new Comparator<Connection>() {
+            @Override
+            public int compare(Connection o1, Connection o2) {
+                return o1.getId().toString().compareTo(o2.getId().toString());
+            }
+        };
         Connection[] newConns = newInstance.getConnections().toArray(new Connection[0]);
         Connection[] expectedConns = expectedInstance.getConnections().toArray(new Connection[0]);
+        Arrays.sort(newConns, connComparator);
+        Arrays.sort(expectedConns, connComparator);
         assertArrayEquals(expectedConns, newConns);
         
-        Vertex[] newVertices = newInstance.getVertices().toArray(new Vertex[0]);
+        //Sort the vertices lists before equality test (map insertion sequences can cause order differences)
+        Comparator<Vertex> comparator = new Comparator<Vertex>() {
+            @Override
+            public int compare(Vertex o1, Vertex o2) {
+                return o1.getId().toString().compareTo(o2.getId().toString());
+            }
+        };
+        Vertex[] newVertices = newInstance.getVertices().toArray(new Vertex[0]);        
         Vertex[] expectedVertices = expectedInstance.getVertices().toArray(new Vertex[0]);
+        Arrays.sort(newVertices, comparator);
+        Arrays.sort(expectedVertices, comparator);
         assertArrayEquals(expectedVertices, newVertices);
         
         ShardOperationFailedException[] newFailures = newInstance.getShardFailures();

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java

@@ -192,33 +192,60 @@ public class MlJobIT extends ESRestTestCase {
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
 
-        String id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1234", 300);
-        Request createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/doc/" + id);
-        createResultRequest.setJsonEntity(String.format(Locale.ROOT,
+        { //create jobId1 docs
+            String id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1234", 300);
+            Request createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/doc/" + id);
+            createResultRequest.setJsonEntity(String.format(Locale.ROOT,
                 "{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}",
                 jobId1, "1234", 1));
-        client().performRequest(createResultRequest);
+            client().performRequest(createResultRequest);
 
-        id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1236", 300);
-        createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/doc/" + id);
-        createResultRequest.setJsonEntity(String.format(Locale.ROOT,
+            id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1236", 300);
+            createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/doc/" + id);
+            createResultRequest.setJsonEntity(String.format(Locale.ROOT,
                 "{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}",
                 jobId1, "1236", 1));
-        client().performRequest(createResultRequest);
+            client().performRequest(createResultRequest);
 
-        client().performRequest(new Request("POST", "/_refresh"));
+            client().performRequest(new Request("POST", "/_refresh"));
 
-        responseAsString = EntityUtils.toString(client().performRequest(
-            new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1 + "/results/buckets")).getEntity());
-        assertThat(responseAsString, containsString("\"count\":2"));
+            responseAsString = EntityUtils.toString(client().performRequest(
+                new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1 + "/results/buckets")).getEntity());
+            assertThat(responseAsString, containsString("\"count\":2"));
 
-        responseAsString = EntityUtils.toString(client().performRequest(
-            new Request("GET", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/_search")).getEntity());
-        assertThat(responseAsString, containsString("\"total\":2"));
+            responseAsString = EntityUtils.toString(client().performRequest(
+                new Request("GET", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/_search")).getEntity());
+            assertThat(responseAsString, containsString("\"total\":2"));
+        }
+        { //create jobId2 docs
+            String id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId2, "1234", 300);
+            Request createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/doc/" + id);
+            createResultRequest.setJsonEntity(String.format(Locale.ROOT,
+                "{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}",
+                jobId2, "1234", 1));
+            client().performRequest(createResultRequest);
+
+            id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId2, "1236", 300);
+            createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/doc/" + id);
+            createResultRequest.setJsonEntity(String.format(Locale.ROOT,
+                "{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}",
+                jobId2, "1236", 1));
+            client().performRequest(createResultRequest);
+
+            client().performRequest(new Request("POST", "/_refresh"));
+
+            responseAsString = EntityUtils.toString(client().performRequest(
+                new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2 + "/results/buckets")).getEntity());
+            assertThat(responseAsString, containsString("\"count\":2"));
+
+            responseAsString = EntityUtils.toString(client().performRequest(
+                new Request("GET", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/_search")).getEntity());
+            assertThat(responseAsString, containsString("\"total\":2"));
+        }
 
         client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1));
 
-        // check that indices still exist, but are empty and aliases are gone
+        // check that indices still exist, but no longer have job1 entries and aliases are gone
         responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
         assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists
@@ -230,7 +257,16 @@ public class MlJobIT extends ESRestTestCase {
 
         responseAsString = EntityUtils.toString(client().performRequest(
                 new Request("GET", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "/_count")).getEntity());
-        assertThat(responseAsString, containsString("\"count\":0"));
+        assertThat(responseAsString, containsString("\"count\":2"));
+
+        // Delete the second job and verify aliases are gone, and original concrete/custom index is gone
+        client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2));
+        responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
+        assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
+
+        client().performRequest(new Request("POST", "/_refresh"));
+        responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
+        assertThat(responseAsString, not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)));
     }
 
     public void testCreateJobInSharedIndexUpdatesMapping() throws Exception {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java

@@ -20,7 +20,6 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.CustomAnalyzer;
-import org.elasticsearch.index.analysis.CustomAnalyzerProvider;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule;
@@ -217,6 +216,8 @@ public class CategorizationAnalyzer implements Closeable {
                                                                       Tuple<String, TokenizerFactory> tokenizerFactory,
                                                                       List<CharFilterFactory> charFilterFactoryList) throws IOException {
         List<CategorizationAnalyzerConfig.NameOrDefinition> tokenFilters = config.getTokenFilters();
+        TransportAnalyzeAction.DeferredTokenFilterRegistry deferredRegistry
+            = new TransportAnalyzeAction.DeferredTokenFilterRegistry(analysisRegistry, null);
         final List<TokenFilterFactory> tokenFilterFactoryList = new ArrayList<>();
         for (CategorizationAnalyzerConfig.NameOrDefinition tokenFilter : tokenFilters) {
             TokenFilterFactory tokenFilterFactory;
@@ -241,8 +242,8 @@ public class CategorizationAnalyzer implements Closeable {
                 // Need to set anonymous "name" of token_filter
                 tokenFilterFactory = tokenFilterFactoryFactory.get(buildDummyIndexSettings(settings), environment, "_anonymous_tokenfilter",
                     settings);
-                tokenFilterFactory = CustomAnalyzerProvider.checkAndApplySynonymFilter(tokenFilterFactory, tokenizerFactory.v1(),
-                    tokenizerFactory.v2(), tokenFilterFactoryList, charFilterFactoryList, environment);
+                tokenFilterFactory = tokenFilterFactory.getChainAwareTokenFilterFactory(tokenizerFactory.v2(),
+                    charFilterFactoryList, tokenFilterFactoryList, deferredRegistry);
             }
             if (tokenFilterFactory == null) {
                 throw new IllegalArgumentException("Failed to find or create token filter [" + tokenFilter + "]");

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java

@@ -422,9 +422,6 @@ public class MonitoringIT extends ESSingleNodeTestCase {
         assertThat((String) indexStats.get("uuid"), not(isEmptyOrNullString()));
         assertThat(indexStats.get("created"), notNullValue());
         assertThat((String) indexStats.get("status"), not(isEmptyOrNullString()));
-        assertThat(indexStats.get("version"), notNullValue());
-        final Map<String, Object> version = (Map<String, Object>) indexStats.get("version");
-        assertEquals(2, version.size());
         assertThat(indexStats.get("shards"), notNullValue());
         final Map<String, Object> shards = (Map<String, Object>) indexStats.get("shards");
         assertEquals(11, shards.size());

x-pack/plugin/sql/src/main/antlr/SqlBase.g4

@@ -163,14 +163,18 @@ expression
 booleanExpression
     : NOT booleanExpression                                                                 #logicalNot
     | EXISTS '(' query ')'                                                                  #exists
-    | QUERY '(' queryString=string (',' options=string)* ')'                                #stringQuery
-    | MATCH '(' singleField=qualifiedName ',' queryString=string (',' options=string)* ')'  #matchQuery
-    | MATCH '(' multiFields=string ',' queryString=string (',' options=string)* ')'         #multiMatchQuery
+    | QUERY '(' queryString=string matchQueryOptions ')'                                    #stringQuery
+    | MATCH '(' singleField=qualifiedName ',' queryString=string matchQueryOptions ')'      #matchQuery
+    | MATCH '(' multiFields=string ',' queryString=string matchQueryOptions ')'             #multiMatchQuery
     | predicated                                                                            #booleanDefault
     | left=booleanExpression operator=AND right=booleanExpression                           #logicalBinary
     | left=booleanExpression operator=OR right=booleanExpression                            #logicalBinary
     ;
 
+matchQueryOptions
+    : (',' string)*
+    ;
+
 // workaround for:
 //  https://github.com/antlr/antlr4/issues/780
 //  https://github.com/antlr/antlr4/issues/781

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/fulltext/FullTextUtils.java

@@ -5,16 +5,16 @@
  */
 package org.elasticsearch.xpack.sql.expression.predicate.fulltext;
 
-import java.util.LinkedHashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.xpack.sql.expression.predicate.fulltext.FullTextPredicate.Operator;
 import org.elasticsearch.xpack.sql.parser.ParsingException;
 import org.elasticsearch.xpack.sql.tree.Location;
 
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
 import static java.util.Collections.emptyMap;
 
 abstract class FullTextUtils {
@@ -26,7 +26,7 @@ abstract class FullTextUtils {
             return emptyMap();
         }
         String[] list = Strings.delimitedListToStringArray(options, DELIMITER);
-        Map<String, String> op = new LinkedHashMap<String, String>(list.length);
+        Map<String, String> op = new LinkedHashMap<>(list.length);
 
         for (String entry : list) {
             String[] split = splitInTwo(entry, "=");

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java

@@ -67,6 +67,7 @@ import org.elasticsearch.xpack.sql.parser.SqlBaseParser.LikePatternContext;
 import org.elasticsearch.xpack.sql.parser.SqlBaseParser.LogicalBinaryContext;
 import org.elasticsearch.xpack.sql.parser.SqlBaseParser.LogicalNotContext;
 import org.elasticsearch.xpack.sql.parser.SqlBaseParser.MatchQueryContext;
+import org.elasticsearch.xpack.sql.parser.SqlBaseParser.MatchQueryOptionsContext;
 import org.elasticsearch.xpack.sql.parser.SqlBaseParser.MultiMatchQueryContext;
 import org.elasticsearch.xpack.sql.parser.SqlBaseParser.NullLiteralContext;
 import org.elasticsearch.xpack.sql.parser.SqlBaseParser.OrderByContext;
@@ -99,6 +100,7 @@ import java.math.BigInteger;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.StringJoiner;
 
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.xpack.sql.type.DataTypeConversion.conversionFor;
@@ -324,18 +326,27 @@ abstract class ExpressionBuilder extends IdentifierBuilder {
     //
     @Override
     public Object visitStringQuery(StringQueryContext ctx) {
-        return new StringQueryPredicate(source(ctx), string(ctx.queryString), string(ctx.options));
+        return new StringQueryPredicate(source(ctx), string(ctx.queryString), getQueryOptions(ctx.matchQueryOptions()));
     }
 
     @Override
     public Object visitMatchQuery(MatchQueryContext ctx) {
         return new MatchQueryPredicate(source(ctx), new UnresolvedAttribute(source(ctx.singleField),
-                visitQualifiedName(ctx.singleField)), string(ctx.queryString), string(ctx.options));
+                visitQualifiedName(ctx.singleField)), string(ctx.queryString), getQueryOptions(ctx.matchQueryOptions()));
     }
 
     @Override
     public Object visitMultiMatchQuery(MultiMatchQueryContext ctx) {
-        return new MultiMatchQueryPredicate(source(ctx), string(ctx.multiFields), string(ctx.queryString), string(ctx.options));
+        return new MultiMatchQueryPredicate(source(ctx), string(ctx.multiFields), string(ctx.queryString),
+            getQueryOptions(ctx.matchQueryOptions()));
+    }
+
+    private String getQueryOptions(MatchQueryOptionsContext optionsCtx) {
+        StringJoiner sj = new StringJoiner(";");
+        for (StringContext sc: optionsCtx.string()) {
+            sj.add(string(sc));
+        }
+        return sj.toString();
     }
 
     @Override
@@ -676,4 +687,4 @@ abstract class ExpressionBuilder extends IdentifierBuilder {
 
         return new Literal(source(ctx), string, DataType.KEYWORD);
     }
-}
+}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java

@@ -527,6 +527,18 @@ class SqlBaseBaseListener implements SqlBaseListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitLogicalBinary(SqlBaseParser.LogicalBinaryContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx) { }
   /**
    * {@inheritDoc}
    *

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java

@@ -312,6 +312,13 @@ class SqlBaseBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements SqlBa
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitLogicalBinary(SqlBaseParser.LogicalBinaryContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseListener.java

@@ -489,6 +489,16 @@ interface SqlBaseListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitLogicalBinary(SqlBaseParser.LogicalBinaryContext ctx);
+  /**
+   * Enter a parse tree produced by {@link SqlBaseParser#matchQueryOptions}.
+   * @param ctx the parse tree
+   */
+  void enterMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx);
+  /**
+   * Exit a parse tree produced by {@link SqlBaseParser#matchQueryOptions}.
+   * @param ctx the parse tree
+   */
+  void exitMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx);
   /**
    * Enter a parse tree produced by {@link SqlBaseParser#predicated}.
    * @param ctx the parse tree

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseVisitor.java

@@ -294,6 +294,12 @@ interface SqlBaseVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitLogicalBinary(SqlBaseParser.LogicalBinaryContext ctx);
+  /**
+   * Visit a parse tree produced by {@link SqlBaseParser#matchQueryOptions}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitMatchQueryOptions(SqlBaseParser.MatchQueryOptionsContext ctx);
   /**
    * Visit a parse tree produced by {@link SqlBaseParser#predicated}.
    * @param ctx the parse tree

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java

@@ -5,16 +5,20 @@
  */
 package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
 
+import org.elasticsearch.bootstrap.JavaVersion;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.NamedDateTimeProcessor.NameExtractor;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.junit.Assume;
 
 import java.io.IOException;
 import java.util.TimeZone;
 
 public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase<NamedDateTimeProcessor> {
+    
     private static final TimeZone UTC = TimeZone.getTimeZone("UTC");
 
     public static NamedDateTimeProcessor randomNamedDateTimeProcessor() {
@@ -37,21 +41,21 @@ public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase
         return new NamedDateTimeProcessor(replaced, UTC);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33621")
     public void testValidDayNamesInUTC() {
+        assumeJava9PlusAndCompatLocaleProviderSetting();
         NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.DAY_NAME, UTC);
         assertEquals("Thursday", proc.process("0"));
         assertEquals("Saturday", proc.process("-64164233612338"));
         assertEquals("Monday", proc.process("64164233612338"));
-
+        
         assertEquals("Thursday", proc.process(new DateTime(0L, DateTimeZone.UTC)));
         assertEquals("Thursday", proc.process(new DateTime(-5400, 12, 25, 2, 0, DateTimeZone.UTC)));
         assertEquals("Friday", proc.process(new DateTime(30, 2, 1, 12, 13, DateTimeZone.UTC)));
         assertEquals("Tuesday", proc.process(new DateTime(10902, 8, 22, 11, 11, DateTimeZone.UTC)));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33621")
     public void testValidDayNamesWithNonUTCTimeZone() {
+        assumeJava9PlusAndCompatLocaleProviderSetting();
         NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.DAY_NAME, TimeZone.getTimeZone("GMT-10:00"));
         assertEquals("Wednesday", proc.process("0"));
         assertEquals("Friday", proc.process("-64164233612338"));
@@ -64,9 +68,9 @@ public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase
         assertEquals("Monday", proc.process(new DateTime(10902, 8, 22, 9, 59, DateTimeZone.UTC)));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33621")
     public void testValidMonthNamesInUTC() {
-        NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.MONTH_NAME, UTC);
+        assumeJava9PlusAndCompatLocaleProviderSetting();
+        NamedDateTimeProcessor proc  = new NamedDateTimeProcessor(NameExtractor.MONTH_NAME, UTC);
         assertEquals("January", proc.process("0"));
         assertEquals("September", proc.process("-64164233612338"));
         assertEquals("April", proc.process("64164233612338"));
@@ -77,8 +81,8 @@ public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase
         assertEquals("August", proc.process(new DateTime(10902, 8, 22, 11, 11, DateTimeZone.UTC)));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33621")
     public void testValidMonthNamesWithNonUTCTimeZone() {
+        assumeJava9PlusAndCompatLocaleProviderSetting();
         NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.MONTH_NAME, TimeZone.getTimeZone("GMT-3:00"));
         assertEquals("December", proc.process("0"));
         assertEquals("August", proc.process("-64165813612338")); // GMT: Tuesday, September 1, -0064 2:53:07.662 AM
@@ -90,4 +94,23 @@ public class NamedDateTimeProcessorTests extends AbstractWireSerializingTestCase
         assertEquals("July", proc.process(new DateTime(10902, 8, 1, 2, 59, DateTimeZone.UTC)));
         assertEquals("August", proc.process(new DateTime(10902, 8, 1, 3, 00, DateTimeZone.UTC)));
     }
+    
+    /*
+     * This method checks the existence of a jvm parameter that should exist in ES jvm.options for Java 9+. If the parameter is
+     * missing, the tests will be skipped. Not doing this, the tests will fail because the day and month names will be in the narrow
+     * format (Mon, Tue, Jan, Feb etc) instead of full format (Monday, Tuesday, January, February etc).
+     * 
+     * Related infra issue: https://github.com/elastic/elasticsearch/issues/33796
+     */
+    private void assumeJava9PlusAndCompatLocaleProviderSetting() {
+        // at least Java 9
+        if (JavaVersion.current().compareTo(JavaVersion.parse("9")) < 0) {
+            return;
+        }
+        String beforeJava9CompatibleLocale = System.getProperty("java.locale.providers");
+        // and COMPAT setting needs to be first on the list
+        boolean isBeforeJava9Compatible = beforeJava9CompatibleLocale != null 
+                && Strings.tokenizeToStringArray(beforeJava9CompatibleLocale, ",")[0].equals("COMPAT");
+        Assume.assumeTrue(isBeforeJava9Compatible);
+    }
 }

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/fulltext/FullTextUtilsTests.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.predicate.fulltext;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.sql.parser.ParsingException;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.is;
+
+public class FullTextUtilsTests extends ESTestCase {
+
+    public void testColonDelimited() {
+        Map<String, String> options = FullTextUtils.parseSettings("k1=v1;k2=v2", new Location(1, 1));
+        assertThat(options.size(), is(2));
+        assertThat(options, hasEntry("k1", "v1"));
+        assertThat(options, hasEntry("k2", "v2"));
+    }
+
+    public void testColonDelimitedErrorString() {
+        ParsingException e = expectThrows(ParsingException.class,
+            () -> FullTextUtils.parseSettings("k1=v1;k2v2", new Location(1, 1)));
+        assertThat(e.getMessage(), is("line 1:3: Cannot parse entry k2v2 in options k1=v1;k2v2"));
+        assertThat(e.getLineNumber(), is(1));
+        assertThat(e.getColumnNumber(), is(3));
+    }
+
+    public void testColonDelimitedErrorDuplicate() {
+        ParsingException e = expectThrows(ParsingException.class,
+            () -> FullTextUtils.parseSettings("k1=v1;k1=v2", new Location(1, 1)));
+        assertThat(e.getMessage(), is("line 1:3: Duplicate option k1=v2 detected in options k1=v1;k1=v2"));
+        assertThat(e.getLineNumber(), is(1));
+        assertThat(e.getColumnNumber(), is(3));
+    }
+}

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java

@@ -11,6 +11,10 @@ import org.elasticsearch.xpack.sql.expression.Order;
 import org.elasticsearch.xpack.sql.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.sql.expression.UnresolvedStar;
 import org.elasticsearch.xpack.sql.expression.function.UnresolvedFunction;
+import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MatchQueryPredicate;
+import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MultiMatchQueryPredicate;
+import org.elasticsearch.xpack.sql.expression.predicate.fulltext.StringQueryPredicate;
+import org.elasticsearch.xpack.sql.plan.logical.Filter;
 import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.sql.plan.logical.OrderBy;
 import org.elasticsearch.xpack.sql.plan.logical.Project;
@@ -19,6 +23,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import static java.util.stream.Collectors.toList;
+import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -92,6 +97,45 @@ public class SqlParserTests extends ESTestCase {
         assertEquals("baz", a.name());
     }
 
+    public void testStringQuery() {
+        LogicalPlan plan =
+            parseStatement("SELECT * FROM FOO WHERE " +
+                "QUERY('foo', 'default_field=last_name;lenient=true', 'fuzzy_rewrite=scoring_boolean')");
+
+        StringQueryPredicate sqp = (StringQueryPredicate) ((Filter) plan.children().get(0).children().get(0)).condition();
+        assertEquals("foo", sqp.query());
+        assertEquals(3, sqp.optionMap().size());
+        assertThat(sqp.optionMap(), hasEntry("default_field", "last_name"));
+        assertThat(sqp.optionMap(), hasEntry("lenient", "true"));
+        assertThat(sqp.optionMap(), hasEntry("fuzzy_rewrite", "scoring_boolean"));
+    }
+
+    public void testMatchQuery() {
+        LogicalPlan plan = parseStatement("SELECT * FROM FOO WHERE " +
+                    "MATCH(first_name, 'foo', 'operator=AND;lenient=true', 'fuzzy_rewrite=scoring_boolean')");
+
+        MatchQueryPredicate mqp = (MatchQueryPredicate) ((Filter) plan.children().get(0).children().get(0)).condition();
+        assertEquals("foo", mqp.query());
+        assertEquals("?first_name", mqp.field().toString());
+        assertEquals(3, mqp.optionMap().size());
+        assertThat(mqp.optionMap(), hasEntry("operator", "AND"));
+        assertThat(mqp.optionMap(), hasEntry("lenient", "true"));
+        assertThat(mqp.optionMap(), hasEntry("fuzzy_rewrite", "scoring_boolean"));
+    }
+
+    public void testMultiMatchQuery() {
+        LogicalPlan plan = parseStatement("SELECT * FROM FOO WHERE " +
+                "MATCH('first_name,last_name', 'foo', 'operator=AND;type=best_fields', 'fuzzy_rewrite=scoring_boolean')");
+
+        MultiMatchQueryPredicate mmqp = (MultiMatchQueryPredicate) ((Filter) plan.children().get(0).children().get(0)).condition();
+        assertEquals("foo", mmqp.query());
+        assertEquals("first_name,last_name", mmqp.fieldString());
+        assertEquals(3, mmqp.optionMap().size());
+        assertThat(mmqp.optionMap(), hasEntry("operator", "AND"));
+        assertThat(mmqp.optionMap(), hasEntry("type", "best_fields"));
+        assertThat(mmqp.optionMap(), hasEntry("fuzzy_rewrite", "scoring_boolean"));
+    }
+
     private LogicalPlan parseStatement(String sql) {
         return new SqlParser().createStatement(sql);
     }
@@ -132,4 +176,4 @@ public class SqlParserTests extends ESTestCase {
         String dirStr = dir.toString();
         return randomBoolean() && dirStr.equals("ASC") ? "" : " " + dirStr;
     }
-}
+}