Internal: made original indices optional for broadcast delete and delete by query shard requests

Shard requests like broadcast delete and delete by query, that needs to be executed on primary and all replicas, get read and written out to the transport on the same node. That means that if we add some field version checks are not enough to maintain bw comp since a newer node that holds the primary might receive the request from an older node, that didn't provide the field. Yet, when writing the request out again to a newer node that holds the replica, we do try and serialize the field although it's missing. The newer fields just needs to be set to optional in these cases, in addition to the version checks.

Re-enabled testDeleteByQuery and testDeleteRoutingRequired bw comp tests since this was the cause of their failures.

Closes #7406

src/main/java/org/elasticsearch/action/delete/ShardDeleteRequest.java

@@ -109,7 +109,7 @@ public class ShardDeleteRequest extends ShardReplicationOperationRequest<ShardDe
         refresh = in.readBoolean();
         version = Versions.readVersion(in);
         if (in.getVersion().onOrAfter(Version.V_1_4_0)) {
-            originalIndex = in.readString();
+            originalIndex = in.readOptionalString();
         }
     }
 
@@ -122,7 +122,7 @@ public class ShardDeleteRequest extends ShardReplicationOperationRequest<ShardDe
         out.writeBoolean(refresh);
         Versions.writeVersion(version, out);
         if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
-            out.writeString(originalIndex);
+            out.writeOptionalString(originalIndex);
         }
     }
 }

src/main/java/org/elasticsearch/action/deletebyquery/ShardDeleteByQueryRequest.java

@@ -141,7 +141,7 @@ public class ShardDeleteByQueryRequest extends ShardReplicationOperationRequest<
         } else {
             nowInMillis = System.currentTimeMillis();
         }
-        originalIndices = OriginalIndices.readOriginalIndices(in);
+        originalIndices = OriginalIndices.readOptionalOriginalIndices(in);
     }
 
     @Override
@@ -169,7 +169,7 @@ public class ShardDeleteByQueryRequest extends ShardReplicationOperationRequest<
         if (out.getVersion().onOrAfter(Version.V_1_2_0)) {
             out.writeVLong(nowInMillis);
         }
-        OriginalIndices.writeOriginalIndices(originalIndices, out);
+        OriginalIndices.writeOptionalOriginalIndices(originalIndices, out);
     }
 
     @Override

src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityTest.java

@@ -22,7 +22,6 @@ import com.carrotsearch.randomizedtesting.LifecycleScope;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.util.English;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
@@ -560,8 +559,6 @@ public class BasicBackwardsCompatibilityTest extends ElasticsearchBackwardsCompa
     }
 
     @Test
-    @LuceneTestCase.AwaitsFix(bugUrl = "working on this")
-    //made this tests a usual integration test to see if it fails in non bw comp mode
     public void testDeleteByQuery() throws ExecutionException, InterruptedException {
         createIndex("test");
         ensureYellow("test");
@@ -591,8 +588,6 @@ public class BasicBackwardsCompatibilityTest extends ElasticsearchBackwardsCompa
         assertThat(searchResponse.getHits().totalHits(), equalTo(1l));
     }
 
-    @LuceneTestCase.AwaitsFix(bugUrl = "working on this")
-    //made this tests a usual integration test to see if it fails in non bw comp mode
     @Test
     public void testDeleteRoutingRequired() throws ExecutionException, InterruptedException, IOException {
         assertAcked(prepareCreate("test").addMapping("test",