Enable 5.x to 6.x BWC tests

This commit enables real BWC testing against a 5.1 snapshot. All
REST tests plus rolling upgrade test now run against a mixed version
cross major version cluster.

core/src/main/java/org/elasticsearch/Version.java

@@ -298,7 +298,26 @@ public class Version {
      * is a beta or RC release then the version itself is returned.
      */
     public Version minimumCompatibilityVersion() {
-        return Version.smallest(this, fromId(major * 1000000 + 99));
+        final int bwcMajor;
+        final int bwcMinor;
+        if (this.onOrAfter(Version.V_6_0_0_alpha1)) {
+            bwcMajor = major-1;
+            bwcMinor = 0; // TODO we have to move this to the latest released minor of the last major but for now we just keep
+        } else {
+            bwcMajor = major;
+            bwcMinor = 0;
+        }
+        return Version.smallest(this, fromId(bwcMajor * 1000000 + bwcMinor * 10000 + 99));
+    }
+
+    /**
+     * Returns <code>true</code> iff both version are compatible. Otherwise <code>false</code>
+     */
+    public boolean isCompatible(Version version) {
+        boolean compatible = onOrAfter(version.minimumCompatibilityVersion())
+            && version.onOrAfter(minimumCompatibilityVersion());
+        assert compatible == false || Math.abs(major - version.major) <= 1;
+        return compatible;
     }
 
     @SuppressForbidden(reason = "System.out.*")

core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java

@@ -64,8 +64,6 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
 
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(PutIndexTemplateRequest.class));
 
-    public static final Version V_5_1_0 = Version.fromId(5010099);
-
     private String name;
 
     private String cause = "";
@@ -473,7 +471,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
         cause = in.readString();
         name = in.readString();
 
-        if (in.getVersion().onOrAfter(V_5_1_0)) {
+        if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
             indexPatterns = in.readList(StreamInput::readString);
         } else {
             indexPatterns = Collections.singletonList(in.readString());
@@ -503,7 +501,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
         super.writeTo(out);
         out.writeString(cause);
         out.writeString(name);
-        if (out.getVersion().onOrAfter(V_5_1_0)) {
+        if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
             out.writeStringList(indexPatterns);
         } else {
             out.writeString(indexPatterns.size() > 0 ? indexPatterns.get(0) : "");

core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java

@@ -49,8 +49,6 @@ import java.util.Set;
 
 public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaData> {
 
-    public static final Version V_5_1_0 = Version.fromId(5010099);
-
     public static final IndexTemplateMetaData PROTO = IndexTemplateMetaData.builder("").build();
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(IndexTemplateMetaData.class));
 
@@ -210,7 +208,7 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
     public IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder(in.readString());
         builder.order(in.readInt());
-        if (in.getVersion().onOrAfter(V_5_1_0)) {
+        if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
             builder.patterns(in.readList(StreamInput::readString));
         } else {
             builder.patterns(Collections.singletonList(in.readString()));
@@ -241,7 +239,7 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeInt(order);
-        if (out.getVersion().onOrAfter(V_5_1_0)) {
+        if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
             out.writeStringList(patterns);
         } else {
             out.writeString(patterns.size() > 0 ? patterns.get(0) : "");

core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.transport;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -68,6 +69,12 @@ public final class TransportAddress implements Writeable {
      * Read from a stream.
      */
     public TransportAddress(StreamInput in) throws IOException {
+        if (in.getVersion().before(Version.V_6_0_0_alpha1)) { // bwc layer for 5.x where we had more than one transport address
+            final short i = in.readShort();
+            if(i != 1) { // we fail hard to ensure nobody tries to use some custom transport address impl even if that is difficult to add
+                throw new AssertionError("illegal transport ID from node of version: " + in.getVersion()  + " got: " + i + " expected: 1");
+            }
+        }
         final int len = in.readByte();
         final byte[] a = new byte[len]; // 4 bytes (IPv4) or 16 bytes (IPv6)
         in.readFully(a);
@@ -78,6 +85,9 @@ public final class TransportAddress implements Writeable {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
+        if (out.getVersion().before(Version.V_6_0_0_alpha1)) {
+            out.writeShort((short)1); // this maps to InetSocketTransportAddress in 5.x
+        }
         byte[] bytes = address.getAddress().getAddress();  // 4 bytes (IPv4) or 16 bytes (IPv6)
         out.writeByte((byte) bytes.length); // 1 byte
         out.write(bytes, 0, bytes.length);

core/src/main/java/org/elasticsearch/transport/TcpTransport.java

@@ -1231,8 +1231,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
                 }
                 streamIn = compressor.streamInput(streamIn);
             }
-            if (version.onOrAfter(getCurrentVersion().minimumCompatibilityVersion()) == false
-                || version.major != getCurrentVersion().major) {
+            if (version.isCompatible(getCurrentVersion()) == false) {
                 throw new IllegalStateException("Received message from unsupported version: [" + version
                     + "] minimal compatible version is: [" + getCurrentVersion().minimumCompatibilityVersion() + "]");
             }

core/src/main/java/org/elasticsearch/transport/TransportService.java

@@ -371,18 +371,13 @@ public class TransportService extends AbstractLifecycleComponent {
 
         if (checkClusterName && !Objects.equals(clusterName, response.clusterName)) {
             throw new IllegalStateException("handshake failed, mismatched cluster name [" + response.clusterName + "] - " + node);
-        } else if (!isVersionCompatible(response.version)) {
+        } else if (response.version.isCompatible((localNode != null ? localNode.getVersion() : Version.CURRENT)) == false) {
             throw new IllegalStateException("handshake failed, incompatible version [" + response.version + "] - " + node);
         }
 
         return response.discoveryNode;
     }
 
-    private boolean isVersionCompatible(Version version) {
-        return version.minimumCompatibilityVersion().equals(
-                localNode != null ? localNode.getVersion().minimumCompatibilityVersion() : Version.CURRENT.minimumCompatibilityVersion());
-    }
-
     static class HandshakeRequest extends TransportRequest {
 
         public static final HandshakeRequest INSTANCE = new HandshakeRequest();

core/src/test/java/org/elasticsearch/VersionTests.java

@@ -134,6 +134,10 @@ public class VersionTests extends ESTestCase {
         assertThat(Version.V_2_2_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0));
         assertThat(Version.V_2_3_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0));
         assertThat(Version.V_5_0_0_alpha1.minimumCompatibilityVersion(), equalTo(Version.V_5_0_0_alpha1));
+        // from 6.0 on we are supporting the latest minor of the previous major... this might fail once we add a new version ie. 5.x is
+        // released since we need to bump the supported minor in Version#minimumCompatibilityVersion()
+        assertThat("did you miss to bump the minor in Version#minimumCompatibilityVersion()",
+            Version.V_6_0_0_alpha1.minimumCompatibilityVersion(), equalTo(VersionUtils.getPreviousVersion(Version.V_6_0_0_alpha1)));
     }
 
     public void testToString() {
@@ -222,7 +226,7 @@ public class VersionTests extends ESTestCase {
                 assertTrue(constantName + " should be final", Modifier.isFinal(versionConstant.getModifiers()));
 
                 Version v = (Version) versionConstant.get(Version.class);
-                logger.info("Checking {}", v);
+                logger.debug("Checking {}", v);
                 assertEquals("Version id " + field.getName() + " does not point to " + constantName, v, Version.fromId(versionId));
                 assertEquals("Version " + constantName + " does not have correct id", versionId, v.id);
                 if (v.major >= 2) {
@@ -277,9 +281,7 @@ public class VersionTests extends ESTestCase {
         assertUnknownVersion(V_20_0_0_UNRELEASED);
         expectThrows(AssertionError.class, () -> assertUnknownVersion(Version.CURRENT));
         assertUnknownVersion(AliasFilter.V_5_1_0); // once we released 5.1.0 and it's added to Version.java we need to remove this constant
-        assertUnknownVersion(IndexTemplateMetaData.V_5_1_0);
         assertUnknownVersion(OsStats.V_5_1_0); // once we released 5.1.0 and it's added to Version.java we need to remove this constant
-        assertUnknownVersion(PutIndexTemplateRequest.V_5_1_0);
         assertUnknownVersion(QueryStringQueryBuilder.V_5_1_0_UNRELEASED);
         assertUnknownVersion(SimpleQueryStringBuilder.V_5_1_0_UNRELEASED);
         // once we released 5.0.0 and it's added to Version.java we need to remove this constant
@@ -291,4 +293,18 @@ public class VersionTests extends ESTestCase {
         assertFalse("Version " + version + " has been releaed don't use a new instance of this version",
             VersionUtils.allVersions().contains(version));
     }
+
+    public void testIsCompatible() {
+        assertTrue(isCompatible(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()));
+        assertTrue(isCompatible(Version.V_5_0_0, Version.V_6_0_0_alpha1));
+        assertFalse(isCompatible(Version.V_2_0_0, Version.V_6_0_0_alpha1));
+        assertFalse(isCompatible(Version.V_2_0_0, Version.V_5_0_0));
+    }
+
+    public boolean isCompatible(Version left, Version right) {
+        boolean result = left.isCompatible(right);
+        assert result == right.isCompatible(left);
+        return result;
+    }
+
 }

qa/backwards-5.0/build.gradle

@@ -18,7 +18,13 @@ integTest {
   cluster {
     numNodes = 2
     numBwcNodes = 1
-    bwcVersion = "6.0.0-alpha1-SNAPSHOT" // this is the same as the current version until we released the first RC
+    bwcVersion = "5.1.0-SNAPSHOT"
     setting 'logger.org.elasticsearch', 'DEBUG'
   }
 }
+
+repositories {
+  maven {
+    url "https://oss.sonatype.org/content/repositories/snapshots/"
+  }
+}

qa/rolling-upgrade/build.gradle

@@ -25,12 +25,8 @@ task oldClusterTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
   cluster {
     distribution = 'zip'
-    // TODO: Right now, this just forms a cluster with the current version of ES,
-    // because we don't support clusters with nodes on different alpha/beta releases of ES.
-    // When the GA is released, we should change the bwcVersion to 5.0.0 and uncomment
-    // numBwcNodes = 2
-    //bwcVersion = '5.0.0-alpha5' // TODO: either randomize, or make this settable with sysprop
-    //numBwcNodes = 2
+    bwcVersion = '5.1.0-SNAPSHOT' // TODO: either randomize, or make this settable with sysprop
+    numBwcNodes = 1
     numNodes = 2
     clusterName = 'rolling-upgrade'
   }
@@ -69,3 +65,9 @@ task integTest {
 test.enabled = false // no unit tests for rolling upgrades, only the rest integration test
 
 check.dependsOn(integTest)
+
+repositories {
+  maven {
+    url "https://oss.sonatype.org/content/repositories/snapshots/"
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/test/cat.templates/10_basic.yaml

@@ -1,8 +1,8 @@
 ---
 "Help":
     - skip:
-        version: " - 5.0.99"
-        reason:  templates were introduced in 5.1.0
+        version: " - 5.99.99"
+        reason:  this uses a new API that has been added in 6.0
     - do:
         cat.templates:
           help: true
@@ -30,9 +30,11 @@
 
 ---
 "Normal templates":
+
     - skip:
-        version: " - 5.0.99"
-        reason:  templates were introduced in 5.1.0
+        version: " - 5.99.99"
+        reason:  this uses a new API that has been added in 6.0
+
     - do:
         indices.put_template:
           name: test
@@ -78,9 +80,11 @@
 
 ---
 "Filtered templates":
+
     - skip:
-        version: " - 5.0.99"
-        reason:  templates were introduced in 5.1.0
+        version: " - 5.99.99"
+        reason:  this uses a new API that has been added in 6.0
+
     - do:
         indices.put_template:
           name: test
@@ -120,8 +124,8 @@
 ---
 "Column headers":
     - skip:
-        version: " - 5.0.99"
-        reason:  templates were introduced in 5.1.0
+        version: " - 5.99.99"
+        reason:  this uses a new API that has been added in 6.0
     - do:
         indices.put_template:
           name: test
@@ -156,8 +160,8 @@
 ---
 "Select columns":
     - skip:
-        version: " - 5.0.99"
-        reason:  templates were introduced in 5.1.0
+        version: " - 5.99.99"
+        reason:  this uses a new API that has been added in 6.0
     - do:
         indices.put_template:
           name: test
@@ -189,8 +193,8 @@
 ---
 "Sort templates":
     - skip:
-        version: " - 5.0.99"
-        reason:  templates were introduced in 5.1.0
+        version: " - 5.99.99"
+        reason:  this uses a new API that has been added in 6.0
     - do:
         indices.put_template:
           name: test
@@ -239,6 +243,9 @@
 
 ---
 "Multiple template":
+    - skip:
+        version: " - 5.99.99"
+        reason:  this uses a new API that has been added in 6.0
     - do:
         indices.put_template:
           name: test_1

rest-api-spec/src/main/resources/rest-api-spec/test/indices.exists_template/10_basic.yaml

@@ -6,6 +6,11 @@ setup:
         ignore: [404]
 ---
 "Test indices.exists_template":
+
+  - skip:
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
+
   - do:
       indices.exists_template:
         name: test

rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_template/10_basic.yaml

@@ -11,6 +11,10 @@ setup:
 ---
 "Get template":
 
+  - skip:
+    version: " - 5.99.99"
+    reason:  this uses a new API that has been added in 6.0
+
   - do:
       indices.get_template:
         name: test
@@ -21,6 +25,10 @@ setup:
 ---
 "Get all templates":
 
+  - skip:
+    version: " - 5.99.99"
+    reason:  this uses a new API that has been added in 6.0
+
   - do:
       indices.put_template:
         name: test2
@@ -38,6 +46,10 @@ setup:
 ---
 "Get template with local flag":
 
+  - skip:
+    version: " - 5.99.99"
+    reason:  this uses a new API that has been added in 6.0
+
   - do:
       indices.get_template:
         name: test
@@ -48,6 +60,10 @@ setup:
 ---
 "Get template with flat settings and master timeout":
 
+  - skip:
+    version: " - 5.99.99"
+    reason:  this uses a new API that has been added in 6.0
+
   - do:
       indices.get_template:
         name: test

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yaml

@@ -1,5 +1,10 @@
 ---
 "Put template":
+
+  - skip:
+    version: " - 5.99.99"
+    reason:  this uses a new API that has been added in 6.0
+
   - do:
       indices.put_template:
         name: test
@@ -19,6 +24,11 @@
 
 ---
 "Put multiple template":
+
+  - skip:
+    version: " - 5.99.99"
+    reason:  this uses a new API that has been added in 6.0
+
   - do:
       indices.put_template:
         name: test
@@ -38,6 +48,11 @@
 
 ---
 "Put template with aliases":
+
+  - skip:
+    version: " - 5.99.99"
+    reason:  this uses a new API that has been added in 6.0
+
   - do:
       indices.put_template:
         name: test
@@ -61,6 +76,11 @@
 
 ---
 "Put template create":
+
+  - skip:
+    version: " - 5.99.99"
+    reason:  this uses a new API that has been added in 6.0
+
   - do:
       indices.put_template:
         name: test
@@ -92,6 +112,11 @@
 
 ---
 "Test Put Versioned Template":
+
+  - skip:
+    version: " - 5.99.99"
+    reason:  this uses a new API that has been added in 6.0
+
   - do:
       indices.put_template:
         name: "my_template"

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java

@@ -100,7 +100,7 @@ public class ClientYamlTestClient {
             Version version = null;
             Version masterVersion = null;
             for (String perNode : split) {
-                final String[] versionAndMaster = perNode.split(" ");
+                final String[] versionAndMaster = perNode.split("\\s+");
                 assert versionAndMaster.length == 2 : "invalid line: " + perNode + " length: " + versionAndMaster.length;
                 final Version currentVersion = Version.fromString(versionAndMaster[0]);
                 final boolean master = versionAndMaster[1].trim().equals("*");