[Bug] Fix mixed cluster support for OpenSearch 2+ (#1191) (#1195)

The version framework only added support for OpenSearch 1.x bwc with legacy
clusters. This commit adds support for v2.0 which will be the last version with
bwc support for legacy clusters (v7.10)

Signed-off-by: Nicholas Walter Knize <nknize@apache.org>

server/src/main/java/org/opensearch/LegacyESVersion.java

@@ -167,7 +167,7 @@ public class LegacyESVersion extends Version {
                             final int minor = Integer.valueOf(fields[2]) * 10000;
                             final int revision = Integer.valueOf(fields[3]) * 100;
                             final int expectedId;
-                            if (fields[1].equals("1")) {
+                            if (major > 0 && major < 6000000) {
                                 expectedId = 0x08000000 ^ (major + minor + revision + 99);
                             } else {
                                 expectedId = (major + minor + revision + 99);

server/src/main/java/org/opensearch/Version.java

@@ -238,8 +238,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
 
     // LegacyESVersion major 7 is equivalent to Version major 1
     public int compareMajor(Version other) {
-        int m = major == 1 ? 7 : major;
+        int m = major == 1 ? 7 : major == 2 ? 8 : major;
-        int om = other.major == 1 ? 7 : other.major;
+        int om = other.major == 1 ? 7 : other.major == 2 ? 8 : other.major;
         return Integer.compare(m, om);
     }
 
@@ -293,6 +293,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
         } else if (major == 6) {
             // force the minimum compatibility for version 6 to 5.6 since we don't reference version 5 anymore
             return Version.fromId(5060099);
+        } else if (major == 2) {
+            return LegacyESVersion.V_7_10_0;
         } else if (major >= 7) {
             // all major versions from 7 onwards are compatible with last minor series of the previous major
             Version bwcVersion = null;
@@ -339,6 +341,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
             bwcMajor = 2; // we jumped from 2 to 5
         } else if (major == 7 || major == 1) {
             return LegacyESVersion.V_6_0_0_beta1;
+        } else if (major == 2) {
+            return LegacyESVersion.V_7_0_0;
         } else {
             bwcMajor = major - 1;
         }
@@ -354,8 +358,20 @@ public class Version implements Comparable<Version>, ToXContentFragment {
             && version.onOrAfter(minimumCompatibilityVersion());
 
         // OpenSearch version 1 is the functional equivalent of predecessor version 7
-        int a = major == 1 ? 7 : major;
+        // OpenSearch version 2 is the functional equivalent of predecessor unreleased version "8"
-        int b = version.major == 1 ? 7 : version.major;
+        // todo refactor this logic after removing deprecated features
+        int a = major;
+        if (major == 1) {
+            a = 7;
+        } else if (major == 2) {
+            a = 8;
+        }
+        int b = version.major;
+        if (version.major == 1) {
+            b = 7;
+        } else if (version.major == 2) {
+            b = 8;
+        }
 
         assert compatible == false || Math.max(a, b) - Math.min(a, b) <= 1;
         return compatible;

server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java

@@ -367,9 +367,8 @@ public class JoinTaskExecutor implements ClusterStateTaskExecutor<JoinTaskExecut
      * version mode
      **/
     public static void ensureMajorVersionBarrier(Version joiningNodeVersion, Version minClusterNodeVersion) {
-        final byte jnMajor = joiningNodeVersion.major == 1 ? 7 : joiningNodeVersion.major;
         final byte clusterMajor = minClusterNodeVersion.major == 1? 7: minClusterNodeVersion.major;
-        if (jnMajor < clusterMajor) {
+        if (joiningNodeVersion.compareMajor(minClusterNodeVersion) < 0) {
             throw new IllegalStateException("node version [" + joiningNodeVersion + "] is not supported. " +
                 "All nodes in the cluster are of a higher major [" + clusterMajor + "].");
         }

server/src/test/java/org/opensearch/VersionTests.java

@@ -211,6 +211,48 @@ public class VersionTests extends OpenSearchTestCase {
         assertEquals(0, LegacyESVersion.V_7_0_0.minimumCompatibilityVersion().revision);
     }
 
+    /** test opensearch min wire compatibility */
+    public void testOpenSearchMinCompatVersion() {
+        Version opensearchVersion = VersionUtils.randomOpenSearchVersion(random());
+        // opensearch 1.x minCompat is Legacy 6.8.0
+        // opensearch 2.x minCompat is Legacy 7.10.0
+        // opensearch 3.x minCompat is 1.{last minor version}.0
+        // until 3.0 is staged the following line will only return legacy versions
+        List<Version> candidates = opensearchVersion.major >= 3 ? VersionUtils.allOpenSearchVersions() : VersionUtils.allLegacyVersions();
+        int opensearchMajor = opensearchVersion.major;
+        int major = opensearchMajor - 1;
+        if (opensearchMajor == 1) {
+            major = 7;
+        } else if (opensearchMajor == 2) {
+            major = 8;
+        }
+        assertEquals(VersionUtils.lastFirstReleasedMinorFromMajor(candidates, major - 1),
+            opensearchVersion.minimumCompatibilityVersion());
+    }
+
+    /** test opensearch min index compatibility */
+    public void testOpenSearchMinIndexCompatVersion() {
+        Version opensearchVersion = VersionUtils.randomOpenSearchVersion(random());
+        // opensearch 1.x minIndexCompat is Legacy 6.8.0
+        // opensearch 2.x minCompat is Legacy 7.10.0
+        // opensearch 3.x minCompat is 1.{last minor version}.0
+        // until 3.0 is staged the following line will only return legacy versions
+        List<Version> candidates = opensearchVersion.major >= 3 ? VersionUtils.allOpenSearchVersions() : VersionUtils.allLegacyVersions();
+        int opensearchMajor = opensearchVersion.major;
+        int major = opensearchMajor - 1;
+        if (opensearchMajor == 1) {
+            major = 7;
+        } else if (opensearchMajor == 2) {
+            major = 8;
+        }
+        Version expected = VersionUtils.getFirstVersionOfMajor(candidates, major - 1);
+        Version actual = opensearchVersion.minimumIndexCompatibilityVersion();
+        // since some legacy versions still support build (alpha, beta, RC) we check major minor revision only
+        assertEquals(expected.major, actual.major);
+        assertEquals(expected.minor, actual.minor);
+        assertEquals(expected.revision, actual.revision);
+    }
+
     public void testToString() {
         // with 2.0.beta we lowercase
         assertEquals("2.0.0-beta1", LegacyESVersion.fromString("2.0.0-beta1").toString());

server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java

@@ -88,8 +88,7 @@ public class JoinTaskExecutorTests extends OpenSearchTestCase {
         Settings.builder().build();
         Metadata.Builder metaBuilder = Metadata.builder();
         IndexMetadata indexMetadata = IndexMetadata.builder("test")
-            .settings(settings(VersionUtils.getPreviousVersion(Version.CURRENT
+            .settings(settings(Version.fromString("5.8.0")))
-                .minimumIndexCompatibilityVersion())))
             .numberOfShards(1)
             .numberOfReplicas(1).build();
         metaBuilder.put(indexMetadata, false);

test/framework/src/main/java/org/opensearch/test/VersionUtils.java

@@ -39,6 +39,7 @@ import org.opensearch.common.collect.Tuple;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -149,6 +150,8 @@ public class VersionUtils {
     private static final List<Version> RELEASED_VERSIONS;
     private static final List<Version> UNRELEASED_VERSIONS;
     private static final List<Version> ALL_VERSIONS;
+    private static final List<Version> ALL_OPENSEARCH_VERSIONS;
+    private static final List<Version> ALL_LEGACY_VERSIONS;
 
     static {
         Tuple<List<Version>, List<Version>> versions = resolveReleasedVersions(Version.CURRENT, LegacyESVersion.class);
@@ -159,6 +162,9 @@ public class VersionUtils {
         allVersions.addAll(UNRELEASED_VERSIONS);
         Collections.sort(allVersions);
         ALL_VERSIONS = Collections.unmodifiableList(allVersions);
+        // @todo remove this when legacy support is no longer needed
+        ALL_OPENSEARCH_VERSIONS = ALL_VERSIONS.stream().filter(v -> v.major < 6).collect(Collectors.toList());
+        ALL_LEGACY_VERSIONS = ALL_VERSIONS.stream().filter(v -> v.major >= 6).collect(Collectors.toList());
     }
 
     /**
@@ -182,6 +188,16 @@ public class VersionUtils {
         return ALL_VERSIONS;
     }
 
+    /** Returns an immutable, sorted list containing all opensearch versions; released and unreleased */
+    public static List<Version> allOpenSearchVersions() {
+        return ALL_OPENSEARCH_VERSIONS;
+    }
+
+    /** Returns an immutable, sorted list containing all legacy versions; released and unreleased */
+    public static List<Version> allLegacyVersions() {
+        return ALL_LEGACY_VERSIONS;
+    }
+
     /**
      * Get the released version before {@code version}.
      */
@@ -224,11 +240,35 @@ public class VersionUtils {
         return RELEASED_VERSIONS.get(0);
     }
 
+    public static Version getFirstVersionOfMajor(List<Version> versions, int major) {
+        Map<Integer, List<Version>> majorVersions = versions.stream().collect(Collectors.groupingBy(v -> (int)v.major));
+        return majorVersions.get(major).get(0);
+    }
+
     /** Returns a random {@link Version} from all available versions. */
     public static Version randomVersion(Random random) {
         return ALL_VERSIONS.get(random.nextInt(ALL_VERSIONS.size()));
     }
 
+    /**
+     * Return a random {@link Version} from all available opensearch versions.
+     **/
+    public static Version randomOpenSearchVersion(Random random) {
+        return ALL_OPENSEARCH_VERSIONS.get(random.nextInt(ALL_OPENSEARCH_VERSIONS.size()));
+    }
+
+    /** Returns the first released (e.g., patch version 0) {@link Version} of the last minor from the requested major version
+     *  e.g., for version 1.0.0 this would be legacy version (7.10.0); the first release (patch 0), of the last
+     *  minor (for 7.x that is minor version 10) for the desired major version (7)
+     **/
+    public static Version lastFirstReleasedMinorFromMajor(List<Version> allVersions, int major) {
+        Map<Integer, List<Version>> majorVersions = allVersions.stream().collect(Collectors.groupingBy(v -> (int)v.major));
+        Map<Integer, List<Version>> groupedByMinor = majorVersions.get(major).stream().collect(
+            Collectors.groupingBy(v -> (int)v.minor));
+        List<Version> candidates = Collections.max(groupedByMinor.entrySet(), Comparator.comparing(Map.Entry::getKey)).getValue();
+        return candidates.get(0);
+    }
+
     /** Returns a random {@link Version} from all available versions, that is compatible with the given version. */
     public static Version randomCompatibleVersion(Random random, Version version) {
         final List<Version> compatible = ALL_VERSIONS.stream().filter(version::isCompatible).collect(Collectors.toList());