Prepare for bump to 6.0.1 on the master branch (#27391)

An assortment of fixes, particularly to version number calculations, in preparation for the bump to 6.0.1.

build.gradle

@@ -81,6 +81,7 @@ List<Version> versions = []
 // keep track of the previous major version's last minor, so we know where wire compat begins
 int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
 int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
+int prevBugfixIndex = -1 // index in the versions list of the last bugfix release from the prev major
 for (String line : versionLines) {
   /* Note that this skips alphas and betas which is fine because they aren't
    * compatible with anything. */
@@ -108,12 +109,19 @@ for (String line : versionLines) {
         lastPrevMinor = minor
       }
     }
+    if (major == prevMajor) {
+      prevBugfixIndex = versions.size() - 1
+    }
   }
 }
 if (versions.toSorted { it.id } != versions) {
   println "Versions: ${versions}"
   throw new GradleException("Versions.java contains out of order version constants")
 }
+if (prevBugfixIndex != -1) {
+  versions[prevBugfixIndex] = new Version(versions[prevBugfixIndex].major, versions[prevBugfixIndex].minor,
+                                          versions[prevBugfixIndex].bugfix, versions[prevBugfixIndex].suffix, true)
+}
 if (currentVersion.bugfix == 0) {
   // If on a release branch, after the initial release of that branch, the bugfix version will
   // be bumped, and will be != 0. On master and N.x branches, we want to test against the
@@ -262,6 +270,11 @@ subprojects {
       ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
       ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
     }
+  } else if (indexCompatVersions[-2].snapshot) {
+    /* This is a terrible hack for the bump to 6.0.1 which will be fixed by #27397 */
+    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
+    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
+    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
   }
   project.afterEvaluate {
     configurations.all {

core/src/main/java/org/elasticsearch/Version.java

@@ -28,6 +28,11 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
 import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 
 public class Version implements Comparable<Version> {
     /*
@@ -363,19 +368,23 @@ public class Version implements Comparable<Version> {
      * is a beta or RC release then the version itself is returned.
      */
     public Version minimumCompatibilityVersion() {
-        final int bwcMajor;
+        if (major >= 6) {
-        final int bwcMinor;
+            // all major versions from 6 onwards are compatible with last minor series of the previous major
-        // TODO: remove this entirely, making it static for each version
+            final List<Version> declaredVersions = getDeclaredVersions(getClass());
-        if (major == 6) { // we only specialize for current major here
+            Version bwcVersion = null;
-            bwcMajor = Version.V_5_6_0.major;
+            for (int i = declaredVersions.size() - 1; i >= 0; i--) {
-            bwcMinor = Version.V_5_6_0.minor;
+                final Version candidateVersion = declaredVersions.get(i);
-        } else if (major == 7) { // we only specialize for current major here
+                if (candidateVersion.major == major - 1 && candidateVersion.isRelease() && after(candidateVersion)) {
-            return V_6_1_0;
+                    if (bwcVersion != null && candidateVersion.minor < bwcVersion.minor) {
-        } else {
+                        break;
-            bwcMajor = major;
-            bwcMinor = 0;
                     }
-        return Version.min(this, fromId(bwcMajor * 1000000 + bwcMinor * 10000 + 99));
+                    bwcVersion = candidateVersion;
+                }
+            }
+            return bwcVersion == null ? this : bwcVersion;
+        }
+
+        return Version.min(this, fromId((int) major * 1000000 + 0 * 10000 + 99));
     }
 
     /**
@@ -485,4 +494,34 @@ public class Version implements Comparable<Version> {
     public boolean isRelease() {
         return build == 99;
     }
+
+    /**
+     * Extracts a sorted list of declared version constants from a class.
+     * The argument would normally be Version.class but is exposed for
+     * testing with other classes-containing-version-constants.
+     */
+    public static List<Version> getDeclaredVersions(final Class<?> versionClass) {
+        final Field[] fields = versionClass.getFields();
+        final List<Version> versions = new ArrayList<>(fields.length);
+        for (final Field field : fields) {
+            final int mod = field.getModifiers();
+            if (false == Modifier.isStatic(mod) && Modifier.isFinal(mod) && Modifier.isPublic(mod)) {
+                continue;
+            }
+            if (field.getType() != Version.class) {
+                continue;
+            }
+            if ("CURRENT".equals(field.getName())) {
+                continue;
+            }
+            assert field.getName().matches("V(_\\d+)+(_(alpha|beta|rc)\\d+)?") : field.getName();
+            try {
+                versions.add(((Version) field.get(null)));
+            } catch (final IllegalAccessException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        Collections.sort(versions);
+        return versions;
+    }
 }

core/src/test/java/org/elasticsearch/VersionTests.java

@@ -337,7 +337,7 @@ public class VersionTests extends ESTestCase {
         assertTrue(isCompatible(Version.V_5_6_0, Version.V_6_0_0_alpha2));
         assertFalse(isCompatible(Version.fromId(2000099), Version.V_6_0_0_alpha2));
         assertFalse(isCompatible(Version.fromId(2000099), Version.V_5_0_0));
-        assertTrue(isCompatible(Version.fromString("6.1.0"), Version.fromString("7.0.0")));
+        assertFalse(isCompatible(Version.fromString("6.0.0"), Version.fromString("7.0.0")));
         assertFalse(isCompatible(Version.fromString("6.0.0-alpha1"), Version.fromString("7.0.0")));
         assertFalse("only compatible with the latest minor",
             isCompatible(VersionUtils.getPreviousMinorVersion(), Version.fromString("7.0.0")));

test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java

@@ -23,10 +23,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -49,72 +46,64 @@ public class VersionUtils {
      * guarantees in v1 and versions without the guranteees in v2
      */
     static Tuple<List<Version>, List<Version>> resolveReleasedVersions(Version current, Class<?> versionClass) {
-        Field[] fields = versionClass.getFields();
+        List<Version> versions = Version.getDeclaredVersions(versionClass);
-        List<Version> versions = new ArrayList<>(fields.length);
-        for (final Field field : fields) {
-            final int mod = field.getModifiers();
-            if (false == Modifier.isStatic(mod) && Modifier.isFinal(mod) && Modifier.isPublic(mod)) {
-                continue;
-            }
-            if (field.getType() != Version.class) {
-                continue;
-            }
-            if ("CURRENT".equals(field.getName())) {
-                continue;
-            }
-            assert field.getName().matches("V(_\\d+)+(_(alpha|beta|rc)\\d+)?") : field.getName();
-            try {
-                versions.add(((Version) field.get(null)));
-            } catch (final IllegalAccessException e) {
-                throw new RuntimeException(e);
-            }
-        }
-        Collections.sort(versions);
         Version last = versions.remove(versions.size() - 1);
         assert last.equals(current) : "The highest version must be the current one "
-            + "but was [" + versions.get(versions.size() - 1) + "] and current was [" + current + "]";
+            + "but was [" + last + "] and current was [" + current + "]";
 
-        if (current.revision != 0) {
+        /* In the 5.x series prior to 5.6, unreleased version constants had an
-            /* If we are in a stable branch there should be no unreleased version constants
+         * `_UNRELEASED` suffix, and when making the first release on a minor release
-             * because we don't expect to release any new versions in older branches. If there
+         * branch the last, unreleased, version constant from the previous minor branch
-             * are extra constants then gradle will yell about it. */
+         * was dropped. After 5.6, there is no `_UNRELEASED` suffix on version constants'
+         * names and, additionally, they are not dropped when a new minor release branch
+         * starts.
+         *
+         * This means that in 6.x and later series the last release _in each
+         * minor branch_ is unreleased, whereas in 5.x it's more complicated: There were
+         * (sometimes, and sometimes multiple) minor branches containing no releases, each
+         * of which contains a single version constant of the form 5.n.0, and these
+         * branches always followed a branch that _did_ contain a version of the
+         * form 5.m.p (p>0). All versions strictly before the last 5.m version are released,
+         * and all other 5.* versions are unreleased.
+         */
+
+        if (current.major == 5 && current.revision != 0) {
+            /* The current (i.e. latest) version is 5.a.b, b nonzero, which
+             * means that all other versions are released. */
             return new Tuple<>(unmodifiableList(versions), singletonList(current));
         }
 
-        /* If we are on a patch release then we know that at least the version before the
+        final List<Version> unreleased = new ArrayList<>();
-         * current one is unreleased. If it is released then gradle would be complaining. */
+        unreleased.add(current);
-        int unreleasedIndex = versions.size() - 1;
+        Version prevConsideredVersion = current;
-        while (true) {
+
-            if (unreleasedIndex < 0) {
+        for (int i = versions.size() - 1; i >= 0; i--) {
-                throw new IllegalArgumentException("Couldn't find first non-alpha release");
+            Version currConsideredVersion = versions.get(i);
-            }
+            if (currConsideredVersion.major == 5) {
-            /* We don't support backwards compatibility for alphas, betas, and rcs. But
+                unreleased.add(currConsideredVersion);
-             * they were released so we add them to the released list. Usually this doesn't
+                versions.remove(i);
-             * matter to consumers, but consumers that do care should filter non-release
+                if (currConsideredVersion.revision != 0) {
-             * versions. */
+                    /* Currently considering the latest version in the 5.x series,
-            if (versions.get(unreleasedIndex).isRelease()) {
+                     * which is (a) unreleased and (b) the only such. So we're done. */
                     break;
                 }
-            unreleasedIndex--;
+                /* ... else we're on a version of the form 5.n.0, and have not yet
+                 * considered a version of the form 5.n.m (m>0), so this entire branch
+                 * is unreleased, so carry on looking for a branch containing releases.
+                 */
+            } else if (currConsideredVersion.major != prevConsideredVersion.major
+                || currConsideredVersion.minor != prevConsideredVersion.minor) {
+                /* Have moved to the end of a new minor branch, so this is
+                 * an unreleased version. */
+                unreleased.add(currConsideredVersion);
+                versions.remove(i);
+            }
+            prevConsideredVersion = currConsideredVersion;
+
         }
 
-        Version unreleased = versions.remove(unreleasedIndex);
+        Collections.reverse(unreleased);
-        if (unreleased.revision == 0) {
+        return new Tuple<>(unmodifiableList(versions), unmodifiableList(unreleased));
-            /*
-             * If the last unreleased version is itself a patch release then Gradle enforces that there is yet another unreleased version
-             * before that. However, we have to skip alpha/betas/RCs too (e.g., consider when the version constants are ..., 5.6.3, 5.6.4,
-             * 6.0.0-alpha1, ..., 6.0.0-rc1, 6.0.0-rc2, 6.0.0, 6.1.0 on the 6.x branch. In this case, we will have pruned 6.0.0 and 6.1.0 as
-             * unreleased versions, but we also need to prune 5.6.4. At this point though, unreleasedIndex will be pointing to 6.0.0-rc2, so
-             * we have to skip backwards until we find a non-alpha/beta/RC again. Then we can prune that version as an unreleased version
-             * too.
-             */
-            do {
-                unreleasedIndex--;
-            } while (versions.get(unreleasedIndex).isRelease() == false);
-            Version earlierUnreleased = versions.remove(unreleasedIndex);
-            return new Tuple<>(unmodifiableList(versions), unmodifiableList(Arrays.asList(earlierUnreleased, unreleased, current)));
-        }
-        return new Tuple<>(unmodifiableList(versions), unmodifiableList(Arrays.asList(unreleased, current)));
     }
 
     private static final List<Version> RELEASED_VERSIONS;

test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java

@@ -101,7 +101,7 @@ public class VersionUtilsTests extends ESTestCase {
         assertEquals(unreleased, VersionUtils.randomVersionBetween(random(), unreleased, unreleased));
     }
 
-    static class TestReleaseBranch {
+    public static class TestReleaseBranch {
         public static final Version V_5_3_0 = Version.fromString("5.3.0");
         public static final Version V_5_3_1 = Version.fromString("5.3.1");
         public static final Version V_5_3_2 = Version.fromString("5.3.2");
@@ -118,7 +118,7 @@ public class VersionUtilsTests extends ESTestCase {
         assertEquals(singletonList(TestReleaseBranch.V_5_4_1), unreleased);
     }
 
-    static class TestStableBranch {
+    public static class TestStableBranch {
         public static final Version V_5_3_0 = Version.fromString("5.3.0");
         public static final Version V_5_3_1 = Version.fromString("5.3.1");
         public static final Version V_5_3_2 = Version.fromString("5.3.2");
@@ -134,7 +134,7 @@ public class VersionUtilsTests extends ESTestCase {
         assertEquals(Arrays.asList(TestStableBranch.V_5_3_2, TestStableBranch.V_5_4_0), unreleased);
     }
 
-    static class TestStableBranchBehindStableBranch {
+    public static class TestStableBranchBehindStableBranch {
         public static final Version V_5_3_0 = Version.fromString("5.3.0");
         public static final Version V_5_3_1 = Version.fromString("5.3.1");
         public static final Version V_5_3_2 = Version.fromString("5.3.2");
@@ -142,7 +142,7 @@ public class VersionUtilsTests extends ESTestCase {
         public static final Version V_5_5_0 = Version.fromString("5.5.0");
         public static final Version CURRENT = V_5_5_0;
     }
-    public void testResolveReleasedVersionsForStableBtranchBehindStableBranch() {
+    public void testResolveReleasedVersionsForStableBranchBehindStableBranch() {
         Tuple<List<Version>, List<Version>> t = VersionUtils.resolveReleasedVersions(TestStableBranchBehindStableBranch.CURRENT,
                 TestStableBranchBehindStableBranch.class);
         List<Version> released = t.v1();
@@ -152,7 +152,7 @@ public class VersionUtilsTests extends ESTestCase {
                 TestStableBranchBehindStableBranch.V_5_5_0), unreleased);
     }
 
-    static class TestUnstableBranch {
+    public static class TestUnstableBranch {
         public static final Version V_5_3_0 = Version.fromString("5.3.0");
         public static final Version V_5_3_1 = Version.fromString("5.3.1");
         public static final Version V_5_3_2 = Version.fromString("5.3.2");
@@ -173,6 +173,87 @@ public class VersionUtilsTests extends ESTestCase {
         assertEquals(Arrays.asList(TestUnstableBranch.V_5_3_2, TestUnstableBranch.V_5_4_0, TestUnstableBranch.V_6_0_0_beta1), unreleased);
     }
 
+    public static class TestNewMajorRelease {
+        public static final Version V_5_6_0 = Version.fromString("5.6.0");
+        public static final Version V_5_6_1 = Version.fromString("5.6.1");
+        public static final Version V_5_6_2 = Version.fromString("5.6.2");
+        public static final Version V_6_0_0_alpha1 = Version.fromString("6.0.0-alpha1");
+        public static final Version V_6_0_0_alpha2 = Version.fromString("6.0.0-alpha2");
+        public static final Version V_6_0_0_beta1 = Version.fromString("6.0.0-beta1");
+        public static final Version V_6_0_0_beta2 = Version.fromString("6.0.0-beta2");
+        public static final Version V_6_0_0 = Version.fromString("6.0.0");
+        public static final Version V_6_0_1 = Version.fromString("6.0.1");
+        public static final Version CURRENT = V_6_0_1;
+    }
+
+    public void testResolveReleasedVersionsAtNewMajorRelease() {
+        Tuple<List<Version>, List<Version>> t = VersionUtils.resolveReleasedVersions(TestNewMajorRelease.CURRENT,
+            TestNewMajorRelease.class);
+        List<Version> released = t.v1();
+        List<Version> unreleased = t.v2();
+        assertEquals(Arrays.asList(TestNewMajorRelease.V_5_6_0, TestNewMajorRelease.V_5_6_1,
+            TestNewMajorRelease.V_6_0_0_alpha1, TestNewMajorRelease.V_6_0_0_alpha2,
+            TestNewMajorRelease.V_6_0_0_beta1, TestNewMajorRelease.V_6_0_0_beta2,
+            TestNewMajorRelease.V_6_0_0), released);
+        assertEquals(Arrays.asList(TestNewMajorRelease.V_5_6_2, TestNewMajorRelease.V_6_0_1), unreleased);
+    }
+
+    public static class TestVersionBumpIn6x {
+        public static final Version V_5_6_0 = Version.fromString("5.6.0");
+        public static final Version V_5_6_1 = Version.fromString("5.6.1");
+        public static final Version V_5_6_2 = Version.fromString("5.6.2");
+        public static final Version V_6_0_0_alpha1 = Version.fromString("6.0.0-alpha1");
+        public static final Version V_6_0_0_alpha2 = Version.fromString("6.0.0-alpha2");
+        public static final Version V_6_0_0_beta1 = Version.fromString("6.0.0-beta1");
+        public static final Version V_6_0_0_beta2 = Version.fromString("6.0.0-beta2");
+        public static final Version V_6_0_0 = Version.fromString("6.0.0");
+        public static final Version V_6_0_1 = Version.fromString("6.0.1");
+        public static final Version V_6_1_0 = Version.fromString("6.1.0");
+        public static final Version CURRENT = V_6_1_0;
+    }
+
+    public void testResolveReleasedVersionsAtVersionBumpIn6x() {
+        Tuple<List<Version>, List<Version>> t = VersionUtils.resolveReleasedVersions(TestVersionBumpIn6x.CURRENT,
+            TestVersionBumpIn6x.class);
+        List<Version> released = t.v1();
+        List<Version> unreleased = t.v2();
+        assertEquals(Arrays.asList(TestVersionBumpIn6x.V_5_6_0, TestVersionBumpIn6x.V_5_6_1,
+            TestVersionBumpIn6x.V_6_0_0_alpha1, TestVersionBumpIn6x.V_6_0_0_alpha2,
+            TestVersionBumpIn6x.V_6_0_0_beta1, TestVersionBumpIn6x.V_6_0_0_beta2,
+            TestVersionBumpIn6x.V_6_0_0), released);
+        assertEquals(Arrays.asList(TestVersionBumpIn6x.V_5_6_2, TestVersionBumpIn6x.V_6_0_1, TestVersionBumpIn6x.V_6_1_0), unreleased);
+    }
+
+    public static class TestNewMinorBranchIn6x {
+        public static final Version V_5_6_0 = Version.fromString("5.6.0");
+        public static final Version V_5_6_1 = Version.fromString("5.6.1");
+        public static final Version V_5_6_2 = Version.fromString("5.6.2");
+        public static final Version V_6_0_0_alpha1 = Version.fromString("6.0.0-alpha1");
+        public static final Version V_6_0_0_alpha2 = Version.fromString("6.0.0-alpha2");
+        public static final Version V_6_0_0_beta1 = Version.fromString("6.0.0-beta1");
+        public static final Version V_6_0_0_beta2 = Version.fromString("6.0.0-beta2");
+        public static final Version V_6_0_0 = Version.fromString("6.0.0");
+        public static final Version V_6_0_1 = Version.fromString("6.0.1");
+        public static final Version V_6_1_0 = Version.fromString("6.1.0");
+        public static final Version V_6_1_1 = Version.fromString("6.1.1");
+        public static final Version V_6_1_2 = Version.fromString("6.1.2");
+        public static final Version V_6_2_0 = Version.fromString("6.2.0");
+        public static final Version CURRENT = V_6_2_0;
+    }
+
+    public void testResolveReleasedVersionsAtNewMinorBranchIn6x() {
+        Tuple<List<Version>, List<Version>> t = VersionUtils.resolveReleasedVersions(TestNewMinorBranchIn6x.CURRENT,
+            TestNewMinorBranchIn6x.class);
+        List<Version> released = t.v1();
+        List<Version> unreleased = t.v2();
+        assertEquals(Arrays.asList(TestNewMinorBranchIn6x.V_5_6_0, TestNewMinorBranchIn6x.V_5_6_1,
+            TestNewMinorBranchIn6x.V_6_0_0_alpha1, TestNewMinorBranchIn6x.V_6_0_0_alpha2,
+            TestNewMinorBranchIn6x.V_6_0_0_beta1, TestNewMinorBranchIn6x.V_6_0_0_beta2,
+            TestNewMinorBranchIn6x.V_6_0_0, TestNewMinorBranchIn6x.V_6_1_0, TestNewMinorBranchIn6x.V_6_1_1), released);
+        assertEquals(Arrays.asList(TestNewMinorBranchIn6x.V_5_6_2, TestNewMinorBranchIn6x.V_6_0_1,
+            TestNewMinorBranchIn6x.V_6_1_2, TestNewMinorBranchIn6x.V_6_2_0), unreleased);
+    }
+
     /**
      * Tests that {@link Version#minimumCompatibilityVersion()} and {@link VersionUtils#allReleasedVersions()}
      * agree with the list of wire and index compatible versions we build in gradle.
@@ -181,8 +262,9 @@ public class VersionUtilsTests extends ESTestCase {
         // First check the index compatible versions
         VersionsFromProperty indexCompatible = new VersionsFromProperty("tests.gradle_index_compat_versions");
         List<Version> released = VersionUtils.allReleasedVersions().stream()
-                // Java lists some non-index compatible versions but gradle does not include them.
+                /* Java lists all versions from the 5.x series onwards, but we only want to consider
-                .filter(v -> v.major == Version.CURRENT.major || v.major == Version.CURRENT.major - 1)
+                 * ones that we're supposed to be compatible with. */
+                .filter(v -> v.onOrAfter(Version.CURRENT.minimumIndexCompatibilityVersion()))
                 /* Gradle will never include *released* alphas or betas because it will prefer
                  * the unreleased branch head. Gradle is willing to use branch heads that are
                  * beta or rc so that we have *something* to test against even though we
@@ -199,6 +281,9 @@ public class VersionUtilsTests extends ESTestCase {
                 /* Gradle skips the current version because being backwards compatible
                  * with yourself is implied. Java lists the version because it is useful. */
                 .filter(v -> v != Version.CURRENT)
+                /* Java lists all versions from the 5.x series onwards, but we only want to consider
+                 * ones that we're supposed to be compatible with. */
+                .filter(v -> v.onOrAfter(Version.CURRENT.minimumIndexCompatibilityVersion()))
                 /* Note that gradle skips alphas because they don't have any backwards
                  * compatibility guarantees but keeps the last beta and rc in a branch
                  * on when there are only betas an RCs in that branch so that we have