diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 87141487126..950885a0254 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -155,6 +155,11 @@ Bug Fixes
 * LUCENE-5934: Fix backwards compatibility for 4.0 indexes.
   (Ian Lea, Uwe Schindler, Robert Muir, Ryan Ernst)
 
+Tests
+
+* LUCENE-5936: Add backcompat checks to verify what is tested matches known versions
+  (Ryan Ernst)
+
 Build
 
 * LUCENE-5909: Smoke tester now has better command line parsing and
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index a05dcb9b70c..47e9901b8f2 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -21,12 +21,17 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
@@ -63,7 +68,6 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.Version;
@@ -296,7 +300,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
       oldIndexDirs.put(name, newFSDirectory(dir));
     }
   }
-  
+
   @AfterClass
   public static void afterClass() throws Exception {
     for (Directory d : oldIndexDirs.values()) {
@@ -304,6 +308,125 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     }
     oldIndexDirs = null;
   }
+
+  public void testAllVersionHaveCfsAndNocfs() {
+    // ensure all tested versions with cfs also have nocfs
+    String[] files = new String[oldNames.length];
+    System.arraycopy(oldNames, 0, files, 0, oldNames.length);
+    Arrays.sort(files);
+    String prevFile = "";
+    for (String file : files) {
+      if (prevFile.endsWith(".cfs")) {
+        String prefix = prevFile.replace(".cfs", "");
+        assertEquals("Missing .nocfs for backcompat index " + prefix, prefix + ".nocfs", file);
+      }
+    }
+  }
+
+  public void testAllVersionsTested() throws Exception {
+    Pattern constantPattern = Pattern.compile("LUCENE_(\\d+)_(\\d+)_(\\d+)(_ALPHA|_BETA)?");
+    // find the unique versions according to Version.java
+    List<String> expectedVersions = new ArrayList<>();
+    int lastPrevMinorIndex = -1;
+    Version lastPrevMajorVersion = null;
+    for (java.lang.reflect.Field field : Version.class.getDeclaredFields()) {
+      if (Modifier.isStatic(field.getModifiers()) && field.getType() == Version.class) {
+        Version v = (Version)field.get(Version.class);
+        if (v.equals(Version.LATEST)) continue;
+
+        Matcher constant = constantPattern.matcher(field.getName());
+        if (constant.matches() == false) continue;
+
+        if (v.major == Version.LATEST.major - 1 &&
+            (lastPrevMajorVersion == null || v.onOrAfter(lastPrevMajorVersion))) {
+          lastPrevMajorVersion = v;
+          lastPrevMinorIndex = expectedVersions.size();
+        }
+
+        String major = constant.group(1);
+        String minor = constant.group(2);
+        String bugfix = constant.group(3);
+        if (bugfix.equals("0")) {
+          bugfix = "";
+        }
+        String prerelease = constant.group(4);
+        if (prerelease != null) {
+          if (prerelease.equals("_ALPHA")) {
+            prerelease = "a";
+          } else { // _BETA
+            prerelease = "b";
+          }
+        } else {
+          prerelease = "";
+        }
+        expectedVersions.add(major + minor + bugfix + prerelease + ".cfs");
+      }
+    }
+    if (Version.LATEST.minor == 0 && Version.LATEST.bugfix == 0 && Version.LATEST.prerelease == 0) {
+      // we are on trunk (latest is a first major release) so the last minor index
+      // for the previous major version is also not yet tested
+      assertNotNull(lastPrevMajorVersion);
+      expectedVersions.remove(lastPrevMinorIndex);
+    }
+    Collections.sort(expectedVersions);
+
+    // find what versions we are testing
+    List<String> testedVersions = new ArrayList<>();
+    for (String testedVersion : oldNames) {
+      if (testedVersion.endsWith(".cfs") == false) continue;
+      testedVersions.add(testedVersion);
+    }
+    Collections.sort(testedVersions);
+
+
+    int i = 0;
+    int j = 0;
+    List<String> missingFiles = new ArrayList<>();
+    List<String> extraFiles = new ArrayList<>();
+    while (i < expectedVersions.size() && j < testedVersions.size()) {
+      String expectedVersion = expectedVersions.get(i);
+      String testedVersion = testedVersions.get(j);
+      int compare = expectedVersion.compareTo(testedVersion);
+      if (compare == 0) { // equal, we can move on
+        ++i;
+        ++j;
+      } else if (compare < 0) { // didn't find test for version constant
+        missingFiles.add(expectedVersion);
+        ++i;
+      } else { // extra test file
+        extraFiles.add(testedVersion);
+        ++j;
+      }
+    }
+    while (i < expectedVersions.size()) {
+      missingFiles.add(expectedVersions.get(i));
+      ++i;
+    }
+    while (j < testedVersions.size()) {
+      missingFiles.add(testedVersions.get(j));
+      ++j;
+    }
+
+    if (missingFiles.isEmpty() && extraFiles.isEmpty()) {
+      // success
+      return;
+    }
+
+    StringBuffer msg = new StringBuffer();
+    if (missingFiles.isEmpty() == false) {
+      msg.append("Missing backcompat test files:\n");
+      for (String missingFile : missingFiles) {
+        msg.append("  " + missingFile + "\n");
+      }
+    }
+    if (extraFiles.isEmpty() == false) {
+      msg.append("Extra backcompat test files:\n");
+      for (String extraFile : extraFiles) {
+        msg.append("  " + extraFile + "\n");
+      }
+    }
+    fail(msg.toString());
+  }
   
   /** This test checks that *only* IndexFormatTooOldExceptions are thrown when you open and operate on too old indexes! */
   public void testUnsupportedOldIndexes() throws Exception {
diff --git a/lucene/core/src/java/org/apache/lucene/util/Version.java b/lucene/core/src/java/org/apache/lucene/util/Version.java
index c65f4fd96c0..95115e49708 100644
--- a/lucene/core/src/java/org/apache/lucene/util/Version.java
+++ b/lucene/core/src/java/org/apache/lucene/util/Version.java
@@ -310,6 +310,15 @@ public final class Version {
     }
   }
 
+  /** Major version, the difference between stable and trunk */
+  public final int major;
+  /** Minor version, incremented within the stable branch */
+  public final int minor;
+  /** Bugfix number, incremented on release branches */
+  public final int bugfix;
+  /** Prerelease version, currently 0 (alpha), 1 (beta), or 2 (final) */
+  public final int prerelease;
+
   // stores the version pieces, with most significant pieces in high bits
   // ie:  | 1 byte | 1 byte | 1 byte |   2 bits   |
   //         major   minor    bugfix   prerelease
@@ -320,6 +329,10 @@ public final class Version {
   }
 
   private Version(int major, int minor, int bugfix, int prerelease) {
+    this.major = major;
+    this.minor = minor;
+    this.bugfix = bugfix;
+    this.prerelease = prerelease;
     if (major > 5 || major < 4) {
       throw new IllegalArgumentException("Lucene 5.x only supports 5.x and 4.x versions");
     }