addBackcompatIndexes.py: Create 'sorted' indexes; on major releases, produce moreterms, dvupdates, and empty indexes. Add 7.0.0 indexes.

dev-tools/scripts/addBackcompatIndexes.py

@@ -31,12 +31,24 @@ import re
 import shutil
 
 def create_and_add_index(source, indextype, index_version, current_version, temp_dir):
+  if not current_version.is_back_compat_with(index_version):
+    prefix = 'unsupported'
+  else:
+    prefix = {
+      'cfs': 'index',
+      'nocfs': 'index',
+      'sorted': 'sorted',
+      'moreterms': 'moreterms',
+      'dvupdates': 'dvupdates',
+      'emptyIndex': 'empty'
+    }[indextype]
   if indextype in ('cfs', 'nocfs'):
     dirname = 'index.%s' % indextype
+    filename = '%s.%s-%s.zip' % (prefix, index_version, indextype)
   else:
     dirname = indextype
-  prefix = 'index' if current_version.is_back_compat_with(index_version) else 'unsupported'
-  filename = '%s.%s-%s.zip' % (prefix, index_version, indextype)
+    filename = '%s.%s.zip' % (prefix, index_version)
+  
   print('  creating %s...' % filename, end='', flush=True)
   module = 'backward-codecs'
   index_dir = os.path.join('lucene', module, 'src/test/org/apache/lucene/index')
@@ -47,7 +59,11 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
 
   test = {
     'cfs': 'testCreateCFS',
-    'nocfs': 'testCreateNoCFS'
+    'nocfs': 'testCreateNoCFS',
+    'sorted': 'testCreateSortedIndex',
+    'moreterms': 'testCreateMoreTermsIndex',
+    'dvupdates': 'testCreateIndexWithDocValuesUpdates',
+    'emptyIndex': 'testCreateEmptyIndex'
   }[indextype]
   ant_args = ' '.join([
     '-Dtests.bwcdir=%s' % temp_dir,
@@ -83,16 +99,22 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
   print('done')
 
 def update_backcompat_tests(types, index_version, current_version):
-  print('  adding new indexes to backcompat tests...', end='', flush=True)
+  print('  adding new indexes %s to backcompat tests...' % types, end='', flush=True)
   module = 'lucene/backward-codecs'
   filename = '%s/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java' % module
-  matcher = re.compile(r'final static String\[\] oldNames = {|};' if current_version.is_back_compat_with(index_version)
-                       else r'final String\[\] unsupportedNames = {|};')
+  if not current_version.is_back_compat_with(index_version):
+    matcher = re.compile(r'final String\[\] unsupportedNames = {|};'),
+  elif 'sorted' in types:
+    matcher = re.compile(r'final static String\[\] oldSortedNames = {|};')
+  else:
+    matcher = re.compile(r'final static String\[\] oldNames = {|};')
+
+  strip_dash_suffix_re = re.compile(r'-.*')
 
   def find_version(x):
     x = x.strip()
-    end = x.index("-")
-    return scriptutil.Version.parse(x[1:end])
+    x = re.sub(strip_dash_suffix_re, '', x) # remove the -suffix if any
+    return scriptutil.Version.parse(x)
 
   class Edit(object):
     start = None
@@ -100,10 +122,12 @@ def update_backcompat_tests(types, index_version, current_version):
       if self.start:
         # find where this version should exist
         i = len(buffer) - 1 
-        v = find_version(buffer[i])
-        while i >= self.start and v.on_or_after(index_version):
-          i -= 1
+        previous_version_exists = '};' not in line # Version list closure => there are no versions
+        if previous_version_exists: # Only look if there is a version here
           v = find_version(buffer[i])
+          while i >= self.start and v.on_or_after(index_version):
+            i -= 1
+            v = find_version(buffer[i])
         i += 1 # readjust since we skipped past by 1
 
         # unfortunately python doesn't have a range remove from list...
@@ -111,14 +135,20 @@ def update_backcompat_tests(types, index_version, current_version):
         while i < len(buffer) and index_version.on_or_after(find_version(buffer[i])):
           buffer.pop(i)
 
-        if i == len(buffer) and not buffer[-1].strip().endswith(","):
+        if i == len(buffer) and previous_version_exists and not buffer[-1].strip().endswith(","):
           # add comma
           buffer[-1] = buffer[-1].rstrip() + ",\n" 
 
-        last = buffer[-1]
-        spaces = ' ' * (len(last) - len(last.lstrip()))
+        if previous_version_exists:
+          last = buffer[-1]
+          spaces = ' ' * (len(last) - len(last.lstrip()))
+        else:
+          spaces = '      '
         for (j, t) in enumerate(types):
-          newline = spaces + ('"%s-%s"' % (index_version, t))
+          if t == 'sorted':
+            newline = spaces + ('"sorted.%s"') % index_version
+          else:
+            newline = spaces + ('"%s-%s"' % (index_version, t))
           if j < len(types) - 1 or i < len(buffer):
             newline += ','
           buffer.insert(i, newline + '\n')
@@ -215,9 +245,16 @@ def main():
   current_version = scriptutil.Version.parse(scriptutil.find_current_version())
   create_and_add_index(source, 'cfs', c.version, current_version, c.temp_dir)
   create_and_add_index(source, 'nocfs', c.version, current_version, c.temp_dir)
+  create_and_add_index(source, 'sorted', c.version, current_version, c.temp_dir)
+  if c.version.minor == 0 and c.version.bugfix == 0:
+    create_and_add_index(source, 'moreterms', c.version, current_version, c.temp_dir)
+    create_and_add_index(source, 'dvupdates', c.version, current_version, c.temp_dir)
+    create_and_add_index(source, 'emptyIndex', c.version, current_version, c.temp_dir)
+    print ('\nMANUAL UPDATE REQUIRED: edit TestBackwardsCompatibility to enable moreterms, dvupdates, and empty index testing')
     
   print('\nAdding backwards compatibility tests')
   update_backcompat_tests(['cfs', 'nocfs'], c.version, current_version)
+  update_backcompat_tests(['sorted'], c.version, current_version)
 
   print('\nTesting changes')
   check_backcompat_tests()

lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java

@@ -156,7 +156,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     IndexWriterConfig conf = new IndexWriterConfig(analyzer)
       .setMergePolicy(mp).setUseCompoundFile(false);
     IndexWriter writer = new IndexWriter(dir, conf);
-    LineFileDocs docs = new LineFileDocs(null);
+    LineFileDocs docs = new LineFileDocs(random());
     for(int i=0;i<50;i++) {
       writer.addDocument(docs.nextDoc());
     }
@@ -275,12 +275,33 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     dir.close();
   }
 
+  public void testCreateEmptyIndex() throws Exception {
+    Path indexDir = getIndexDir().resolve("emptyIndex");
+    Files.deleteIfExists(indexDir);
+    IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()))
+        .setUseCompoundFile(false).setMergePolicy(NoMergePolicy.INSTANCE);
+    try (Directory dir = newFSDirectory(indexDir);
+         IndexWriter writer = new IndexWriter(dir, conf)) {
+      writer.flush();
+    }
+  }
+
   final static String[] oldNames = {
+      "7.0.0-cfs",
+      "7.0.0-nocfs"
   };
 
   public static String[] getOldNames() {
     return oldNames;
   }
+  
+  final static String[] oldSortedNames = {
+      "sorted.7.0.0"
+  };
+
+  public static String[] getOldSortedNames() {
+    return oldSortedNames;
+  }
 
   final String[] unsupportedNames = {
       "1.9.0-cfs",
@@ -765,7 +786,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
         Directory targetDir2 = newDirectory();
         IndexWriter w = new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TestUtil.addIndexesSlowly(w, reader));
-        assertEquals(e.getMessage(), "Cannot merge a segment that has been created with major version 6 into this index which has been created by major version 7");
+        assertEquals(e.getMessage(), "Cannot merge a segment that has been created with major version 7 into this index which has been created by major version 8");
         w.close();
         targetDir2.close();
 
@@ -1449,14 +1470,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
   public static final String emptyIndex = "empty.7.0.0.zip";
 
   public void testUpgradeEmptyOldIndex() throws Exception {
-    assumeTrue("Reenable when 7.0 is released", false);
     Path oldIndexDir = createTempDir("emptyIndex");
     TestUtil.unzip(getDataInputStream(emptyIndex), oldIndexDir);
     Directory dir = newFSDirectory(oldIndexDir);
 
     newIndexUpgrader(dir).upgrade();
 
-    checkAllSegmentsUpgraded(dir, 6);
+    checkAllSegmentsUpgraded(dir, 7);
     
     dir.close();
   }
@@ -1464,7 +1484,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
   public static final String moreTermsIndex = "moreterms.7.0.0.zip";
 
   public void testMoreTerms() throws Exception {
-    assumeTrue("Reenable when 7.0 is released", false);
     Path oldIndexDir = createTempDir("moreterms");
     TestUtil.unzip(getDataInputStream(moreTermsIndex), oldIndexDir);
     Directory dir = newFSDirectory(oldIndexDir);
@@ -1509,7 +1528,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
   }
 
   public void testDocValuesUpdates() throws Exception {
-    assumeTrue("Reenable when 7.0 is released", false);
     Path oldIndexDir = createTempDir("dvupdates");
     TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);
     Directory dir = newFSDirectory(oldIndexDir);
@@ -1572,12 +1590,10 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
   }
 
   public void testSortedIndex() throws Exception {
-    assumeTrue("Reenable when 7.0 is released", false);
-    String[] versions = new String[] {};
-    for(String version : versions) {
+    for(String name : oldSortedNames) {
       Path path = createTempDir("sorted");
-      InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream("sorted." + version + ".zip");
-      assertNotNull("Sorted index index " + version + " not found", resource);
+      InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name + ".zip");
+      assertNotNull("Sorted index index " + name + " not found", resource);
       TestUtil.unzip(resource, path);
 
       // TODO: more tests