SOLR-11406: Solr 7.0 cannot read indexes from 6.x versions

dev-tools/maven/lucene/backward-codecs/pom.xml.template

@@ -71,5 +71,18 @@
         </excludes>
       </testResource>
     </testResources>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
   </build>
 </project>

dev-tools/maven/solr/core/src/test/pom.xml.template

@@ -61,6 +61,13 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-backward-codecs</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
 @solr-core.internal.test.dependencies@
 @solr-core.external.test.dependencies@
   </dependencies>

lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java

@@ -277,7 +277,11 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
 
   final static String[] oldNames = {
   };
-  
+
+  public static String[] getOldNames() {
+    return oldNames;
+  }
+
   final String[] unsupportedNames = {
       "1.9.0-cfs",
       "1.9.0-nocfs",
@@ -440,7 +444,11 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
   // TODO: on 6.0.0 release, gen the single segment indices and add here:
   final static String[] oldSingleSegmentNames = {
   };
-  
+
+  public static String[] getOldSingleSegmentNames() {
+    return oldSingleSegmentNames;
+  }
+
   static Map<String,Directory> oldIndexDirs;
 
   /**

solr/CHANGES.txt

@@ -235,6 +235,27 @@ Other Changes
 
 * SOLR-10451: Remove contrib/ltr/lib from lib includes in the techproducts example config (sungjunyoung via janhoy)
    
+==================  7.0.1 ==================
+
+Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
+
+Versions of Major Components
+---------------------
+Apache Tika 1.13
+Carrot2 3.15.0
+Velocity 1.7 and Velocity Tools 2.0
+Apache UIMA 2.3.1
+Apache ZooKeeper 3.4.10
+Jetty 9.3.14.v20161028
+
+Bug Fixes
+----------------------
+
+* SOLR-11297: Message "Lock held by this virtual machine" during startup.  Solr is trying to start some cores twice.
+  (Luiz Armesto, Shawn Heisey, Erick Erickson)
+
+* SOLR-11406: Solr 7.0 cannot read indexes from 6.x versions. (Shawn Heisey, Steve Rowe)
+
 ==================  7.0.0 ==================
 
 Versions of Major Components

solr/common-build.xml

@@ -493,6 +493,12 @@
     </ant>
   </target>
 
+  <target name="-compile-test-lucene-backward-codecs">
+    <ant dir="${common.dir}/backward-codecs" target="compile-test" inheritAll="false">
+      <propertyset refid="uptodate.and.compiled.properties"/>
+    </ant>
+  </target>
+
   <!-- Solr contrib targets -->
   <target name="-compile-analysis-extras">
     <ant dir="${common-solr.dir}/contrib/analysis-extras" target="compile" inheritAll="false">

solr/core/build.xml

@@ -31,7 +31,7 @@
 
   <target name="compile-core" depends="compile-solrj,common-solr.compile-core"/>
 
-  <target name="compile-test" depends="jar-analyzers-icu,-compile-test-lucene-queryparser,-compile-analysis-extras,common-solr.compile-test"/>
+  <target name="compile-test" depends="jar-analyzers-icu,-compile-test-lucene-queryparser,-compile-test-lucene-backward-codecs,-compile-analysis-extras,common-solr.compile-test"/>
 
   <path id="test.classpath">
     <path refid="solr.test.base.classpath"/>
@@ -39,6 +39,7 @@
     <pathelement location="${analyzers-icu.jar}"/>
     <pathelement location="${common-solr.dir}/build/contrib/solr-analysis-extras/classes/java"/>
     <pathelement location="${common.dir}/build/queryparser/classes/test"/>
+    <pathelement location="${common.dir}/build/backward-codecs/classes/test"/>
     <fileset dir="${common-solr.dir}/contrib/analysis-extras/lib" includes="icu4j*.jar"/>
   </path>
 

solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java

@@ -69,12 +69,16 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
     if (reader.leaves().isEmpty()) {
       metaData = new LeafMetaData(Version.LATEST.major, Version.LATEST, null);
     } else {
-      Version minVersion = reader.leaves().stream()
-          .map(LeafReaderContext::reader)
-          .map(LeafReader::getMetaData)
-          .map(LeafMetaData::getMinVersion)
-          .reduce((v1, v2) -> v1 == null ? null : v2 == null ? null : v2.onOrAfter(v1) ? v1 : v2)
-          .get();
+      Version minVersion = Version.LATEST;
+      for (LeafReaderContext leafReaderContext : reader.leaves()) {
+        Version leafVersion = leafReaderContext.reader().getMetaData().getMinVersion();
+        if (leafVersion == null) {
+          minVersion = null;
+          break;
+        } else if (minVersion.onOrAfter(leafVersion)) {
+          minVersion = leafVersion;
+        }
+      }
       metaData = new LeafMetaData(reader.leaves().get(0).reader().getMetaData().getCreatedVersionMajor(), minVersion, null);
     }
   }

solr/core/src/test-files/solr/configsets/backcompat/conf/schema.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="minimal" version="1.1">
+  <fieldType name="string" class="solr.StrField"/>
+  <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
+  <uniquekey>id</uniquekey>
+</schema>

solr/core/src/test-files/solr/configsets/backcompat/conf/solrconfig.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+  <dataDir>${solr.data.dir:}</dataDir>
+
+  <directoryFactory name="DirectoryFactory" class="solr.StandardDirectoryFactory"/>
+
+  <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+  <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+  <updateHandler class="solr.DirectUpdateHandler2">
+    <commitWithin>
+      <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+    </commitWithin>
+  </updateHandler>
+
+  <requestHandler name="/select" class="solr.SearchHandler">
+    <lst name="defaults">
+      <str name="echoParams">explicit</str>
+      <str name="indent">true</str>
+    </lst>
+  </requestHandler>
+</config>

solr/core/src/test/org/apache/solr/backcompat/TestLuceneIndexBackCompat.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.backcompat;
+
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.index.TestBackwardsCompatibility;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.util.TestHarness;
+import org.junit.Test;
+
+/** Verify we can read/write previous versions' Lucene indexes. */
+public class TestLuceneIndexBackCompat extends SolrTestCaseJ4 {
+  private static final String[] oldNames = TestBackwardsCompatibility.getOldNames();
+  private static final String[] oldSingleSegmentNames = TestBackwardsCompatibility.getOldSingleSegmentNames();
+
+  @Test
+  public void testOldIndexes() throws Exception {
+    List<String> names = new ArrayList<>(oldNames.length + oldSingleSegmentNames.length);
+    names.addAll(Arrays.asList(oldNames));
+    names.addAll(Arrays.asList(oldSingleSegmentNames));
+    for (String name : names) {
+      setupCore(name);
+
+      assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='35']");
+
+      assertU(adoc("id", "id_123456789"));
+      assertU(commit());
+
+      deleteCore();
+    }
+  }
+  
+  private void setupCore(String coreName) throws Exception {
+    if (h != null) {
+      h.close();
+    }
+    Path solrHome = createTempDir(coreName).toAbsolutePath();
+    Files.createDirectories(solrHome);
+    Path coreDir = solrHome.resolve(coreName);
+    Path confDir = coreDir.resolve("conf");
+    Files.createDirectories(confDir);
+    Path dataDir = coreDir.resolve("data");
+    Path indexDir = dataDir.resolve("index");
+    Files.createDirectories(indexDir);
+
+    Files.copy(getFile("solr/solr.xml").toPath(), solrHome.resolve("solr.xml"));
+    FileUtils.copyDirectory(configset("backcompat").toFile(), confDir.toFile());
+
+    try (Writer writer = new OutputStreamWriter(Files.newOutputStream(coreDir.resolve("core.properties")), StandardCharsets.UTF_8)) {
+      Properties coreProps = new Properties();
+      coreProps.put("name", coreName);
+      coreProps.store(writer, null);
+    }
+
+    InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream("index." + coreName + ".zip");
+    assertNotNull("Index name " + coreName + " not found", resource);
+    TestUtil.unzip(resource, indexDir);
+
+    configString = "solrconfig.xml";
+    schemaString = "schema.xml";
+    testSolrHome = solrHome;
+    System.setProperty("solr.solr.home", solrHome.toString());
+    ignoreException("ignore_exception");
+    solrConfig = TestHarness.createConfig(testSolrHome, coreName, getSolrConfigFile());
+    h = new TestHarness(coreName, dataDir.toString(), solrConfig, getSchemaFile());
+    lrf = h.getRequestFactory("",0,20, CommonParams.VERSION,"2.2");
+  }
+}