SOLR-658 -- Allow Solr to load index from arbitrary directory in dataDir

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@703981 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -45,6 +45,9 @@ New Features
  
  3. SOLR-657: Replace deprecated calls with the non-deprecated equivalents
     (Lars Kotthoff via ryan)
+
+ 4. SOLR-658: Allow Solr to load index from arbitrary directory in dataDir
+    (Noble Paul, Akshay Ukey via shalin)
     
 
 Optimizations

src/java/org/apache/solr/core/SolrCore.java

@@ -51,6 +51,7 @@ import org.apache.solr.util.plugin.AbstractPluginLoader;
 import org.apache.solr.util.plugin.NamedListInitializedPlugin;
 import org.apache.solr.util.plugin.NamedListPluginLoader;
 import org.apache.solr.util.plugin.SolrCoreAware;
+import org.apache.commons.io.IOUtils;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
@@ -61,7 +62,9 @@ import javax.xml.xpath.XPath;
 import javax.xml.xpath.XPathExpressionException;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -174,11 +177,47 @@ public final class SolrCore implements SolrInfoMBean {
   public String getDataDir() {
     return dataDir;
   }
-  
+
   public String getIndexDir() {
-    return dataDir + "index/";
+    if (_searcher == null)
+      return dataDir + "index/";
+    SolrIndexSearcher searcher = _searcher.get();
+    return searcher.getIndexDir() == null ? dataDir + "index/" : searcher.getIndexDir();
   }
-  
+
+
+  /**
+   * Returns the indexdir as given in index.properties. If index.properties exists in dataDir and
+   * there is a property <i>index</i> available and it points to a valid directory
+   * in dataDir that is returned Else dataDir/index is returned. Only called for creating new indexSearchers
+   * and indexwriters. Use the getIndexDir() method to know the active index directory
+   *
+   * @return the indexdir as given in index.properties
+   */
+  public String getNewIndexDir() {
+    String result = dataDir + "index/";
+    File propsFile = new File(dataDir + "index.properties");
+    if (propsFile.exists()) {
+      Properties p = new Properties();
+      InputStream is = null;
+      try {
+        is = new FileInputStream(propsFile);
+        p.load(is);
+      } catch (IOException e) {
+        /*no op*/
+      } finally {
+        IOUtils.closeQuietly(is);
+      }
+      String s = p.getProperty("index");
+      if (s != null && s.trim().length() > 0) {
+        File tmp = new File(dataDir + s);
+        if (tmp.exists() && tmp.isDirectory())
+          result = dataDir + s;
+      }
+    }
+    return result;
+  }
+
   public String getName() {
     return name;
   }
@@ -290,7 +329,7 @@ public final class SolrCore implements SolrInfoMBean {
   // currently only called with SolrCore.class lock held
   void initIndex() {
     try {
-      File dirFile = new File(getIndexDir());
+      File dirFile = new File(getNewIndexDir());
       boolean indexExists = dirFile.canRead();
       boolean firstTime = dirs.add(dirFile.getCanonicalPath());
       boolean removeLocks = solrConfig.getBool("mainIndex/unlockOnStartup", false);
@@ -971,15 +1010,20 @@ public final class SolrCore implements SolrInfoMBean {
       newestSearcher = getNewestSearcher(false);
       if (newestSearcher != null) {
         IndexReader currentReader = newestSearcher.get().getReader();
-        IndexReader newReader = currentReader.reopen();
+        String newIndexDir = getNewIndexDir();
+        if(new File(getIndexDir()).equals(new File(newIndexDir)))  {
+          IndexReader newReader = currentReader.reopen();
 
-        if(newReader == currentReader) {
-          currentReader.incRef();
+          if(newReader == currentReader) {
+            currentReader.incRef();
+          }
+
+          tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true);
+        } else  {
+          tmp = new SolrIndexSearcher(this, schema, "main", newIndexDir, true);
         }
-        
-        tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true);
       } else {
-        tmp = new SolrIndexSearcher(this, schema, "main", IndexReader.open(FSDirectory.getDirectory(getIndexDir()), true), true, true);
+        tmp = new SolrIndexSearcher(this, schema, "main", getNewIndexDir(), true);
       }
     } catch (Throwable th) {
       synchronized(searcherLock) {

src/java/org/apache/solr/search/SolrIndexSearcher.java

@@ -23,6 +23,7 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.search.*;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.PriorityQueue;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
@@ -57,6 +58,7 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
   private static Logger log = LoggerFactory.getLogger(SolrIndexSearcher.class);
   private final SolrCore core;
   private final IndexSchema schema;
+  private String indexDir;
 
   private final String name;
   private long openTime = System.currentTimeMillis();
@@ -111,6 +113,11 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
 
     log.info("Opening " + this.name);
 
+    if (r.directory() instanceof FSDirectory) {
+      FSDirectory fsDirectory = (FSDirectory) r.directory();
+      indexDir = fsDirectory.getFile().getAbsolutePath();
+    }
+
     reader = r;
     searcher = new IndexSearcher(r);
     this.closeReader = closeReader;
@@ -311,6 +318,14 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
     return searcher.docFreq(term);
   }
 
+  /**
+   * @return the indexDir on which this searcher is opened
+   * @see org.apache.solr.search.SolrIndexSearcher#SolrIndexSearcher(org.apache.solr.core.SolrCore, org.apache.solr.schema.IndexSchema, String, String, boolean)
+   */
+  public String getIndexDir() {
+    return indexDir;
+  }
+
   /* ********************** Document retrieval *************************/
    
   /* Future optimizations (yonik)

src/java/org/apache/solr/update/UpdateHandler.java

@@ -120,7 +120,7 @@ public abstract class UpdateHandler implements SolrInfoMBean {
   }
 
   protected SolrIndexWriter createMainIndexWriter(String name, boolean removeAllExisting) throws IOException {
-    return new SolrIndexWriter(name,core.getIndexDir(), removeAllExisting, schema, core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy());
+    return new SolrIndexWriter(name,core.getNewIndexDir(), removeAllExisting, schema, core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy());
   }
 
   protected final Term idTerm(String readableId) {

src/test/org/apache/solr/core/TestArbitraryIndexDir.java

@@ -0,0 +1,102 @@
+package org.apache.solr.core;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.util.TestHarness;
+import org.junit.Test;
+import org.xml.sax.SAXException;
+
+/**
+ * @version $Id$
+ */
+public class TestArbitraryIndexDir extends AbstractSolrTestCase{
+
+  public void setUp() throws Exception {
+    dataDir = new File(System.getProperty("java.io.tmpdir")
+        + System.getProperty("file.separator")
+        + getClass().getName() + "-" + System.currentTimeMillis() + System.getProperty("file.separator") + "solr"
+        + System.getProperty("file.separator") + "data");
+    dataDir.mkdirs();
+
+    solrConfig = h.createConfig(getSolrConfigFile());
+    h = new TestHarness( dataDir.getAbsolutePath(),
+        solrConfig,
+        getSchemaFile());
+    lrf = h.getRequestFactory
+    ("standard",0,20,"version","2.2");
+  }
+  
+  public void tearDown() throws Exception {
+    super.tearDown();
+
+  }
+
+  @Override
+  public String getSchemaFile() {
+    return "schema.xml";
+  }
+
+  @Override
+  public String getSolrConfigFile() {
+    return "solrconfig.xml";
+  }
+
+  @Test
+  public void testLoadNewIndexDir() throws IOException, ParserConfigurationException, SAXException, ParseException{
+    //add a doc in original index dir
+    assertU(adoc("id", String.valueOf(1),
+        "name", "name"+String.valueOf(1)));
+    //create a new index dir and index.properties file
+    File idxprops = new File(h.getCore().getDataDir() + "index.properties");
+    Properties p = new Properties();
+    File newDir = new File(h.getCore().getDataDir() + "index_temp");
+    newDir.mkdirs();
+    p.put("index", newDir.getName());
+    FileOutputStream os = null;
+    try {
+      os = new FileOutputStream(idxprops);
+      p.store(os, "index properties");
+    } catch (Exception e) {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+          "Unable to write index.properties", e);
+    }
+
+    //add a doc in the new index dir
+    Directory dir = FSDirectory.getDirectory(newDir);
+    IndexWriter iw = new IndexWriter(dir, new StandardAnalyzer(), new MaxFieldLength(1000));
+    Document doc = new Document();
+    doc.add(new Field("id", "2", Field.Store.YES, Field.Index.TOKENIZED));
+    doc.add(new Field("name", "name2", Field.Store.YES, Field.Index.TOKENIZED));
+    iw.addDocument(doc);
+    iw.commit();
+    iw.close();
+
+    //commit will cause searcher to open with the new index dir
+    assertU(commit());
+    //new index dir contains just 1 doc.
+    assertQ("return doc with id 2",
+        req("id:2"),
+        "*[count(//doc)=1]"
+    );
+    newDir.delete();
+  }
+}