From 25813774b23d7c86430e9eb5d9d55aaa7d545f69 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Mon, 13 Oct 2008 09:12:42 +0000 Subject: [PATCH] SOLR-658 -- Allow Solr to load index from arbitrary directory in dataDir git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@703981 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 3 + src/java/org/apache/solr/core/SolrCore.java | 64 +++++++++-- .../apache/solr/search/SolrIndexSearcher.java | 15 +++ .../org/apache/solr/update/UpdateHandler.java | 2 +- .../solr/core/TestArbitraryIndexDir.java | 102 ++++++++++++++++++ 5 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 src/test/org/apache/solr/core/TestArbitraryIndexDir.java diff --git a/CHANGES.txt b/CHANGES.txt index 8efa45e708d..9ddb903100c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -45,6 +45,9 @@ New Features 3. SOLR-657: Replace deprecated calls with the non-deprecated equivalents (Lars Kotthoff via ryan) + + 4. SOLR-658: Allow Solr to load index from arbitrary directory in dataDir + (Noble Paul, Akshay Ukey via shalin) Optimizations diff --git a/src/java/org/apache/solr/core/SolrCore.java b/src/java/org/apache/solr/core/SolrCore.java index d97f92af5a7..f637560d227 100644 --- a/src/java/org/apache/solr/core/SolrCore.java +++ b/src/java/org/apache/solr/core/SolrCore.java @@ -51,6 +51,7 @@ import org.apache.solr.util.plugin.AbstractPluginLoader; import org.apache.solr.util.plugin.NamedListInitializedPlugin; import org.apache.solr.util.plugin.NamedListPluginLoader; import org.apache.solr.util.plugin.SolrCoreAware; +import org.apache.commons.io.IOUtils; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; @@ -61,7 +62,9 @@ import javax.xml.xpath.XPath; import javax.xml.xpath.XPathExpressionException; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; @@ -174,11 +177,47 @@ public final class SolrCore implements SolrInfoMBean { public String getDataDir() { return dataDir; } - + public String getIndexDir() { - return dataDir + "index/"; + if (_searcher == null) + return dataDir + "index/"; + SolrIndexSearcher searcher = _searcher.get(); + return searcher.getIndexDir() == null ? dataDir + "index/" : searcher.getIndexDir(); } - + + + /** + * Returns the indexdir as given in index.properties. If index.properties exists in dataDir and + * there is a property index available and it points to a valid directory + * in dataDir that is returned Else dataDir/index is returned. Only called for creating new indexSearchers + * and indexwriters. Use the getIndexDir() method to know the active index directory + * + * @return the indexdir as given in index.properties + */ + public String getNewIndexDir() { + String result = dataDir + "index/"; + File propsFile = new File(dataDir + "index.properties"); + if (propsFile.exists()) { + Properties p = new Properties(); + InputStream is = null; + try { + is = new FileInputStream(propsFile); + p.load(is); + } catch (IOException e) { + /*no op*/ + } finally { + IOUtils.closeQuietly(is); + } + String s = p.getProperty("index"); + if (s != null && s.trim().length() > 0) { + File tmp = new File(dataDir + s); + if (tmp.exists() && tmp.isDirectory()) + result = dataDir + s; + } + } + return result; + } + public String getName() { return name; } @@ -290,7 +329,7 @@ public final class SolrCore implements SolrInfoMBean { // currently only called with SolrCore.class lock held void initIndex() { try { - File dirFile = new File(getIndexDir()); + File dirFile = new File(getNewIndexDir()); boolean indexExists = dirFile.canRead(); boolean firstTime = dirs.add(dirFile.getCanonicalPath()); boolean removeLocks = solrConfig.getBool("mainIndex/unlockOnStartup", false); @@ -971,15 +1010,20 @@ public final class SolrCore implements SolrInfoMBean { newestSearcher = getNewestSearcher(false); if (newestSearcher != null) { IndexReader currentReader = newestSearcher.get().getReader(); - IndexReader newReader = currentReader.reopen(); + String newIndexDir = getNewIndexDir(); + if(new File(getIndexDir()).equals(new File(newIndexDir))) { + IndexReader newReader = currentReader.reopen(); - if(newReader == currentReader) { - currentReader.incRef(); + if(newReader == currentReader) { + currentReader.incRef(); + } + + tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true); + } else { + tmp = new SolrIndexSearcher(this, schema, "main", newIndexDir, true); } - - tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true); } else { - tmp = new SolrIndexSearcher(this, schema, "main", IndexReader.open(FSDirectory.getDirectory(getIndexDir()), true), true, true); + tmp = new SolrIndexSearcher(this, schema, "main", getNewIndexDir(), true); } } catch (Throwable th) { synchronized(searcherLock) { diff --git a/src/java/org/apache/solr/search/SolrIndexSearcher.java b/src/java/org/apache/solr/search/SolrIndexSearcher.java index d586683f7e8..9a2082e1724 100644 --- a/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.PriorityQueue; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; @@ -57,6 +58,7 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean { private static Logger log = LoggerFactory.getLogger(SolrIndexSearcher.class); private final SolrCore core; private final IndexSchema schema; + private String indexDir; private final String name; private long openTime = System.currentTimeMillis(); @@ -111,6 +113,11 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean { log.info("Opening " + this.name); + if (r.directory() instanceof FSDirectory) { + FSDirectory fsDirectory = (FSDirectory) r.directory(); + indexDir = fsDirectory.getFile().getAbsolutePath(); + } + reader = r; searcher = new IndexSearcher(r); this.closeReader = closeReader; @@ -311,6 +318,14 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean { return searcher.docFreq(term); } + /** + * @return the indexDir on which this searcher is opened + * @see org.apache.solr.search.SolrIndexSearcher#SolrIndexSearcher(org.apache.solr.core.SolrCore, org.apache.solr.schema.IndexSchema, String, String, boolean) + */ + public String getIndexDir() { + return indexDir; + } + /* ********************** Document retrieval *************************/ /* Future optimizations (yonik) diff --git a/src/java/org/apache/solr/update/UpdateHandler.java b/src/java/org/apache/solr/update/UpdateHandler.java index a6b9e14236b..620b8436dd4 100644 --- a/src/java/org/apache/solr/update/UpdateHandler.java +++ b/src/java/org/apache/solr/update/UpdateHandler.java @@ -120,7 +120,7 @@ public abstract class UpdateHandler implements SolrInfoMBean { } protected SolrIndexWriter createMainIndexWriter(String name, boolean removeAllExisting) throws IOException { - return new SolrIndexWriter(name,core.getIndexDir(), removeAllExisting, schema, core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy()); + return new SolrIndexWriter(name,core.getNewIndexDir(), removeAllExisting, schema, core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy()); } protected final Term idTerm(String readableId) { diff --git a/src/test/org/apache/solr/core/TestArbitraryIndexDir.java b/src/test/org/apache/solr/core/TestArbitraryIndexDir.java new file mode 100644 index 00000000000..895f6343f7b --- /dev/null +++ b/src/test/org/apache/solr/core/TestArbitraryIndexDir.java @@ -0,0 +1,102 @@ +package org.apache.solr.core; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Properties; + +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriter.MaxFieldLength; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.solr.common.SolrException; +import org.apache.solr.util.AbstractSolrTestCase; +import org.apache.solr.util.TestHarness; +import org.junit.Test; +import org.xml.sax.SAXException; + +/** + * @version $Id$ + */ +public class TestArbitraryIndexDir extends AbstractSolrTestCase{ + + public void setUp() throws Exception { + dataDir = new File(System.getProperty("java.io.tmpdir") + + System.getProperty("file.separator") + + getClass().getName() + "-" + System.currentTimeMillis() + System.getProperty("file.separator") + "solr" + + System.getProperty("file.separator") + "data"); + dataDir.mkdirs(); + + solrConfig = h.createConfig(getSolrConfigFile()); + h = new TestHarness( dataDir.getAbsolutePath(), + solrConfig, + getSchemaFile()); + lrf = h.getRequestFactory + ("standard",0,20,"version","2.2"); + } + + public void tearDown() throws Exception { + super.tearDown(); + + } + + @Override + public String getSchemaFile() { + return "schema.xml"; + } + + @Override + public String getSolrConfigFile() { + return "solrconfig.xml"; + } + + @Test + public void testLoadNewIndexDir() throws IOException, ParserConfigurationException, SAXException, ParseException{ + //add a doc in original index dir + assertU(adoc("id", String.valueOf(1), + "name", "name"+String.valueOf(1))); + //create a new index dir and index.properties file + File idxprops = new File(h.getCore().getDataDir() + "index.properties"); + Properties p = new Properties(); + File newDir = new File(h.getCore().getDataDir() + "index_temp"); + newDir.mkdirs(); + p.put("index", newDir.getName()); + FileOutputStream os = null; + try { + os = new FileOutputStream(idxprops); + p.store(os, "index properties"); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unable to write index.properties", e); + } + + //add a doc in the new index dir + Directory dir = FSDirectory.getDirectory(newDir); + IndexWriter iw = new IndexWriter(dir, new StandardAnalyzer(), new MaxFieldLength(1000)); + Document doc = new Document(); + doc.add(new Field("id", "2", Field.Store.YES, Field.Index.TOKENIZED)); + doc.add(new Field("name", "name2", Field.Store.YES, Field.Index.TOKENIZED)); + iw.addDocument(doc); + iw.commit(); + iw.close(); + + //commit will cause searcher to open with the new index dir + assertU(commit()); + //new index dir contains just 1 doc. + assertQ("return doc with id 2", + req("id:2"), + "*[count(//doc)=1]" + ); + newDir.delete(); + } +}