From 0502d828932c87b519e0bd6addb22126ac252d3e Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Mon, 8 Jun 2009 14:56:31 +0000 Subject: [PATCH] SOLR-243: Add configurable IndexReaderFactory so that alternate IndexReader implementations can be specified via solrconfig.xml. git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@782660 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 3 + example/solr/conf/schema.xml | 9 ++- example/solr/conf/solrconfig.xml | 7 +++ .../apache/solr/core/IndexReaderFactory.java | 51 ++++++++++++++++ src/java/org/apache/solr/core/SolrCore.java | 47 ++++++++++---- .../solr/core/StandardIndexReaderFactory.java | 39 ++++++++++++ .../apache/solr/search/SolrIndexSearcher.java | 18 +++--- .../solr/core/AlternateIndexReaderTest.java | 61 +++++++++++++++++++ .../solr/conf/solrconfig-altdirectory.xml | 5 ++ 9 files changed, 221 insertions(+), 19 deletions(-) create mode 100644 src/java/org/apache/solr/core/IndexReaderFactory.java create mode 100644 src/java/org/apache/solr/core/StandardIndexReaderFactory.java create mode 100644 src/test/org/apache/solr/core/AlternateIndexReaderTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 91df91c4ce3..b058ae66fd8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -234,6 +234,9 @@ New Features 59. SOLR-1189: Support providing username and password for basic HTTP authentication in Java replication (Matthew Gregg, shalin) + +60. SOLR-243: Add configurable IndexReaderFactory so that alternate IndexReader implementations + can be specified via solrconfig.xml. (Andrzej Bialecki, hossman, Mark Miller, John Wang) Optimizations ---------------------- diff --git a/example/solr/conf/schema.xml b/example/solr/conf/schema.xml index a46a72375d8..e1ac3f7a05c 100755 --- a/example/solr/conf/schema.xml +++ b/example/solr/conf/schema.xml @@ -435,10 +435,17 @@ + + + + + - + + + diff --git a/src/java/org/apache/solr/core/IndexReaderFactory.java b/src/java/org/apache/solr/core/IndexReaderFactory.java new file mode 100644 index 00000000000..900e7fffc29 --- /dev/null +++ b/src/java/org/apache/solr/core/IndexReaderFactory.java @@ -0,0 +1,51 @@ +package org.apache.solr.core; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.store.Directory; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.util.plugin.NamedListInitializedPlugin; + +/** + * Factory used to build a new IndexReader instance. + */ +public abstract class IndexReaderFactory implements NamedListInitializedPlugin { + + /** + * init will be called just once, immediately after creation. + *

+ * The args are user-level initialization parameters that may be specified + * when declaring an indexReaderFactory in solrconfig.xml + */ + public void init(NamedList args) { + /* :NOOP: */ + } + + /** + * Creates a new IndexReader instance using the given Directory. + * + * @param indexDir indexDir index location + * @param readOnly return readOnly IndexReader + * @return An IndexReader instance + * @throws IOException + */ + public abstract IndexReader newReader(Directory indexDir, boolean readOnly) + throws IOException; +} diff --git a/src/java/org/apache/solr/core/SolrCore.java b/src/java/org/apache/solr/core/SolrCore.java index d0f2d7c4661..dfb85be8210 100644 --- a/src/java/org/apache/solr/core/SolrCore.java +++ b/src/java/org/apache/solr/core/SolrCore.java @@ -97,6 +97,7 @@ public final class SolrCore implements SolrInfoMBean { private final Map infoRegistry; private IndexDeletionPolicyWrapper solrDelPolicy; private DirectoryFactory directoryFactory; + private IndexReaderFactory indexReaderFactory; public long getStartTime() { return startTime; } @@ -223,6 +224,10 @@ public final class SolrCore implements SolrInfoMBean { return directoryFactory; } + public IndexReaderFactory getIndexReaderFactory() { + return indexReaderFactory; + } + public String getName() { return name; } @@ -341,6 +346,24 @@ public final class SolrCore implements SolrInfoMBean { directoryFactory = dirFactory; } + private void initIndexReaderFactory() { + String xpath = "indexReaderFactory"; + Node node = (Node) solrConfig.evaluate(xpath, XPathConstants.NODE); + IndexReaderFactory indexReaderFactory; + if (node != null) { + Map registry = new HashMap(); + NamedListPluginLoader indexReaderFactoryLoader = new NamedListPluginLoader( + "[solrconfig.xml] " + xpath, registry); + + indexReaderFactory = indexReaderFactoryLoader.loadSingle(solrConfig + .getResourceLoader(), node); + } else { + indexReaderFactory = new StandardIndexReaderFactory(); + } + + this.indexReaderFactory = indexReaderFactory; + } + // protect via synchronized(SolrCore.class) private static Set dirs = new HashSet(); @@ -355,6 +378,7 @@ public final class SolrCore implements SolrInfoMBean { boolean removeLocks = solrConfig.unlockOnStartup; initDirectoryFactory(); + initIndexReaderFactory(); if (indexExists && firstTime && removeLocks) { // to remove locks, the directory must already exist... so we create it @@ -1048,21 +1072,22 @@ public final class SolrCore implements SolrInfoMBean { try { newestSearcher = getNewestSearcher(false); String newIndexDir = getNewIndexDir(); - if (newestSearcher != null) { + File indexDirFile = new File(getIndexDir()).getCanonicalFile(); + File newIndexDirFile = new File(newIndexDir).getCanonicalFile(); + + if (newestSearcher != null && solrConfig.reopenReaders + && indexDirFile.equals(newIndexDirFile)) { IndexReader currentReader = newestSearcher.get().getReader(); - if(solrConfig.reopenReaders && new File(getIndexDir()).getCanonicalFile().equals(new File(newIndexDir).getCanonicalFile())) { - IndexReader newReader = currentReader.reopen(); + IndexReader newReader = currentReader.reopen(); - if(newReader == currentReader) { - currentReader.incRef(); - } - - tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true); - } else { - tmp = new SolrIndexSearcher(this, schema, "main", getDirectoryFactory().open(newIndexDir), true, true); + if (newReader == currentReader) { + currentReader.incRef(); } + + tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true); } else { - tmp = new SolrIndexSearcher(this, schema, "main", getDirectoryFactory().open(newIndexDir), true, true); + IndexReader reader = getIndexReaderFactory().newReader(getDirectoryFactory().open(newIndexDir), true); + tmp = new SolrIndexSearcher(this, schema, "main", reader, true, true); } } catch (Throwable th) { synchronized(searcherLock) { diff --git a/src/java/org/apache/solr/core/StandardIndexReaderFactory.java b/src/java/org/apache/solr/core/StandardIndexReaderFactory.java new file mode 100644 index 00000000000..971f3222987 --- /dev/null +++ b/src/java/org/apache/solr/core/StandardIndexReaderFactory.java @@ -0,0 +1,39 @@ +package org.apache.solr.core; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.store.Directory; + +/** + * Default IndexReaderFactory implementation. Returns a standard Lucene + * IndexReader. + * + * @see IndexReader#open(Directory) + */ +public class StandardIndexReaderFactory extends IndexReaderFactory { + + /* (non-Javadoc) + * @see org.apache.solr.core.IndexReaderFactory#newReader(org.apache.lucene.store.Directory, boolean) + */ + public IndexReader newReader(Directory indexDir, boolean readOnly) + throws IOException { + return IndexReader.open(indexDir, readOnly); + } +} diff --git a/src/java/org/apache/solr/search/SolrIndexSearcher.java b/src/java/org/apache/solr/search/SolrIndexSearcher.java index 188644a066b..c27514d38dd 100644 --- a/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -24,7 +24,6 @@ import org.apache.lucene.index.TermDocs; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.util.PriorityQueue; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrConfig; @@ -33,7 +32,6 @@ import org.apache.solr.core.SolrInfoMBean; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.request.UnInvertedField; -import org.apache.solr.search.function.BoostedQuery; import org.apache.lucene.util.OpenBitSet; import java.io.IOException; @@ -92,17 +90,23 @@ public class SolrIndexSearcher extends IndexSearcher implements SolrInfoMBean { * @deprecated use alternate constructor */ public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, String path, boolean enableCache) throws IOException { - this(core, schema,name,IndexReader.open(path), true, enableCache); + this(core, schema,name, core.getIndexReaderFactory().newReader(core.getDirectoryFactory().open(path), false), true, enableCache); } - /** Creates a searcher searching the index in the provided directory. */ - public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, Directory directory, boolean enableCache) throws IOException { - this(core, schema,name,IndexReader.open(directory), true, enableCache); + /* + * Creates a searcher searching the index in the provided directory. Note: + * uses the main IndexReaderFactory for the specified SolrCore. + * + * @see SolrCore#getMainIndexReaderFactory + */ + public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, + Directory directory, boolean enableCache) throws IOException { + this(core, schema,name, core.getIndexReaderFactory().newReader(directory, false), true, enableCache); } /** Creates a searcher searching the index in the provided directory. */ public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, Directory directory, boolean readOnly, boolean enableCache) throws IOException { - this(core, schema,name,IndexReader.open(directory, readOnly), true, enableCache); + this(core, schema,name, core.getIndexReaderFactory().newReader(directory, readOnly), true, enableCache); } /** Creates a searcher searching the provided index. */ diff --git a/src/test/org/apache/solr/core/AlternateIndexReaderTest.java b/src/test/org/apache/solr/core/AlternateIndexReaderTest.java new file mode 100644 index 00000000000..8af4e067428 --- /dev/null +++ b/src/test/org/apache/solr/core/AlternateIndexReaderTest.java @@ -0,0 +1,61 @@ +package org.apache.solr.core; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.store.Directory; +import org.apache.solr.util.AbstractSolrTestCase; + +public class AlternateIndexReaderTest extends AbstractSolrTestCase { + + public String getSchemaFile() { + return "schema.xml"; + } + + public String getSolrConfigFile() { + return "solrconfig-altdirectory.xml"; + } + + /** + * Simple test to ensure that alternate IndexReaderFactory is being used. + * + * @throws Exception + */ + public void testAltReaderUsed() throws Exception { + assertTrue(TestIndexReaderFactory.newReaderCalled); + } + + static public class TestIndexReaderFactory extends IndexReaderFactory { + + static boolean newReaderCalled = false; + + public IndexReader newReader(Directory indexDir) throws IOException { + TestIndexReaderFactory.newReaderCalled = true; + return IndexReader.open(indexDir); + } + + public IndexReader newReader(Directory indexDir, boolean readOnly) + throws IOException { + TestIndexReaderFactory.newReaderCalled = true; + return IndexReader.open(indexDir, readOnly); + } + + } + +} diff --git a/src/test/test-files/solr/conf/solrconfig-altdirectory.xml b/src/test/test-files/solr/conf/solrconfig-altdirectory.xml index 04b2f433a1e..401c5442625 100755 --- a/src/test/test-files/solr/conf/solrconfig-altdirectory.xml +++ b/src/test/test-files/solr/conf/solrconfig-altdirectory.xml @@ -133,6 +133,11 @@ + + + + +