Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-465: Add configurable DirectoryProvider so that alternate Directory implementations can be specified via solrconfig.xml 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@712014 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2008-11-06 23:41:39 +00:00
parent e5c7179e7b
commit ee38ca0813
6 changed files with 164 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGES.txt
View File

@ -77,6 +77,11 @@ New Features
contention and synchronization overhead, to utilize multiple CPU cores more effectively.
(Fuad Efendi, Noble Paul, yonik via shalin)
14. SOLR-465: Add configurable DirectoryProvider so that alternate Directory
implementations can be specified via solrconfig.xml. The default
DirectoryProvider will use NIOFSDirectory for better concurrency
on non Windows platforms. (Mark Miller, TJ Laurenzo via yonik)
Optimizations
----------------------
1. SOLR-374: Use IndexReader.reopen to save resources by re-using parts of the

38
src/java/org/apache/solr/core/SolrCore.java
View File

@ -96,6 +96,7 @@ public final class SolrCore implements SolrInfoMBean {
private final Map<String,UpdateRequestProcessorChain> updateProcessorChains;
private final Map<String, SolrInfoMBean> infoRegistry;
private IndexDeletionPolicyWrapper solrDelPolicy;
private DirectoryFactory directoryFactory;
public long getStartTime() { return startTime; }
@ -217,7 +218,11 @@ public final class SolrCore implements SolrInfoMBean {
}
return result;
}
public DirectoryFactory getDirectoryFactory() {
return directoryFactory;
}
public String getName() {
return name;
}
@ -319,9 +324,28 @@ public final class SolrCore implements SolrInfoMBean {
// gets a non-caching searcher
public SolrIndexSearcher newSearcher(String name, boolean readOnly) throws IOException {
return new SolrIndexSearcher(this, schema, "main", IndexReader.open(FSDirectory.getDirectory(getIndexDir()), readOnly), true, false);
return new SolrIndexSearcher(this, schema, name, directoryFactory.open(getIndexDir()), false);
//return new SolrIndexSearcher(this, schema, "main", IndexReader.open(FSDirectory.getDirectory(getIndexDir()), readOnly), true, false);
}
private void initDirectoryFactory() {
String xpath = "directoryFactory";
Node node = (Node) solrConfig.evaluate(xpath, XPathConstants.NODE);
DirectoryFactory dirFactory;
if (node != null) {
Map<String, DirectoryFactory> registry = new HashMap<String, DirectoryFactory>();
NamedListPluginLoader<DirectoryFactory> indexReaderFactoryLoader = new NamedListPluginLoader<DirectoryFactory>(
"[solrconfig.xml] " + xpath, registry);
dirFactory = indexReaderFactoryLoader.loadSingle(solrConfig
.getResourceLoader(), node);
} else {
dirFactory = new StandardDirectoryFactory();
}
// And set it
directoryFactory = dirFactory;
}
// protect via synchronized(SolrCore.class)
private static Set<String> dirs = new HashSet<String>();
@ -351,6 +375,8 @@ public final class SolrCore implements SolrInfoMBean {
SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex",getIndexDir(), true, schema, solrConfig.mainIndexConfig);
writer.close();
}
initDirectoryFactory();
} catch (IOException e) {
throw new RuntimeException(e);
@ -1011,22 +1037,22 @@ public final class SolrCore implements SolrInfoMBean {
try {
newestSearcher = getNewestSearcher(false);
String newIndexDir = getNewIndexDir();
if (newestSearcher != null) {
IndexReader currentReader = newestSearcher.get().getReader();
String newIndexDir = getNewIndexDir();
if(new File(getIndexDir()).equals(new File(newIndexDir))) {
IndexReader newReader = currentReader.reopen();
if(newReader == currentReader) {
currentReader.incRef();
}
tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true);
} else {
tmp = new SolrIndexSearcher(this, schema, "main", newIndexDir, true);
tmp = new SolrIndexSearcher(this, schema, "main", getDirectoryFactory().open(newIndexDir), true, true);
}
} else {
tmp = new SolrIndexSearcher(this, schema, "main", getNewIndexDir(), true);
tmp = new SolrIndexSearcher(this, schema, "main", getDirectoryFactory().open(newIndexDir), true, true);
}
} catch (Throwable th) {
synchronized(searcherLock) {

2
src/java/org/apache/solr/schema/UUIDField.java
View File

@ -33,7 +33,7 @@ import org.apache.solr.request.XMLWriter;
*
* @see UUID#toString
* @see UUID#randomUUID
* @version $Id:$
* @version $Id$
*/
public class UUIDField extends FieldType {
private static final String NEW = "NEW";

10
src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -91,7 +91,10 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
private final SolrCache[] cacheList;
private static final SolrCache[] noCaches = new SolrCache[0];
/** Creates a searcher searching the index in the named directory. */
/** Creates a searcher searching the index in the named directory.
*
* @deprecated use alternate constructor
*/
public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, String path, boolean enableCache) throws IOException {
this(core, schema,name,IndexReader.open(path), true, enableCache);
}
@ -100,6 +103,11 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, Directory directory, boolean enableCache) throws IOException {
this(core, schema,name,IndexReader.open(directory), true, enableCache);
}
/** Creates a searcher searching the index in the provided directory. */
public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, Directory directory, boolean readOnly, boolean enableCache) throws IOException {
this(core, schema,name,IndexReader.open(directory, readOnly), true, enableCache);
}
/** Creates a searcher searching the provided index. */
public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, IndexReader r, boolean enableCache) {

65
src/java/org/apache/solr/update/SolrIndexWriter.java
View File

@ -20,6 +20,7 @@ package org.apache.solr.update;
import org.apache.lucene.index.*;
import org.apache.lucene.store.*;
import org.apache.solr.common.SolrException;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.schema.IndexSchema;
import org.slf4j.Logger;
@ -75,12 +76,13 @@ public class SolrIndexWriter extends IndexWriter {
}
public static Directory getDirectory(String path, SolrIndexConfig config) throws IOException {
Directory d = FSDirectory.getDirectory(path);
public static Directory getDirectory(String path, DirectoryFactory directoryFactory, SolrIndexConfig config) throws IOException {
Directory d = directoryFactory.open(path);
String rawLockType = (null == config) ? null : config.lockType;
if (null == rawLockType) {
// we default to "simple" for backwards compatiblitiy
// we default to "simple" for backwards compatibility
log.warn("No lockType configured for " + path + " assuming 'simple'");
rawLockType = "simple";
}
@ -95,7 +97,7 @@ public class SolrIndexWriter extends IndexWriter {
if (!(d.getLockFactory() instanceof SingleInstanceLockFactory))
d.setLockFactory(new SingleInstanceLockFactory());
} else if ("none".equals(lockType)) {
// recipie for disaster
// Recipe for disaster
log.error("CONFIGURATION WARNING: locks are disabled on " + path);
d.setLockFactory(new NoLockFactory());
} else {
@ -104,12 +106,67 @@ public class SolrIndexWriter extends IndexWriter {
}
return d;
}
/**
* @deprecated use getDirectory(DirectoryFactory directoryFactory, SolrIndexConfig config)
*/
public static Directory getDirectory(String path, SolrIndexConfig config) throws IOException {
Directory d = FSDirectory.getDirectory(path);
String rawLockType = (null == config) ? null : config.lockType;
if (null == rawLockType) {
// we default to "simple" for backwards compatibility
log.warn("No lockType configured for " + path + " assuming 'simple'");
rawLockType = "simple";
}
final String lockType = rawLockType.toLowerCase().trim();
if ("simple".equals(lockType)) {
// multiple SimpleFSLockFactory instances should be OK
d.setLockFactory(new SimpleFSLockFactory(path));
} else if ("native".equals(lockType)) {
d.setLockFactory(new NativeFSLockFactory(path));
} else if ("single".equals(lockType)) {
if (!(d.getLockFactory() instanceof SingleInstanceLockFactory))
d.setLockFactory(new SingleInstanceLockFactory());
} else if ("none".equals(lockType)) {
// Recipe for disaster
log.error("CONFIGURATION WARNING: locks are disabled on " + path);
d.setLockFactory(new NoLockFactory());
} else {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unrecognized lockType: " + rawLockType);
}
return d;
}
/**
*
*/
public SolrIndexWriter(String name, String path, DirectoryFactory dirFactory, boolean create, IndexSchema schema) throws IOException {
super(getDirectory(path, dirFactory, null), false, schema.getAnalyzer(), create);
init(name, schema, null);
}
/**
*
*/
public SolrIndexWriter(String name, String path, DirectoryFactory dirFactory, boolean create, IndexSchema schema, SolrIndexConfig config) throws IOException {
super(getDirectory(path, dirFactory, null), config.luceneAutoCommit, schema.getAnalyzer(), create);
init(name, schema, config);
}
/**
* @deprecated
*/
public SolrIndexWriter(String name, String path, boolean create, IndexSchema schema) throws IOException {
super(getDirectory(path, null), false, schema.getAnalyzer(), create);
init(name, schema, null);
}
/**
* @deprecated
*/
public SolrIndexWriter(String name, String path, boolean create, IndexSchema schema, SolrIndexConfig config) throws IOException {
super(getDirectory(path, config), config.luceneAutoCommit, schema.getAnalyzer(), create);
init(name, schema, config);

56
src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java
View File

@ -183,6 +183,62 @@ public abstract class AbstractPluginLoader<T>
return defaultPlugin;
}
/**
* Given a NodeList from XML in the form:
*
* <plugin name="name1" class="solr.ClassName" > ... </plugin>
*
* This will initialize and register a single plugin. A class will be
* generated for the plugin and registered to the given name.
*
* If 'preRegister' is true, the plugin will be registered *before* it is
* initialized This may be useful for implementations that need to inspect
* other registered plugins at startup.
*
* The created class for the plugin will be returned from this function.
*
*/
public T loadSingle(ResourceLoader loader, Node node) {
List<PluginInitInfo> info = new ArrayList<PluginInitInfo>();
T plugin = null;
try {
String name = DOMUtil.getAttr(node, "name", requireName ? type : null);
String className = DOMUtil.getAttr(node, "class", type);
plugin = create(loader, name, className, node);
log.info("created " + name + ": " + plugin.getClass().getName());
// Either initialize now or wait till everything has been registered
if (preRegister) {
info.add(new PluginInitInfo(plugin, node));
} else {
init(plugin, node);
}
T old = register(name, plugin);
if (old != null && !(name == null && !requireName)) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Multiple " + type + " registered to the same name: " + name
+ " ignoring: " + old);
}
} catch (Exception e) {
SolrConfig.severeErrors.add(e);
SolrException.logOnce(log, null, e);
}
// If everything needs to be registered *first*, this will initialize later
for (PluginInitInfo pinfo : info) {
try {
init(pinfo.plugin, pinfo.node);
} catch (Exception ex) {
SolrConfig.severeErrors.add(ex);
SolrException.logOnce(log, null, ex);
}
}
return plugin;
}
/**
* Internal class to hold onto initialization info so that it can be initialized