Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix for SOLR-4229, changing "swappable" to "transient" 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1426826 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erick Erickson 2012-12-29 20:30:42 +00:00
parent cb8065c947
commit b6d4d6e907
5 changed files with 58 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
solr/CHANGES.txt
View File

@ -172,6 +172,11 @@ New Features
* SOLR-4110: Configurable Content-Type headers for PHPResponseWriters and
PHPSerializedResponseWriter. (Dominik Siebel via Mark Miller)
* SOLR-1028: The ability to specify "transient" and "loadOnStartup" as a new properties of
<core> tags in solr.xml. Can specify "transientCacheSize" in the <cores> tag. Together
these allow cores to be loaded only when needed and only transientCacheSize transient
cores will be loaded at a time, the rest aged out on an LRU basis.
Optimizations
----------------------

61
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -77,7 +77,6 @@ import org.apache.solr.logging.LogWatcher;
import org.apache.solr.logging.jul.JulWatcher;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.update.SolrCoreState;
import org.apache.solr.util.AdjustableSemaphore;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.FileUtils;
@ -119,12 +118,12 @@ public class CoreContainer
private static final String CORE_ROLES = "roles";
private static final String CORE_PROPERTIES = "properties";
private static final String CORE_LOADONSTARTUP = "loadOnStartup";
private static final String CORE_SWAPPABLE = "swappable";
private static final String CORE_TRANSIENT = "transient";
protected final Map<String, SolrCore> cores = new LinkedHashMap<String, SolrCore>(); // For "permanent" cores
protected Map<String, SolrCore> swappableCores = new LinkedHashMap<String, SolrCore>(); // For "lazily loaded" cores
protected Map<String, SolrCore> transientCores = new LinkedHashMap<String, SolrCore>(); // For "lazily loaded" cores
protected final Map<String, CoreDescriptor> dynamicDescriptors = new LinkedHashMap<String, CoreDescriptor>();
@ -160,7 +159,7 @@ public class CoreContainer
private String leaderVoteWait = LEADER_VOTE_WAIT;
private int distribUpdateConnTimeout = 0;
private int distribUpdateSoTimeout = 0;
protected int swappableCacheSize = Integer.MAX_VALUE; // Use as a flag too, if swappableCacheSize set in solr.xml this will be changed
protected int transientCacheSize = Integer.MAX_VALUE; // Use as a flag too, if transientCacheSize set in solr.xml this will be changed
private int coreLoadThreads;
{
@ -371,14 +370,14 @@ public class CoreContainer
return p;
}
// Trivial helper method for load, note it implements LRU on swappable cores
// Trivial helper method for load, note it implements LRU on transient cores
private void allocateLazyCores(Config cfg) {
swappableCacheSize = cfg.getInt("solr/cores/@swappableCacheSize", Integer.MAX_VALUE);
if (swappableCacheSize != Integer.MAX_VALUE) {
swappableCores = new LinkedHashMap<String, SolrCore>(swappableCacheSize, 0.75f, true) {
transientCacheSize = cfg.getInt("solr/cores/@transientCacheSize", Integer.MAX_VALUE);
if (transientCacheSize != Integer.MAX_VALUE) {
transientCores = new LinkedHashMap<String, SolrCore>(transientCacheSize, 0.75f, true) {
@Override
protected boolean removeEldestEntry(Map.Entry<String, SolrCore> eldest) {
if (size() > swappableCacheSize) {
if (size() > transientCacheSize) {
eldest.getValue().close();
return true;
}
@ -611,9 +610,9 @@ public class CoreContainer
.equalsIgnoreCase(opt)) ? true : false);
}
opt = DOMUtil.getAttr(node, CORE_SWAPPABLE, null);
opt = DOMUtil.getAttr(node, CORE_TRANSIENT, null);
if (opt != null) {
p.setSwappable(("true".equalsIgnoreCase(opt) || "on"
p.setTransient(("true".equalsIgnoreCase(opt) || "on"
.equalsIgnoreCase(opt)) ? true : false);
}
@ -625,7 +624,7 @@ public class CoreContainer
SolrCore c = null;
try {
c = create(p);
if (p.isSwappable()) {
if (p.isTransient()) {
registerLazyCore(name, c, false);
} else {
register(name, c, false);
@ -646,7 +645,7 @@ public class CoreContainer
} else {
// Store it away for later use. includes non-swappable but not
// Store it away for later use. includes non-transient but not
// loaded at startup cores.
dynamicDescriptors.put(rawName, p);
}
@ -755,15 +754,15 @@ public class CoreContainer
}
cores.clear();
}
synchronized (swappableCores) {
for (SolrCore core : swappableCores.values()) {
synchronized (transientCores) {
for (SolrCore core : transientCores.values()) {
try {
core.close();
} catch (Throwable t) {
SolrException.log(log, "Error shutting down core", t);
}
}
swappableCores.clear();
transientCores.clear();
}
} finally {
if (shardHandlerFactory != null) {
@ -822,7 +821,7 @@ public class CoreContainer
}
protected SolrCore registerLazyCore(String name, SolrCore core, boolean returnPrevNotClosed) {
return registerCore(swappableCores, name, core, returnPrevNotClosed);
return registerCore(transientCores, name, core, returnPrevNotClosed);
}
protected SolrCore registerCore(Map<String,SolrCore> whichCores, String name, SolrCore core, boolean returnPrevNotClosed) {
@ -1057,8 +1056,8 @@ public class CoreContainer
synchronized (cores) {
lst.addAll(this.cores.keySet());
}
synchronized (swappableCores) {
lst.addAll(this.swappableCores.keySet());
synchronized (transientCores) {
lst.addAll(this.transientCores.keySet());
}
return lst;
}
@ -1075,8 +1074,8 @@ public class CoreContainer
}
}
}
synchronized (swappableCores) {
for (Map.Entry<String,SolrCore> entry : swappableCores.entrySet()) {
synchronized (transientCores) {
for (Map.Entry<String,SolrCore> entry : transientCores.entrySet()) {
if (core == entry.getValue()) {
lst.add(entry.getKey());
}
@ -1264,25 +1263,25 @@ public class CoreContainer
}
}
if (dynamicDescriptors.size() == 0) return null; // Nobody even tried to define any swappable cores, so we're done.
if (dynamicDescriptors.size() == 0) return null; // Nobody even tried to define any transient cores, so we're done.
// Now look for already loaded swappable cores.
synchronized (swappableCores) {
core = swappableCores.get(name);
// Now look for already loaded transient cores.
synchronized (transientCores) {
core = transientCores.get(name);
if (core != null) {
core.open();
return core;
}
}
CoreDescriptor desc = dynamicDescriptors.get(name);
if (desc == null) { //Nope, no swappable core with this name
if (desc == null) { //Nope, no transient core with this name
return null;
}
try {
core = create(desc); // This should throw an error if it fails.
core.open();
if (desc.isSwappable()) {
registerLazyCore(name, core, false); // This is a swappable core
if (desc.isTransient()) {
registerLazyCore(name, core, false); // This is a transient core
} else {
register(name, core, false); // This is a "permanent", although deferred-load core
}
@ -1391,8 +1390,8 @@ public class CoreContainer
coresAttribs.put("defaultCoreName", defaultCoreName);
}
if (swappableCacheSize != Integer.MAX_VALUE) {
coresAttribs.put("swappableCacheSize", Integer.toString(swappableCacheSize));
if (transientCacheSize != Integer.MAX_VALUE) {
coresAttribs.put("transientCacheSize", Integer.toString(transientCacheSize));
}
addCoresAttrib(coresAttribs, "hostPort", this.hostPort, DEFAULT_HOST_PORT);
@ -1471,7 +1470,7 @@ public class CoreContainer
String dataDir = dcore.dataDir;
addCoreProperty(coreAttribs, coreNode, CORE_DATADIR, dataDir, null);
addCoreProperty(coreAttribs, coreNode, CORE_SWAPPABLE, Boolean.toString(dcore.isSwappable()), null);
addCoreProperty(coreAttribs, coreNode, CORE_TRANSIENT, Boolean.toString(dcore.isTransient()), null);
addCoreProperty(coreAttribs, coreNode, CORE_LOADONSTARTUP, Boolean.toString(dcore.isLoadOnStartup()), null);
CloudDescriptor cd = dcore.getCloudDescriptor();

10
solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
View File

@ -38,7 +38,7 @@ public class CoreDescriptor {
private final CoreContainer coreContainer;
private Properties coreProperties;
private boolean loadOnStartup = true;
private boolean swappable = false;
private boolean isTransient = false;
private CloudDescriptor cloudDesc;
@ -219,12 +219,12 @@ public class CoreDescriptor {
this.loadOnStartup = loadOnStartup;
}
public boolean isSwappable() {
return swappable;
public boolean isTransient() {
return isTransient;
}
public void setSwappable(boolean swappable) {
this.swappable = swappable;
public void setTransient(boolean aTransient) {
this.isTransient = aTransient;
}
public String getUlogDir() {

8
solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
View File

@ -110,9 +110,9 @@ public class TestCoreContainer extends SolrTestCaseJ4 {
cores.persistFile(oneXml);
assertXmlFile(oneXml, "/solr[@persistent='true']",
"/solr/cores[@defaultCoreName='collection1' and not(@swappableCacheSize)]",
"/solr/cores[@defaultCoreName='collection1' and not(@transientCacheSize)]",
"/solr/cores/core[@name='collection1' and @instanceDir='" + instDir +
"' and @swappable='false' and @loadOnStartup='true' ]", "1=count(/solr/cores/core)");
"' and @transient='false' and @loadOnStartup='true' ]", "1=count(/solr/cores/core)");
// create some new cores and sanity check the persistence
@ -142,12 +142,12 @@ public class TestCoreContainer extends SolrTestCaseJ4 {
assertEquals("cores not added?", 3, cores.getCoreNames().size());
final File twoXml = new File(workDir, "2.solr.xml");
cores.swappableCacheSize = 32;
cores.transientCacheSize = 32;
cores.persistFile(twoXml);
assertXmlFile(twoXml, "/solr[@persistent='true']",
"/solr/cores[@defaultCoreName='collection1' and @swappableCacheSize='32']",
"/solr/cores[@defaultCoreName='collection1' and @transientCacheSize='32']",
"/solr/cores/core[@name='collection1' and @instanceDir='" + instDir
+ "']", "/solr/cores/core[@name='X' and @instanceDir='" + instDir
+ "' and @dataDir='" + dataX + "']",

28
solr/core/src/test/org/apache/solr/core/TestLazyCores.java
View File

@ -96,24 +96,24 @@ public class TestLazyCores extends SolrTestCaseJ4 {
"collectionLazy8", "collectionLazy9");
SolrCore core1 = cc.getCore("collection1");
assertFalse("core1 should not be swappable", core1.getCoreDescriptor().isSwappable());
assertFalse("core1 should not be transient", core1.getCoreDescriptor().isTransient());
assertTrue("core1 should be loadable", core1.getCoreDescriptor().isLoadOnStartup());
assertNotNull(core1.getSolrConfig());
SolrCore core2 = cc.getCore("collectionLazy2");
assertTrue("core2 should not be swappable", core2.getCoreDescriptor().isSwappable());
assertTrue("core2 should not be transient", core2.getCoreDescriptor().isTransient());
assertTrue("core2 should be loadable", core2.getCoreDescriptor().isLoadOnStartup());
SolrCore core3 = cc.getCore("collectionLazy3");
assertTrue("core3 should not be swappable", core3.getCoreDescriptor().isSwappable());
assertTrue("core3 should not be transient", core3.getCoreDescriptor().isTransient());
assertFalse("core3 should not be loadable", core3.getCoreDescriptor().isLoadOnStartup());
SolrCore core4 = cc.getCore("collectionLazy4");
assertFalse("core4 should not be swappable", core4.getCoreDescriptor().isSwappable());
assertFalse("core4 should not be transient", core4.getCoreDescriptor().isTransient());
assertFalse("core4 should not be loadable", core4.getCoreDescriptor().isLoadOnStartup());
SolrCore core5 = cc.getCore("collectionLazy5");
assertFalse("core5 should not be swappable", core5.getCoreDescriptor().isSwappable());
assertFalse("core5 should not be transient", core5.getCoreDescriptor().isTransient());
assertTrue("core5 should be loadable", core5.getCoreDescriptor().isLoadOnStartup());
core1.close();
@ -292,16 +292,16 @@ public class TestLazyCores extends SolrTestCaseJ4 {
}
private final static String LOTS_SOLR_XML = " <solr persistent=\"false\"> " +
"<cores adminPath=\"/admin/cores\" defaultCoreName=\"collectionLazy2\" swappableCacheSize=\"4\"> " +
"<cores adminPath=\"/admin/cores\" defaultCoreName=\"collectionLazy2\" transientCacheSize=\"4\"> " +
"<core name=\"collection1\" instanceDir=\"collection1\" /> " +
"<core name=\"collectionLazy2\" instanceDir=\"collection2\" swappable=\"true\" loadOnStartup=\"true\" /> " +
"<core name=\"collectionLazy3\" instanceDir=\"collection3\" swappable=\"on\" loadOnStartup=\"false\"/> " +
"<core name=\"collectionLazy4\" instanceDir=\"collection4\" swappable=\"false\" loadOnStartup=\"false\"/> " +
"<core name=\"collectionLazy5\" instanceDir=\"collection5\" swappable=\"false\" loadOnStartup=\"true\"/> " +
"<core name=\"collectionLazy6\" instanceDir=\"collection6\" swappable=\"true\" loadOnStartup=\"false\" /> " +
"<core name=\"collectionLazy7\" instanceDir=\"collection7\" swappable=\"true\" loadOnStartup=\"false\" /> " +
"<core name=\"collectionLazy8\" instanceDir=\"collection8\" swappable=\"true\" loadOnStartup=\"false\" /> " +
"<core name=\"collectionLazy9\" instanceDir=\"collection9\" swappable=\"true\" loadOnStartup=\"false\" /> " +
"<core name=\"collectionLazy2\" instanceDir=\"collection2\" transient=\"true\" loadOnStartup=\"true\" /> " +
"<core name=\"collectionLazy3\" instanceDir=\"collection3\" transient=\"on\" loadOnStartup=\"false\"/> " +
"<core name=\"collectionLazy4\" instanceDir=\"collection4\" transient=\"false\" loadOnStartup=\"false\"/> " +
"<core name=\"collectionLazy5\" instanceDir=\"collection5\" transient=\"false\" loadOnStartup=\"true\"/> " +
"<core name=\"collectionLazy6\" instanceDir=\"collection6\" transient=\"true\" loadOnStartup=\"false\" /> " +
"<core name=\"collectionLazy7\" instanceDir=\"collection7\" transient=\"true\" loadOnStartup=\"false\" /> " +
"<core name=\"collectionLazy8\" instanceDir=\"collection8\" transient=\"true\" loadOnStartup=\"false\" /> " +
"<core name=\"collectionLazy9\" instanceDir=\"collection9\" transient=\"true\" loadOnStartup=\"false\" /> " +
"</cores> " +
"</solr>";
}