From b6d4d6e907be41de71b31356f31149d8125a4235 Mon Sep 17 00:00:00 2001 From: Erick Erickson Date: Sat, 29 Dec 2012 20:30:42 +0000 Subject: [PATCH] Fix for SOLR-4229, changing "swappable" to "transient" git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1426826 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 5 ++ .../org/apache/solr/core/CoreContainer.java | 61 +++++++++---------- .../org/apache/solr/core/CoreDescriptor.java | 10 +-- .../apache/solr/core/TestCoreContainer.java | 8 +-- .../org/apache/solr/core/TestLazyCores.java | 28 ++++----- 5 files changed, 58 insertions(+), 54 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index af56f70b89f..47d9d4be1ba 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -172,6 +172,11 @@ New Features * SOLR-4110: Configurable Content-Type headers for PHPResponseWriters and PHPSerializedResponseWriter. (Dominik Siebel via Mark Miller) + +* SOLR-1028: The ability to specify "transient" and "loadOnStartup" as a new properties of + tags in solr.xml. Can specify "transientCacheSize" in the tag. Together + these allow cores to be loaded only when needed and only transientCacheSize transient + cores will be loaded at a time, the rest aged out on an LRU basis. Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index e3da2f8a6c1..6ef404319b0 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -77,7 +77,6 @@ import org.apache.solr.logging.LogWatcher; import org.apache.solr.logging.jul.JulWatcher; import org.apache.solr.schema.IndexSchema; import org.apache.solr.update.SolrCoreState; -import org.apache.solr.util.AdjustableSemaphore; import org.apache.solr.util.DOMUtil; import org.apache.solr.util.DefaultSolrThreadFactory; import org.apache.solr.util.FileUtils; @@ -119,12 +118,12 @@ public class CoreContainer private static final String CORE_ROLES = "roles"; private static final String CORE_PROPERTIES = "properties"; private static final String CORE_LOADONSTARTUP = "loadOnStartup"; - private static final String CORE_SWAPPABLE = "swappable"; + private static final String CORE_TRANSIENT = "transient"; protected final Map cores = new LinkedHashMap(); // For "permanent" cores - protected Map swappableCores = new LinkedHashMap(); // For "lazily loaded" cores + protected Map transientCores = new LinkedHashMap(); // For "lazily loaded" cores protected final Map dynamicDescriptors = new LinkedHashMap(); @@ -160,7 +159,7 @@ public class CoreContainer private String leaderVoteWait = LEADER_VOTE_WAIT; private int distribUpdateConnTimeout = 0; private int distribUpdateSoTimeout = 0; - protected int swappableCacheSize = Integer.MAX_VALUE; // Use as a flag too, if swappableCacheSize set in solr.xml this will be changed + protected int transientCacheSize = Integer.MAX_VALUE; // Use as a flag too, if transientCacheSize set in solr.xml this will be changed private int coreLoadThreads; { @@ -371,14 +370,14 @@ public class CoreContainer return p; } - // Trivial helper method for load, note it implements LRU on swappable cores + // Trivial helper method for load, note it implements LRU on transient cores private void allocateLazyCores(Config cfg) { - swappableCacheSize = cfg.getInt("solr/cores/@swappableCacheSize", Integer.MAX_VALUE); - if (swappableCacheSize != Integer.MAX_VALUE) { - swappableCores = new LinkedHashMap(swappableCacheSize, 0.75f, true) { + transientCacheSize = cfg.getInt("solr/cores/@transientCacheSize", Integer.MAX_VALUE); + if (transientCacheSize != Integer.MAX_VALUE) { + transientCores = new LinkedHashMap(transientCacheSize, 0.75f, true) { @Override protected boolean removeEldestEntry(Map.Entry eldest) { - if (size() > swappableCacheSize) { + if (size() > transientCacheSize) { eldest.getValue().close(); return true; } @@ -611,9 +610,9 @@ public class CoreContainer .equalsIgnoreCase(opt)) ? true : false); } - opt = DOMUtil.getAttr(node, CORE_SWAPPABLE, null); + opt = DOMUtil.getAttr(node, CORE_TRANSIENT, null); if (opt != null) { - p.setSwappable(("true".equalsIgnoreCase(opt) || "on" + p.setTransient(("true".equalsIgnoreCase(opt) || "on" .equalsIgnoreCase(opt)) ? true : false); } @@ -625,7 +624,7 @@ public class CoreContainer SolrCore c = null; try { c = create(p); - if (p.isSwappable()) { + if (p.isTransient()) { registerLazyCore(name, c, false); } else { register(name, c, false); @@ -646,7 +645,7 @@ public class CoreContainer } else { - // Store it away for later use. includes non-swappable but not + // Store it away for later use. includes non-transient but not // loaded at startup cores. dynamicDescriptors.put(rawName, p); } @@ -755,15 +754,15 @@ public class CoreContainer } cores.clear(); } - synchronized (swappableCores) { - for (SolrCore core : swappableCores.values()) { + synchronized (transientCores) { + for (SolrCore core : transientCores.values()) { try { core.close(); } catch (Throwable t) { SolrException.log(log, "Error shutting down core", t); } } - swappableCores.clear(); + transientCores.clear(); } } finally { if (shardHandlerFactory != null) { @@ -822,7 +821,7 @@ public class CoreContainer } protected SolrCore registerLazyCore(String name, SolrCore core, boolean returnPrevNotClosed) { - return registerCore(swappableCores, name, core, returnPrevNotClosed); + return registerCore(transientCores, name, core, returnPrevNotClosed); } protected SolrCore registerCore(Map whichCores, String name, SolrCore core, boolean returnPrevNotClosed) { @@ -1057,8 +1056,8 @@ public class CoreContainer synchronized (cores) { lst.addAll(this.cores.keySet()); } - synchronized (swappableCores) { - lst.addAll(this.swappableCores.keySet()); + synchronized (transientCores) { + lst.addAll(this.transientCores.keySet()); } return lst; } @@ -1075,8 +1074,8 @@ public class CoreContainer } } } - synchronized (swappableCores) { - for (Map.Entry entry : swappableCores.entrySet()) { + synchronized (transientCores) { + for (Map.Entry entry : transientCores.entrySet()) { if (core == entry.getValue()) { lst.add(entry.getKey()); } @@ -1264,25 +1263,25 @@ public class CoreContainer } } - if (dynamicDescriptors.size() == 0) return null; // Nobody even tried to define any swappable cores, so we're done. + if (dynamicDescriptors.size() == 0) return null; // Nobody even tried to define any transient cores, so we're done. - // Now look for already loaded swappable cores. - synchronized (swappableCores) { - core = swappableCores.get(name); + // Now look for already loaded transient cores. + synchronized (transientCores) { + core = transientCores.get(name); if (core != null) { core.open(); return core; } } CoreDescriptor desc = dynamicDescriptors.get(name); - if (desc == null) { //Nope, no swappable core with this name + if (desc == null) { //Nope, no transient core with this name return null; } try { core = create(desc); // This should throw an error if it fails. core.open(); - if (desc.isSwappable()) { - registerLazyCore(name, core, false); // This is a swappable core + if (desc.isTransient()) { + registerLazyCore(name, core, false); // This is a transient core } else { register(name, core, false); // This is a "permanent", although deferred-load core } @@ -1391,8 +1390,8 @@ public class CoreContainer coresAttribs.put("defaultCoreName", defaultCoreName); } - if (swappableCacheSize != Integer.MAX_VALUE) { - coresAttribs.put("swappableCacheSize", Integer.toString(swappableCacheSize)); + if (transientCacheSize != Integer.MAX_VALUE) { + coresAttribs.put("transientCacheSize", Integer.toString(transientCacheSize)); } addCoresAttrib(coresAttribs, "hostPort", this.hostPort, DEFAULT_HOST_PORT); @@ -1471,7 +1470,7 @@ public class CoreContainer String dataDir = dcore.dataDir; addCoreProperty(coreAttribs, coreNode, CORE_DATADIR, dataDir, null); - addCoreProperty(coreAttribs, coreNode, CORE_SWAPPABLE, Boolean.toString(dcore.isSwappable()), null); + addCoreProperty(coreAttribs, coreNode, CORE_TRANSIENT, Boolean.toString(dcore.isTransient()), null); addCoreProperty(coreAttribs, coreNode, CORE_LOADONSTARTUP, Boolean.toString(dcore.isLoadOnStartup()), null); CloudDescriptor cd = dcore.getCloudDescriptor(); diff --git a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java index 99435d5b238..b5b5a6a6467 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java +++ b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java @@ -38,7 +38,7 @@ public class CoreDescriptor { private final CoreContainer coreContainer; private Properties coreProperties; private boolean loadOnStartup = true; - private boolean swappable = false; + private boolean isTransient = false; private CloudDescriptor cloudDesc; @@ -219,12 +219,12 @@ public class CoreDescriptor { this.loadOnStartup = loadOnStartup; } - public boolean isSwappable() { - return swappable; + public boolean isTransient() { + return isTransient; } - public void setSwappable(boolean swappable) { - this.swappable = swappable; + public void setTransient(boolean aTransient) { + this.isTransient = aTransient; } public String getUlogDir() { diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java index b1ffe3cdb54..9995763ec6d 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java +++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java @@ -110,9 +110,9 @@ public class TestCoreContainer extends SolrTestCaseJ4 { cores.persistFile(oneXml); assertXmlFile(oneXml, "/solr[@persistent='true']", - "/solr/cores[@defaultCoreName='collection1' and not(@swappableCacheSize)]", + "/solr/cores[@defaultCoreName='collection1' and not(@transientCacheSize)]", "/solr/cores/core[@name='collection1' and @instanceDir='" + instDir + - "' and @swappable='false' and @loadOnStartup='true' ]", "1=count(/solr/cores/core)"); + "' and @transient='false' and @loadOnStartup='true' ]", "1=count(/solr/cores/core)"); // create some new cores and sanity check the persistence @@ -142,12 +142,12 @@ public class TestCoreContainer extends SolrTestCaseJ4 { assertEquals("cores not added?", 3, cores.getCoreNames().size()); final File twoXml = new File(workDir, "2.solr.xml"); - cores.swappableCacheSize = 32; + cores.transientCacheSize = 32; cores.persistFile(twoXml); assertXmlFile(twoXml, "/solr[@persistent='true']", - "/solr/cores[@defaultCoreName='collection1' and @swappableCacheSize='32']", + "/solr/cores[@defaultCoreName='collection1' and @transientCacheSize='32']", "/solr/cores/core[@name='collection1' and @instanceDir='" + instDir + "']", "/solr/cores/core[@name='X' and @instanceDir='" + instDir + "' and @dataDir='" + dataX + "']", diff --git a/solr/core/src/test/org/apache/solr/core/TestLazyCores.java b/solr/core/src/test/org/apache/solr/core/TestLazyCores.java index 1af8653239f..bc9da080eaf 100644 --- a/solr/core/src/test/org/apache/solr/core/TestLazyCores.java +++ b/solr/core/src/test/org/apache/solr/core/TestLazyCores.java @@ -96,24 +96,24 @@ public class TestLazyCores extends SolrTestCaseJ4 { "collectionLazy8", "collectionLazy9"); SolrCore core1 = cc.getCore("collection1"); - assertFalse("core1 should not be swappable", core1.getCoreDescriptor().isSwappable()); + assertFalse("core1 should not be transient", core1.getCoreDescriptor().isTransient()); assertTrue("core1 should be loadable", core1.getCoreDescriptor().isLoadOnStartup()); assertNotNull(core1.getSolrConfig()); SolrCore core2 = cc.getCore("collectionLazy2"); - assertTrue("core2 should not be swappable", core2.getCoreDescriptor().isSwappable()); + assertTrue("core2 should not be transient", core2.getCoreDescriptor().isTransient()); assertTrue("core2 should be loadable", core2.getCoreDescriptor().isLoadOnStartup()); SolrCore core3 = cc.getCore("collectionLazy3"); - assertTrue("core3 should not be swappable", core3.getCoreDescriptor().isSwappable()); + assertTrue("core3 should not be transient", core3.getCoreDescriptor().isTransient()); assertFalse("core3 should not be loadable", core3.getCoreDescriptor().isLoadOnStartup()); SolrCore core4 = cc.getCore("collectionLazy4"); - assertFalse("core4 should not be swappable", core4.getCoreDescriptor().isSwappable()); + assertFalse("core4 should not be transient", core4.getCoreDescriptor().isTransient()); assertFalse("core4 should not be loadable", core4.getCoreDescriptor().isLoadOnStartup()); SolrCore core5 = cc.getCore("collectionLazy5"); - assertFalse("core5 should not be swappable", core5.getCoreDescriptor().isSwappable()); + assertFalse("core5 should not be transient", core5.getCoreDescriptor().isTransient()); assertTrue("core5 should be loadable", core5.getCoreDescriptor().isLoadOnStartup()); core1.close(); @@ -292,16 +292,16 @@ public class TestLazyCores extends SolrTestCaseJ4 { } private final static String LOTS_SOLR_XML = " " + - " " + + " " + " " + - " " + - " " + - " " + - " " + - " " + - " " + - " " + - " " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + " " + ""; }