From 6a7da3cd508f5b1e445df08aa2f1fa926d586e99 Mon Sep 17 00:00:00 2001 From: Atri Sharma Date: Thu, 27 Aug 2020 14:06:22 +0530 Subject: [PATCH] Update Circuit Breaker configured as a standard plugin (#1785) --- .../java/org/apache/solr/core/SolrConfig.java | 33 +---------- .../java/org/apache/solr/core/SolrCore.java | 3 +- .../solr/handler/component/SearchHandler.java | 2 +- .../circuitbreaker/CPUCircuitBreaker.java | 9 ++- .../util/circuitbreaker/CircuitBreaker.java | 47 ++++++++++++++-- .../circuitbreaker/CircuitBreakerManager.java | 55 +++++++++++++++---- .../circuitbreaker/MemoryCircuitBreaker.java | 9 ++- .../EditableSolrConfigAttributes.json | 11 ---- .../conf/solrconfig-memory-circuitbreaker.xml | 10 ++-- .../org/apache/solr/core/SolrCoreTest.java | 3 - .../apache/solr/core/TestConfigOverlay.java | 5 -- .../apache/solr/util/TestCircuitBreaker.java | 40 +++++++++----- solr/example/files/conf/solrconfig.xml | 24 +++++++- solr/solr-ref-guide/src/circuit-breakers.adoc | 37 ++++++++++--- .../src/query-settings-in-solrconfig.adoc | 37 +++++++++++-- 15 files changed, 213 insertions(+), 112 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 0a86f0c0e36..d6bb169a65b 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -78,6 +78,7 @@ import org.apache.solr.update.UpdateLog; import org.apache.solr.update.processor.UpdateRequestProcessorChain; import org.apache.solr.update.processor.UpdateRequestProcessorFactory; import org.apache.solr.util.DOMUtil; +import org.apache.solr.util.circuitbreaker.CircuitBreakerManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Node; @@ -227,14 +228,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { queryResultWindowSize = Math.max(1, getInt("query/queryResultWindowSize", 1)); queryResultMaxDocsCached = getInt("query/queryResultMaxDocsCached", Integer.MAX_VALUE); enableLazyFieldLoading = getBool("query/enableLazyFieldLoading", false); - - useCircuitBreakers = getBool("circuitBreakers/@enabled", false); - cpuCBEnabled = getBool("circuitBreakers/cpuBreaker/@enabled", false); - memCBEnabled = getBool("circuitBreakers/memBreaker/@enabled", false); - memCBThreshold = getInt("circuitBreakers/memBreaker/@threshold", 95); - cpuCBThreshold = getInt("circuitBreakers/cpuBreaker/@threshold", 95); - - validateCircuitBreakerThresholds(); filterCacheConfig = CacheConfig.getConfig(this, "query/filterCache"); queryResultCacheConfig = CacheConfig.getConfig(this, "query/queryResultCache"); @@ -365,6 +358,7 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { .add(new SolrPluginInfo(IndexSchemaFactory.class, "schemaFactory", REQUIRE_CLASS)) .add(new SolrPluginInfo(RestManager.class, "restManager")) .add(new SolrPluginInfo(StatsCache.class, "statsCache", REQUIRE_CLASS)) + .add(new SolrPluginInfo(CircuitBreakerManager.class, "circuitBreaker")) .build(); public static final Map classVsSolrPluginInfo; @@ -531,13 +525,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { public final int queryResultMaxDocsCached; public final boolean enableLazyFieldLoading; - // Circuit Breaker Configuration - public final boolean useCircuitBreakers; - public final int memCBThreshold; - public final boolean memCBEnabled; - public final boolean cpuCBEnabled; - public final int cpuCBThreshold; - // IndexConfig settings public final SolrIndexConfig indexConfig; @@ -817,16 +804,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { loader.reloadLuceneSPI(); } - private void validateCircuitBreakerThresholds() { - if (useCircuitBreakers) { - if (memCBEnabled) { - if (memCBThreshold > 95 || memCBThreshold < 50) { - throw new IllegalArgumentException("Valid value range of memoryCircuitBreakerThresholdPct is 50 - 95"); - } - } - } - } - public int getMultipartUploadLimitKB() { return multipartUploadLimitKB; } @@ -896,11 +873,7 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { m.put("queryResultMaxDocsCached", queryResultMaxDocsCached); m.put("enableLazyFieldLoading", enableLazyFieldLoading); m.put("maxBooleanClauses", booleanQueryMaxClauseCount); - m.put("useCircuitBreakers", useCircuitBreakers); - m.put("cpuCircuitBreakerEnabled", cpuCBEnabled); - m.put("memoryCircuitBreakerEnabled", memCBEnabled); - m.put("memoryCircuitBreakerThresholdPct", memCBThreshold); - m.put("cpuCircuitBreakerThreshold", cpuCBThreshold); + for (SolrPluginInfo plugin : plugins) { List infos = getPluginInfos(plugin.clazz.getName()); if (infos == null || infos.isEmpty()) continue; diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index f2a48664768..b44866e30f7 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -1174,7 +1174,8 @@ public final class SolrCore implements SolrInfoBean, Closeable { } private CircuitBreakerManager initCircuitBreakerManager() { - CircuitBreakerManager circuitBreakerManager = CircuitBreakerManager.build(solrConfig); + final PluginInfo info = solrConfig.getPluginInfo(CircuitBreakerManager.class.getName()); + CircuitBreakerManager circuitBreakerManager = CircuitBreakerManager.build(info); return circuitBreakerManager; } diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index 3934721ef1b..24cc7066e10 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -303,7 +303,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware, final RTimerTree timer = rb.isDebug() ? req.getRequestTimer() : null; - if (req.getCore().getSolrConfig().useCircuitBreakers) { + if (req.getCore().getCircuitBreakerManager().isEnabled()) { List trippedCircuitBreakers; if (timer != null) { diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java index 45dc1e8f05d..ffef4e5940a 100644 --- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java +++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java @@ -21,7 +21,6 @@ import java.lang.invoke.MethodHandles; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; -import org.apache.solr.core.SolrConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,11 +51,11 @@ public class CPUCircuitBreaker extends CircuitBreaker { private static final ThreadLocal allowedCPUUsage = ThreadLocal.withInitial(() -> 0.0); - public CPUCircuitBreaker(SolrConfig solrConfig) { - super(solrConfig); + public CPUCircuitBreaker(CircuitBreakerConfig config) { + super(config); - this.enabled = solrConfig.cpuCBEnabled; - this.cpuUsageThreshold = solrConfig.cpuCBThreshold; + this.enabled = config.getCpuCBEnabled(); + this.cpuUsageThreshold = config.getCpuCBThreshold(); } @Override diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java index 63ad7fd084e..90fda7a90e2 100644 --- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java +++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java @@ -17,8 +17,6 @@ package org.apache.solr.util.circuitbreaker; -import org.apache.solr.core.SolrConfig; - /** * Default class to define circuit breakers for Solr. *

@@ -35,16 +33,16 @@ import org.apache.solr.core.SolrConfig; public abstract class CircuitBreaker { public static final String NAME = "circuitbreaker"; - protected final SolrConfig solrConfig; + protected final CircuitBreakerConfig config; - public CircuitBreaker(SolrConfig solrConfig) { - this.solrConfig = solrConfig; + public CircuitBreaker(CircuitBreakerConfig config) { + this.config = config; } // Global config for all circuit breakers. For specific circuit breaker configs, define // your own config. protected boolean isEnabled() { - return solrConfig.useCircuitBreakers; + return config.isEnabled(); } /** @@ -61,4 +59,41 @@ public abstract class CircuitBreaker { * Get error message when the circuit breaker triggers */ public abstract String getErrorMessage(); + + public static class CircuitBreakerConfig { + private final boolean enabled; + private final boolean memCBEnabled; + private final int memCBThreshold; + private final boolean cpuCBEnabled; + private final int cpuCBThreshold; + + public CircuitBreakerConfig(final boolean enabled, final boolean memCBEnabled, final int memCBThreshold, + final boolean cpuCBEnabled, final int cpuCBThreshold) { + this.enabled = enabled; + this.memCBEnabled = memCBEnabled; + this.memCBThreshold = memCBThreshold; + this.cpuCBEnabled = cpuCBEnabled; + this.cpuCBThreshold = cpuCBThreshold; + } + + public boolean isEnabled() { + return enabled; + } + + public boolean getMemCBEnabled() { + return memCBEnabled; + } + + public int getMemCBThreshold() { + return memCBThreshold; + } + + public boolean getCpuCBEnabled() { + return cpuCBEnabled; + } + + public int getCpuCBThreshold() { + return cpuCBThreshold; + } + } } \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerManager.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerManager.java index ed7f62de9c6..975ca4b124e 100644 --- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerManager.java +++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerManager.java @@ -21,7 +21,9 @@ import java.util.ArrayList; import java.util.List; import com.google.common.annotations.VisibleForTesting; -import org.apache.solr.core.SolrConfig; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.PluginInfo; +import org.apache.solr.util.plugin.PluginInfoInitialized; /** * Manages all registered circuit breaker instances. Responsible for a holistic view @@ -37,7 +39,7 @@ import org.apache.solr.core.SolrConfig; * NOTE: The current way of registering new default circuit breakers is minimal and not a long term * solution. There will be a follow up with a SIP for a schema API design. */ -public class CircuitBreakerManager { +public class CircuitBreakerManager implements PluginInfoInitialized { // Class private to potentially allow "family" of circuit breakers to be enabled or disabled private final boolean enableCircuitBreakerManager; @@ -47,6 +49,18 @@ public class CircuitBreakerManager { this.enableCircuitBreakerManager = enableCircuitBreakerManager; } + @Override + public void init(PluginInfo pluginInfo) { + CircuitBreaker.CircuitBreakerConfig circuitBreakerConfig = buildCBConfig(pluginInfo); + + // Install the default circuit breakers + CircuitBreaker memoryCircuitBreaker = new MemoryCircuitBreaker(circuitBreakerConfig); + CircuitBreaker cpuCircuitBreaker = new CPUCircuitBreaker(circuitBreakerConfig); + + register(memoryCircuitBreaker); + register(cpuCircuitBreaker); + } + public void register(CircuitBreaker circuitBreaker) { circuitBreakerList.add(circuitBreaker); } @@ -121,19 +135,40 @@ public class CircuitBreakerManager { * * Any default circuit breakers should be registered here. */ - public static CircuitBreakerManager build(SolrConfig solrConfig) { - CircuitBreakerManager circuitBreakerManager = new CircuitBreakerManager(solrConfig.useCircuitBreakers); + @SuppressWarnings({"rawtypes"}) + public static CircuitBreakerManager build(PluginInfo pluginInfo) { + CircuitBreakerManager circuitBreakerManager = new CircuitBreakerManager(Boolean.parseBoolean(pluginInfo.attributes.get("enabled"))); - // Install the default circuit breakers - CircuitBreaker memoryCircuitBreaker = new MemoryCircuitBreaker(solrConfig); - CircuitBreaker cpuCircuitBreaker = new CPUCircuitBreaker(solrConfig); - - circuitBreakerManager.register(memoryCircuitBreaker); - circuitBreakerManager.register(cpuCircuitBreaker); + circuitBreakerManager.init(pluginInfo); return circuitBreakerManager; } + @VisibleForTesting + @SuppressWarnings({"rawtypes"}) + public static CircuitBreaker.CircuitBreakerConfig buildCBConfig(PluginInfo pluginInfo) { + boolean enabled = Boolean.parseBoolean(pluginInfo.attributes.get("enabled")); + boolean cpuCBEnabled = false; + boolean memCBEnabled = false; + int memCBThreshold = 100; + int cpuCBThreshold = 100; + + NamedList args = pluginInfo.initArgs; + + if (args != null) { + cpuCBEnabled = args.getBooleanArg("cpuEnabled"); + memCBEnabled = args.getBooleanArg("memEnabled"); + memCBThreshold = Integer.parseInt((String) args.get("memThreshold")); + cpuCBThreshold = Integer.parseInt((String) args.get("cpuThreshold")); + } + + return new CircuitBreaker.CircuitBreakerConfig(enabled, memCBEnabled, memCBThreshold, cpuCBEnabled, cpuCBThreshold); + } + + public boolean isEnabled() { + return enableCircuitBreakerManager; + } + @VisibleForTesting public List getRegisteredCircuitBreakers() { return circuitBreakerList; diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java index 797677dc85f..0fc7b01167f 100644 --- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java +++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java @@ -21,7 +21,6 @@ import java.lang.invoke.MethodHandles; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; -import org.apache.solr.core.SolrConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,10 +49,10 @@ public class MemoryCircuitBreaker extends CircuitBreaker { private static final ThreadLocal seenMemory = ThreadLocal.withInitial(() -> 0L); private static final ThreadLocal allowedMemory = ThreadLocal.withInitial(() -> 0L); - public MemoryCircuitBreaker(SolrConfig solrConfig) { - super(solrConfig); + public MemoryCircuitBreaker(CircuitBreakerConfig config) { + super(config); - this.enabled = solrConfig.memCBEnabled; + this.enabled = config.getMemCBEnabled(); long currentMaxHeap = MEMORY_MX_BEAN.getHeapMemoryUsage().getMax(); @@ -61,7 +60,7 @@ public class MemoryCircuitBreaker extends CircuitBreaker { throw new IllegalArgumentException("Invalid JVM state for the max heap usage"); } - int thresholdValueInPercentage = solrConfig.memCBThreshold; + int thresholdValueInPercentage = config.getMemCBThreshold(); double thresholdInFraction = thresholdValueInPercentage / (double) 100; heapMemoryThreshold = (long) (currentMaxHeap * thresholdInFraction); diff --git a/solr/core/src/resources/EditableSolrConfigAttributes.json b/solr/core/src/resources/EditableSolrConfigAttributes.json index 1f499823b9e..03bb1b67d90 100644 --- a/solr/core/src/resources/EditableSolrConfigAttributes.json +++ b/solr/core/src/resources/EditableSolrConfigAttributes.json @@ -55,17 +55,6 @@ "queryResultMaxDocsCached":1, "enableLazyFieldLoading":1, "boolTofilterOptimizer":1, - "circuitBreakers":{ - "enabled":10, - "memBreaker":{ - "enabled":10, - "threshold":20 - }, - "cpuBreaker":{ - "enabled":10, - "threshold":20 - } - }, "maxBooleanClauses":1}, "jmx":{ "agentId":0, diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-memory-circuitbreaker.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-memory-circuitbreaker.xml index d6adf924b67..699a7bd77e4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-memory-circuitbreaker.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-memory-circuitbreaker.xml @@ -78,10 +78,12 @@ - - - - + + true + 75 + true + 75 + diff --git a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java index 59069dbcd7e..09b481ce156 100644 --- a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java +++ b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java @@ -266,9 +266,6 @@ public class SolrCoreTest extends SolrTestCaseJ4 { assertEquals("wrong config for maxBooleanClauses", 1024, solrConfig.booleanQueryMaxClauseCount); assertEquals("wrong config for enableLazyFieldLoading", true, solrConfig.enableLazyFieldLoading); assertEquals("wrong config for queryResultWindowSize", 10, solrConfig.queryResultWindowSize); - assertEquals("wrong config for useCircuitBreakers", false, solrConfig.useCircuitBreakers); - assertEquals("wrong config for memoryCircuitBreakerThresholdPct", 95, solrConfig.memCBThreshold); - assertEquals("wrong config for cpuCircuitBreakerThreshold", 95, solrConfig.cpuCBThreshold); } /** diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java b/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java index 404cbc867b2..1c39fa98683 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java @@ -46,11 +46,6 @@ public class TestConfigOverlay extends SolrTestCase { assertTrue(isEditableProp("query.queryResultMaxDocsCached", false, null)); assertTrue(isEditableProp("query.enableLazyFieldLoading", false, null)); assertTrue(isEditableProp("query.boolTofilterOptimizer", false, null)); - assertTrue(isEditableProp("query.circuitBreakers.enabled", false, null)); - assertTrue(isEditableProp("query.circuitBreakers.memBreaker.enabled", false, null)); - assertTrue(isEditableProp("query.circuitBreakers.memBreaker.threshold", false, null)); - assertTrue(isEditableProp("query.circuitBreakers.cpuBreaker.enabled", false, null)); - assertTrue(isEditableProp("query.circuitBreakers.cpuBreaker.threshold", false, null)); assertTrue(isEditableProp("jmx.agentId", false, null)); assertTrue(isEditableProp("jmx.serviceUrl", false, null)); assertTrue(isEditableProp("jmx.rootName", false, null)); diff --git a/solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java b/solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java index 347568c3baa..9b1075cb11e 100644 --- a/solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java +++ b/solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java @@ -32,10 +32,11 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.apache.solr.core.SolrConfig; +import org.apache.solr.core.PluginInfo; import org.apache.solr.search.QueryParsing; import org.apache.solr.util.circuitbreaker.CPUCircuitBreaker; import org.apache.solr.util.circuitbreaker.CircuitBreaker; +import org.apache.solr.util.circuitbreaker.CircuitBreakerManager; import org.apache.solr.util.circuitbreaker.MemoryCircuitBreaker; import org.junit.After; import org.junit.BeforeClass; @@ -93,7 +94,11 @@ public class TestCircuitBreaker extends SolrTestCaseJ4 { removeAllExistingCircuitBreakers(); - CircuitBreaker circuitBreaker = new MockCircuitBreaker(h.getCore().getSolrConfig()); + PluginInfo pluginInfo = h.getCore().getSolrConfig().getPluginInfo(CircuitBreakerManager.class.getName()); + + CircuitBreaker.CircuitBreakerConfig circuitBreakerConfig = CircuitBreakerManager.buildCBConfig(pluginInfo); + + CircuitBreaker circuitBreaker = new MockCircuitBreaker(circuitBreakerConfig); h.getCore().getCircuitBreakerManager().register(circuitBreaker); @@ -110,7 +115,10 @@ public class TestCircuitBreaker extends SolrTestCaseJ4 { removeAllExistingCircuitBreakers(); - CircuitBreaker circuitBreaker = new FakeMemoryPressureCircuitBreaker(h.getCore().getSolrConfig()); + PluginInfo pluginInfo = h.getCore().getSolrConfig().getPluginInfo(CircuitBreakerManager.class.getName()); + + CircuitBreaker.CircuitBreakerConfig circuitBreakerConfig = CircuitBreakerManager.buildCBConfig(pluginInfo); + CircuitBreaker circuitBreaker = new FakeMemoryPressureCircuitBreaker(circuitBreakerConfig); h.getCore().getCircuitBreakerManager().register(circuitBreaker); @@ -132,7 +140,10 @@ public class TestCircuitBreaker extends SolrTestCaseJ4 { try { removeAllExistingCircuitBreakers(); - CircuitBreaker circuitBreaker = new BuildingUpMemoryPressureCircuitBreaker(h.getCore().getSolrConfig()); + PluginInfo pluginInfo = h.getCore().getSolrConfig().getPluginInfo(CircuitBreakerManager.class.getName()); + + CircuitBreaker.CircuitBreakerConfig circuitBreakerConfig = CircuitBreakerManager.buildCBConfig(pluginInfo); + CircuitBreaker circuitBreaker = new BuildingUpMemoryPressureCircuitBreaker(circuitBreakerConfig); h.getCore().getCircuitBreakerManager().register(circuitBreaker); @@ -185,7 +196,10 @@ public class TestCircuitBreaker extends SolrTestCaseJ4 { try { removeAllExistingCircuitBreakers(); - CircuitBreaker circuitBreaker = new FakeCPUCircuitBreaker(h.getCore().getSolrConfig()); + PluginInfo pluginInfo = h.getCore().getSolrConfig().getPluginInfo(CircuitBreakerManager.class.getName()); + + CircuitBreaker.CircuitBreakerConfig circuitBreakerConfig = CircuitBreakerManager.buildCBConfig(pluginInfo); + CircuitBreaker circuitBreaker = new FakeCPUCircuitBreaker(circuitBreakerConfig); h.getCore().getCircuitBreakerManager().register(circuitBreaker); @@ -260,8 +274,8 @@ public class TestCircuitBreaker extends SolrTestCaseJ4 { private static class MockCircuitBreaker extends MemoryCircuitBreaker { - public MockCircuitBreaker(SolrConfig solrConfig) { - super(solrConfig); + public MockCircuitBreaker(CircuitBreakerConfig config) { + super(config); } @Override @@ -278,8 +292,8 @@ public class TestCircuitBreaker extends SolrTestCaseJ4 { private static class FakeMemoryPressureCircuitBreaker extends MemoryCircuitBreaker { - public FakeMemoryPressureCircuitBreaker(SolrConfig solrConfig) { - super(solrConfig); + public FakeMemoryPressureCircuitBreaker(CircuitBreakerConfig config) { + super(config); } @Override @@ -292,8 +306,8 @@ public class TestCircuitBreaker extends SolrTestCaseJ4 { private static class BuildingUpMemoryPressureCircuitBreaker extends MemoryCircuitBreaker { private AtomicInteger count; - public BuildingUpMemoryPressureCircuitBreaker(SolrConfig solrConfig) { - super(solrConfig); + public BuildingUpMemoryPressureCircuitBreaker(CircuitBreakerConfig config) { + super(config); this.count = new AtomicInteger(0); } @@ -321,8 +335,8 @@ public class TestCircuitBreaker extends SolrTestCaseJ4 { } private static class FakeCPUCircuitBreaker extends CPUCircuitBreaker { - public FakeCPUCircuitBreaker(SolrConfig solrConfig) { - super(solrConfig); + public FakeCPUCircuitBreaker(CircuitBreakerConfig config) { + super(config); } @Override diff --git a/solr/example/files/conf/solrconfig.xml b/solr/example/files/conf/solrconfig.xml index a6cce10428d..5d7bedd240a 100644 --- a/solr/example/files/conf/solrconfig.xml +++ b/solr/example/files/conf/solrconfig.xml @@ -550,7 +550,7 @@ Circuit Breaker Section - This section consists of configurations for circuit breakers ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ --> - + - false + + + true - 100 + 75 + + + true + + + 75 + diff --git a/solr/solr-ref-guide/src/circuit-breakers.adoc b/solr/solr-ref-guide/src/circuit-breakers.adoc index 56451b08ef7..b28bda68631 100644 --- a/solr/solr-ref-guide/src/circuit-breakers.adoc +++ b/solr/solr-ref-guide/src/circuit-breakers.adoc @@ -27,19 +27,24 @@ are enabled, requests may be rejected under the condition of high node duress wi It is up to the client to handle this error and potentially build a retrial logic as this should ideally be a transient situation. == Circuit Breaker Configurations -The following flag controls the global activation/deactivation of circuit breakers. If this flag is disabled, all circuit breakers -will be disabled globally. Per circuit breaker configurations are specified in their respective sections later. +All circuit breaker configurations are listed in the circuitBreaker tags in solrconfig.xml as shown below: [source,xml] ---- - + - + ---- -This flag acts as the highest authority and global controller of circuit breakers. For using specific circuit breakers, each one +The "enabled" attribute controls the global activation/deactivation of circuit breakers. If this flag is disabled, all circuit breakers +will be disabled globally. Per circuit breaker configurations are specified in their respective sections later. + +This attribute acts as the highest authority and global controller of circuit breakers. For using specific circuit breakers, each one needs to be individually enabled in addition to this flag being enabled. +CircuitBreakerManager is the default manager for all circuit breakers that should be defined in the tag unless the user wishes to use +a custom implementation. + == Currently Supported Circuit Breakers === JVM Heap Usage Based Circuit Breaker @@ -51,11 +56,18 @@ Configuration for JVM heap usage based circuit breaker: [source,xml] ---- - +true ---- Note that this configuration will be overridden by the global circuit breaker flag -- if circuit breakers are disabled, this flag -will not help you. Also, the triggering threshold is defined as a percentage of the max heap allocated to the JVM. +will not help you. + +The triggering threshold is defined as a percentage of the max heap allocated to the JVM. It is controlled by the below configuration: + +[source,xml] +---- +75 +---- It does not logically make sense to have a threshold below 50% and above 95% of the max heap allocated to the JVM. Hence, the range of valid values for this parameter is [50, 95], both inclusive. @@ -76,11 +88,18 @@ Configuration for CPU utilization based circuit breaker: [source,xml] ---- - +true ---- Note that this configuration will be overridden by the global circuit breaker flag -- if circuit breakers are disabled, this flag -will not help you. The triggering threshold is defined in units of CPU utilization. +will not help you. + +The triggering threshold is defined in units of CPU utilization. The configuration to control this is as below: + +[source,xml] +---- +75 +---- == Performance Considerations It is worth noting that while JVM or CPU circuit breakers do not add any noticeable overhead per query, having too many diff --git a/solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc b/solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc index 4219ccd24f2..f2092337806 100644 --- a/solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc @@ -172,24 +172,49 @@ This parameter sets the maximum number of documents to cache for any entry in th 200 ---- -=== useCircuitBreakers +=== circuitBreaker -Global control flag for enabling circuit breakers. +This set of configurations control the behaviour of circuit breakers. [source,xml] ---- -true + + + ---- -=== memoryCircuitBreakerThresholdPct +To control whether Circuit Breakers are globally enabled, use the "enabled" attribute. + +=== Memory Circuit Breaker Settings + +To turn memory circuit breaker on/off, use the following flag: +[source,xml] +---- +true +---- Memory threshold in percentage for JVM heap usage defined in percentage of maximum heap allocated to the JVM (-Xmx). Ideally, this value should be in the range of 75-80% of maximum heap allocated -to the JVM. +to the JVM. The enabled flag can be used to control the specific toggle for this circuit breaker. [source,xml] ---- -75 +75 +---- + +=== CPU Circuit Breaker Settings + +To control turning on/off this feature, use the following flag: +[source,xml] +---- +true +---- + +Defines the triggering threshold in terms of the average per minute CPU load. The enabled flag can be used to control the specific toggle for this circuit breaker. + +[source,xml] +---- +75 ---- === useColdSearcher