From 3f9cc227f159f1e0583dd5aad2ec7a8bd102415f Mon Sep 17 00:00:00 2001 From: Atri Sharma Date: Thu, 2 Jul 2020 12:43:48 +0530 Subject: [PATCH] SOLR-14588: Implement Circuit Breakers (#1626) * SOLR-14588: Implement Circuit Breakers This commit consists of two parts: add circuit breakers infrastructure and a "real" JVM heap memory based circuit breaker which monitors incoming search requests and rejects them with SERVICE_TOO_BUSY error if the defined threshold is breached, thus giving headroom to existing indexing and search requests to complete. --- solr/CHANGES.txt | 2 + .../java/org/apache/solr/core/SolrConfig.java | 19 ++ .../java/org/apache/solr/core/SolrCore.java | 16 +- .../solr/handler/component/SearchHandler.java | 30 ++- .../util/circuitbreaker/CircuitBreaker.java | 56 +++++ .../circuitbreaker/CircuitBreakerManager.java | 134 +++++++++++ .../circuitbreaker/MemoryCircuitBreaker.java | 114 +++++++++ .../EditableSolrConfigAttributes.json | 2 + .../conf/solrconfig-memory-circuitbreaker.xml | 95 ++++++++ .../org/apache/solr/core/SolrCoreTest.java | 2 + .../apache/solr/core/TestConfigOverlay.java | 2 + .../apache/solr/util/TestCircuitBreaker.java | 218 ++++++++++++++++++ solr/example/files/conf/solrconfig.xml | 38 +++ .../configsets/_default/conf/solrconfig.xml | 59 +++++ .../conf/solrconfig.xml | 42 ++++ solr/solr-ref-guide/src/circuit-breakers.adoc | 68 ++++++ solr/solr-ref-guide/src/config-api.adoc | 7 + solr/solr-ref-guide/src/index.adoc | 3 + .../src/query-settings-in-solrconfig.adoc | 20 ++ 19 files changed, 925 insertions(+), 2 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java create mode 100644 solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerManager.java create mode 100644 solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-memory-circuitbreaker.xml create mode 100644 solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java create mode 100644 solr/solr-ref-guide/src/circuit-breakers.adoc diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index db8c74730b2..1861ba5cc95 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -12,6 +12,8 @@ New Features --------------------- * SOLR-14440: Introduce new Certificate Authentication Plugin to load Principal from certificate subject. (Mike Drob) +* SOLR-14588: Introduce Circuit Breaker Infrastructure and a JVM heap usage memory tracking circuit breaker implementation (Atri Sharma) + Improvements ---------------------- * LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta) diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index bd0c45d721e..2daaa95ebeb 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -224,6 +224,11 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { queryResultWindowSize = Math.max(1, getInt("query/queryResultWindowSize", 1)); queryResultMaxDocsCached = getInt("query/queryResultMaxDocsCached", Integer.MAX_VALUE); enableLazyFieldLoading = getBool("query/enableLazyFieldLoading", false); + + useCircuitBreakers = getBool("circuitBreaker/useCircuitBreakers", false); + memoryCircuitBreakerThresholdPct = getInt("circuitBreaker/memoryCircuitBreakerThresholdPct", 95); + + validateMemoryBreakerThreshold(); useRangeVersionsForPeerSync = getBool("peerSync/useRangeVersions", true); @@ -522,6 +527,10 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { public final int queryResultWindowSize; public final int queryResultMaxDocsCached; public final boolean enableLazyFieldLoading; + + // Circuit Breaker Configuration + public final boolean useCircuitBreakers; + public final int memoryCircuitBreakerThresholdPct; public final boolean useRangeVersionsForPeerSync; @@ -804,6 +813,14 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { loader.reloadLuceneSPI(); } + private void validateMemoryBreakerThreshold() { + if (useCircuitBreakers) { + if (memoryCircuitBreakerThresholdPct > 95 || memoryCircuitBreakerThresholdPct < 50) { + throw new IllegalArgumentException("Valid value range of memoryCircuitBreakerThresholdPct is 50 - 95"); + } + } + } + public int getMultipartUploadLimitKB() { return multipartUploadLimitKB; } @@ -873,6 +890,8 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { m.put("queryResultMaxDocsCached", queryResultMaxDocsCached); m.put("enableLazyFieldLoading", enableLazyFieldLoading); m.put("maxBooleanClauses", booleanQueryMaxClauseCount); + m.put("useCircuitBreakers", useCircuitBreakers); + m.put("memoryCircuitBreakerThresholdPct", memoryCircuitBreakerThresholdPct); for (SolrPluginInfo plugin : plugins) { List infos = getPluginInfos(plugin.clazz.getName()); if (infos == null || infos.isEmpty()) continue; diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 9b635e49402..04d4879531b 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -94,6 +94,7 @@ import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.ObjectReleaseTracker; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.common.util.Utils; import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.snapshots.SolrSnapshotManager; @@ -157,13 +158,13 @@ import org.apache.solr.update.processor.RunUpdateProcessorFactory; import org.apache.solr.update.processor.UpdateRequestProcessorChain; import org.apache.solr.update.processor.UpdateRequestProcessorChain.ProcessorInfo; import org.apache.solr.update.processor.UpdateRequestProcessorFactory; -import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.util.IOFunction; import org.apache.solr.util.NumberUtils; import org.apache.solr.util.PropertiesInputStream; import org.apache.solr.util.PropertiesOutputStream; import org.apache.solr.util.RefCounted; import org.apache.solr.util.TestInjection; +import org.apache.solr.util.circuitbreaker.CircuitBreakerManager; import org.apache.solr.util.plugin.NamedListInitializedPlugin; import org.apache.solr.util.plugin.PluginInfoInitialized; import org.apache.solr.util.plugin.SolrCoreAware; @@ -219,6 +220,8 @@ public final class SolrCore implements SolrInfoBean, Closeable { private final Codec codec; private final MemClassLoader memClassLoader; + private final CircuitBreakerManager circuitBreakerManager; + private final List confListeners = new CopyOnWriteArrayList<>(); private final ReentrantLock ruleExpiryLock; @@ -938,6 +941,7 @@ public final class SolrCore implements SolrInfoBean, Closeable { this.configSetProperties = configSet.getProperties(); // Initialize the metrics manager this.coreMetricManager = initCoreMetricManager(solrConfig); + this.circuitBreakerManager = initCircuitBreakerManager(); solrMetricsContext = coreMetricManager.getSolrMetricsContext(); this.coreMetricManager.loadReporters(); @@ -1164,6 +1168,12 @@ public final class SolrCore implements SolrInfoBean, Closeable { return coreMetricManager; } + private CircuitBreakerManager initCircuitBreakerManager() { + CircuitBreakerManager circuitBreakerManager = CircuitBreakerManager.build(solrConfig); + + return circuitBreakerManager; + } + @Override public void initializeMetrics(SolrMetricsContext parentContext, String scope) { newSearcherCounter = parentContext.counter("new", Category.SEARCHER.toString()); @@ -1499,6 +1509,10 @@ public final class SolrCore implements SolrInfoBean, Closeable { return updateProcessors; } + public CircuitBreakerManager getCircuitBreakerManager() { + return circuitBreakerManager; + } + // this core current usage count private final AtomicInteger refCount = new AtomicInteger(1); diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index b30a76362cd..cb555b8ba60 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -51,13 +51,17 @@ import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.PermissionNameProvider; import org.apache.solr.util.RTimerTree; import org.apache.solr.util.SolrPluginUtils; +import org.apache.solr.util.circuitbreaker.CircuitBreaker; +import org.apache.solr.util.circuitbreaker.CircuitBreakerManager; import org.apache.solr.util.plugin.PluginInfoInitialized; import org.apache.solr.util.plugin.SolrCoreAware; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.solr.common.params.CommonParams.DISTRIB; +import static org.apache.solr.common.params.CommonParams.FAILURE; import static org.apache.solr.common.params.CommonParams.PATH; +import static org.apache.solr.common.params.CommonParams.STATUS; /** @@ -297,6 +301,30 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware, final RTimerTree timer = rb.isDebug() ? req.getRequestTimer() : null; + if (req.getCore().getSolrConfig().useCircuitBreakers) { + List trippedCircuitBreakers; + + if (timer != null) { + RTimerTree subt = timer.sub("circuitbreaker"); + rb.setTimer(subt); + + CircuitBreakerManager circuitBreakerManager = req.getCore().getCircuitBreakerManager(); + trippedCircuitBreakers = circuitBreakerManager.checkTripped(); + + rb.getTimer().stop(); + } else { + CircuitBreakerManager circuitBreakerManager = req.getCore().getCircuitBreakerManager(); + trippedCircuitBreakers = circuitBreakerManager.checkTripped(); + } + + if (trippedCircuitBreakers != null) { + String errorMessage = CircuitBreakerManager.toErrorMessage(trippedCircuitBreakers); + rsp.add(STATUS, FAILURE); + rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Circuit Breakers tripped " + errorMessage)); + return; + } + } + final ShardHandler shardHandler1 = getAndPrepShardHandler(req, rb); // creates a ShardHandler object only if it's needed if (timer == null) { @@ -308,7 +336,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware, // debugging prepare phase RTimerTree subt = timer.sub( "prepare" ); for( SearchComponent c : components ) { - rb.setTimer( subt.sub( c.getName() ) ); + rb.setTimer(subt.sub( c.getName() ) ); c.prepare(rb); rb.getTimer().stop(); } diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java new file mode 100644 index 00000000000..f56f81e8e8c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util.circuitbreaker; + +import org.apache.solr.core.SolrConfig; + +/** + * Default class to define circuit breakers for Solr. + *

+ * There are two (typical) ways to use circuit breakers: + * 1. Have them checked at admission control by default (use CircuitBreakerManager for the same). + * 2. Use the circuit breaker in a specific code path(s). + * + * TODO: This class should be grown as the scope of circuit breakers grow. + *

+ */ +public abstract class CircuitBreaker { + public static final String NAME = "circuitbreaker"; + + protected final SolrConfig solrConfig; + + public CircuitBreaker(SolrConfig solrConfig) { + this.solrConfig = solrConfig; + } + + // Global config for all circuit breakers. For specific circuit breaker configs, define + // your own config. + protected boolean isEnabled() { + return solrConfig.useCircuitBreakers; + } + + /** + * Check if circuit breaker is tripped. + */ + public abstract boolean isTripped(); + + /** + * Get debug useful info. + */ + public abstract String getDebugInfo(); +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerManager.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerManager.java new file mode 100644 index 00000000000..584b9334806 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerManager.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util.circuitbreaker; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.solr.core.SolrConfig; + +/** + * Manages all registered circuit breaker instances. Responsible for a holistic view + * of whether a circuit breaker has tripped or not. + * + * There are two typical ways of using this class's instance: + * 1. Check if any circuit breaker has triggered -- and know which circuit breaker has triggered. + * 2. Get an instance of a specific circuit breaker and perform checks. + * + * It is a good practice to register new circuit breakers here if you want them checked for every + * request. + * + * NOTE: The current way of registering new default circuit breakers is minimal and not a long term + * solution. There will be a follow up with a SIP for a schema API design. + */ +public class CircuitBreakerManager { + // Class private to potentially allow "family" of circuit breakers to be enabled or disabled + private final boolean enableCircuitBreakerManager; + + private final List circuitBreakerList = new ArrayList<>(); + + public CircuitBreakerManager(final boolean enableCircuitBreakerManager) { + this.enableCircuitBreakerManager = enableCircuitBreakerManager; + } + + public void register(CircuitBreaker circuitBreaker) { + circuitBreakerList.add(circuitBreaker); + } + + public void deregisterAll() { + circuitBreakerList.clear(); + } + /** + * Check and return circuit breakers that have triggered + * @return CircuitBreakers which have triggered, null otherwise. + */ + public List checkTripped() { + List triggeredCircuitBreakers = null; + + if (enableCircuitBreakerManager) { + for (CircuitBreaker circuitBreaker : circuitBreakerList) { + if (circuitBreaker.isEnabled() && + circuitBreaker.isTripped()) { + if (triggeredCircuitBreakers == null) { + triggeredCircuitBreakers = new ArrayList<>(); + } + + triggeredCircuitBreakers.add(circuitBreaker); + } + } + } + + return triggeredCircuitBreakers; + } + + /** + * Returns true if *any* circuit breaker has triggered, false if none have triggered. + * + *

+ * NOTE: This method short circuits the checking of circuit breakers -- the method will + * return as soon as it finds a circuit breaker that is enabled and has triggered. + *

+ */ + public boolean checkAnyTripped() { + if (enableCircuitBreakerManager) { + for (CircuitBreaker circuitBreaker : circuitBreakerList) { + if (circuitBreaker.isEnabled() && + circuitBreaker.isTripped()) { + return true; + } + } + } + + return false; + } + + /** + * Construct the final error message to be printed when circuit breakers trip. + * + * @param circuitBreakerList Input list for circuit breakers. + * @return Constructed error message. + */ + public static String toErrorMessage(List circuitBreakerList) { + StringBuilder sb = new StringBuilder(); + + for (CircuitBreaker circuitBreaker : circuitBreakerList) { + sb.append(circuitBreaker.getClass().getName()); + sb.append(" "); + sb.append(circuitBreaker.getDebugInfo()); + sb.append("\n"); + } + + return sb.toString(); + } + + /** + * Register default circuit breakers and return a constructed CircuitBreakerManager + * instance which serves the given circuit breakers. + * + * Any default circuit breakers should be registered here. + */ + public static CircuitBreakerManager build(SolrConfig solrConfig) { + CircuitBreakerManager circuitBreakerManager = new CircuitBreakerManager(solrConfig.useCircuitBreakers); + + // Install the default circuit breakers + CircuitBreaker memoryCircuitBreaker = new MemoryCircuitBreaker(solrConfig); + circuitBreakerManager.register(memoryCircuitBreaker); + + return circuitBreakerManager; + } +} diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java new file mode 100644 index 00000000000..629d84aa29f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util.circuitbreaker; + +import java.lang.invoke.MethodHandles; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; + +import org.apache.solr.core.SolrConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + *

+ * Tracks the current JVM heap usage and triggers if it exceeds the defined percentage of the maximum + * heap size allocated to the JVM. This circuit breaker is a part of the default CircuitBreakerManager + * so is checked for every request -- hence it is realtime. Once the memory usage goes below the threshold, + * it will start allowing queries again. + *

+ * + *

+ * The memory threshold is defined as a percentage of the maximum memory allocated -- see memoryCircuitBreakerThresholdPct + * in solrconfig.xml. + *

+ */ + +public class MemoryCircuitBreaker extends CircuitBreaker { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final MemoryMXBean MEMORY_MX_BEAN = ManagementFactory.getMemoryMXBean(); + + private final long heapMemoryThreshold; + + // Assumption -- the value of these parameters will be set correctly before invoking getDebugInfo() + private final ThreadLocal seenMemory = new ThreadLocal<>(); + private final ThreadLocal allowedMemory = new ThreadLocal<>(); + + public MemoryCircuitBreaker(SolrConfig solrConfig) { + super(solrConfig); + + long currentMaxHeap = MEMORY_MX_BEAN.getHeapMemoryUsage().getMax(); + + if (currentMaxHeap <= 0) { + throw new IllegalArgumentException("Invalid JVM state for the max heap usage"); + } + + int thresholdValueInPercentage = solrConfig.memoryCircuitBreakerThresholdPct; + double thresholdInFraction = thresholdValueInPercentage / (double) 100; + heapMemoryThreshold = (long) (currentMaxHeap * thresholdInFraction); + + if (heapMemoryThreshold <= 0) { + throw new IllegalStateException("Memory limit cannot be less than or equal to zero"); + } + } + + // TODO: An optimization can be to trip the circuit breaker for a duration of time + // after the circuit breaker condition is matched. This will optimize for per call + // overhead of calculating the condition parameters but can result in false positives. + @Override + public boolean isTripped() { + if (!isEnabled()) { + return false; + } + + long localAllowedMemory = getCurrentMemoryThreshold(); + long localSeenMemory = calculateLiveMemoryUsage(); + + allowedMemory.set(localAllowedMemory); + + seenMemory.set(localSeenMemory); + + return (localSeenMemory >= localAllowedMemory); + } + + @Override + public String getDebugInfo() { + if (seenMemory.get() == 0L || allowedMemory.get() == 0L) { + log.warn("MemoryCircuitBreaker's monitored values (seenMemory, allowedMemory) not set"); + } + + return "seenMemory=" + seenMemory.get() + " allowedMemory=" + allowedMemory.get(); + } + + private long getCurrentMemoryThreshold() { + return heapMemoryThreshold; + } + + /** + * Calculate the live memory usage for the system. This method has package visibility + * to allow using for testing. + * @return Memory usage in bytes. + */ + protected long calculateLiveMemoryUsage() { + // NOTE: MemoryUsageGaugeSet provides memory usage statistics but we do not use them + // here since it will require extra allocations and incur cost, hence it is cheaper to use + // MemoryMXBean directly. Ideally, this call should not add noticeable + // latency to a query -- but if it does, please signify on SOLR-14588 + return MEMORY_MX_BEAN.getHeapMemoryUsage().getUsed(); + } +} diff --git a/solr/core/src/resources/EditableSolrConfigAttributes.json b/solr/core/src/resources/EditableSolrConfigAttributes.json index ed61e1ffa89..c441c30c1db 100644 --- a/solr/core/src/resources/EditableSolrConfigAttributes.json +++ b/solr/core/src/resources/EditableSolrConfigAttributes.json @@ -55,6 +55,8 @@ "queryResultMaxDocsCached":1, "enableLazyFieldLoading":1, "boolTofilterOptimizer":1, + "useCircuitBreakers":10, + "memoryCircuitBreakerThresholdPct":20, "maxBooleanClauses":1}, "jmx":{ "agentId":0, diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-memory-circuitbreaker.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-memory-circuitbreaker.xml new file mode 100644 index 00000000000..b6b20ff8c99 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-memory-circuitbreaker.xml @@ -0,0 +1,95 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + ${solr.data.dir:} + + + + + + + + ${solr.max.booleanClauses:1024} + + + + + + + + + + + + + + + true + + 10 + + + + + + + + + true + + 75 + + + + + + text + + + + diff --git a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java index 09b481ce156..8b20471b553 100644 --- a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java +++ b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java @@ -266,6 +266,8 @@ public class SolrCoreTest extends SolrTestCaseJ4 { assertEquals("wrong config for maxBooleanClauses", 1024, solrConfig.booleanQueryMaxClauseCount); assertEquals("wrong config for enableLazyFieldLoading", true, solrConfig.enableLazyFieldLoading); assertEquals("wrong config for queryResultWindowSize", 10, solrConfig.queryResultWindowSize); + assertEquals("wrong config for useCircuitBreakers", false, solrConfig.useCircuitBreakers); + assertEquals("wrong config for memoryCircuitBreakerThresholdPct", 95, solrConfig.memoryCircuitBreakerThresholdPct); } /** diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java b/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java index 1c39fa98683..cfdc1e302a6 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java @@ -46,6 +46,8 @@ public class TestConfigOverlay extends SolrTestCase { assertTrue(isEditableProp("query.queryResultMaxDocsCached", false, null)); assertTrue(isEditableProp("query.enableLazyFieldLoading", false, null)); assertTrue(isEditableProp("query.boolTofilterOptimizer", false, null)); + assertTrue(isEditableProp("query.useCircuitBreakers", false, null)); + assertTrue(isEditableProp("query.memoryCircuitBreakerThresholdPct", false, null)); assertTrue(isEditableProp("jmx.agentId", false, null)); assertTrue(isEditableProp("jmx.serviceUrl", false, null)); assertTrue(isEditableProp("jmx.rootName", false, null)); diff --git a/solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java b/solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java new file mode 100644 index 00000000000..00b8d1a908a --- /dev/null +++ b/solr/core/src/test/org/apache/solr/util/TestCircuitBreaker.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util; + +import java.util.HashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.core.SolrConfig; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.util.circuitbreaker.CircuitBreaker; +import org.apache.solr.util.circuitbreaker.MemoryCircuitBreaker; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +public class TestCircuitBreaker extends SolrTestCaseJ4 { + private final static int NUM_DOCS = 20; + + @Rule + public TestRule solrTestRules = RuleChain.outerRule(new SystemPropertiesRestoreRule()); + + @BeforeClass + public static void setUpClass() throws Exception { + System.setProperty("filterCache.enabled", "false"); + System.setProperty("queryResultCache.enabled", "false"); + System.setProperty("documentCache.enabled", "true"); + + initCore("solrconfig-memory-circuitbreaker.xml", "schema.xml"); + for (int i = 0 ; i < NUM_DOCS ; i ++) { + assertU(adoc("name", "john smith", "id", "1")); + assertU(adoc("name", "johathon smith", "id", "2")); + assertU(adoc("name", "john percival smith", "id", "3")); + assertU(adoc("id", "1", "title", "this is a title.", "inStock_b1", "true")); + assertU(adoc("id", "2", "title", "this is another title.", "inStock_b1", "true")); + assertU(adoc("id", "3", "title", "Mary had a little lamb.", "inStock_b1", "false")); + + //commit inside the loop to get multiple segments to make search as realistic as possible + assertU(commit()); + } + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + @After + public void after() { + h.getCore().getCircuitBreakerManager().deregisterAll(); + } + + public void testCBAlwaysTrips() { + HashMap args = new HashMap(); + + args.put(QueryParsing.DEFTYPE, CircuitBreaker.NAME); + args.put(CommonParams.FL, "id"); + + CircuitBreaker circuitBreaker = new MockCircuitBreaker(h.getCore().getSolrConfig()); + + h.getCore().getCircuitBreakerManager().register(circuitBreaker); + + expectThrows(SolrException.class, () -> { + h.query(req("name:\"john smith\"")); + }); + } + + public void testCBFakeMemoryPressure() { + HashMap args = new HashMap(); + + args.put(QueryParsing.DEFTYPE, CircuitBreaker.NAME); + args.put(CommonParams.FL, "id"); + + CircuitBreaker circuitBreaker = new FakeMemoryPressureCircuitBreaker(h.getCore().getSolrConfig()); + + h.getCore().getCircuitBreakerManager().register(circuitBreaker); + + expectThrows(SolrException.class, () -> { + h.query(req("name:\"john smith\"")); + }); + } + + public void testBuildingMemoryPressure() { + ExecutorService executor = ExecutorUtil.newMDCAwareCachedThreadPool( + new SolrNamedThreadFactory("TestCircuitBreaker")); + HashMap args = new HashMap(); + + args.put(QueryParsing.DEFTYPE, CircuitBreaker.NAME); + args.put(CommonParams.FL, "id"); + + AtomicInteger failureCount = new AtomicInteger(); + + try { + CircuitBreaker circuitBreaker = new BuildingUpMemoryPressureCircuitBreaker(h.getCore().getSolrConfig()); + + h.getCore().getCircuitBreakerManager().register(circuitBreaker); + + for (int i = 0; i < 5; i++) { + executor.submit(() -> { + try { + h.query(req("name:\"john smith\"")); + } catch (SolrException e) { + failureCount.incrementAndGet(); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + }); + } + + executor.shutdown(); + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e.getMessage()); + } + + assertEquals("Number of failed queries is not correct", 1, failureCount.get()); + } finally { + if (!executor.isShutdown()) { + executor.shutdown(); + } + } + } + + public void testResponseWithCBTiming() { + assertQ(req("q", "*:*", CommonParams.DEBUG_QUERY, "true"), + "//str[@name='rawquerystring']='*:*'", + "//str[@name='querystring']='*:*'", + "//str[@name='parsedquery']='MatchAllDocsQuery(*:*)'", + "//str[@name='parsedquery_toString']='*:*'", + "count(//lst[@name='explain']/*)=3", + "//lst[@name='explain']/str[@name='1']", + "//lst[@name='explain']/str[@name='2']", + "//lst[@name='explain']/str[@name='3']", + "//str[@name='QParser']", + "count(//lst[@name='timing']/*)=4", + "//lst[@name='timing']/double[@name='time']", + "count(//lst[@name='circuitbreaker']/*)>0", + "//lst[@name='circuitbreaker']/double[@name='time']", + "count(//lst[@name='prepare']/*)>0", + "//lst[@name='prepare']/double[@name='time']", + "count(//lst[@name='process']/*)>0", + "//lst[@name='process']/double[@name='time']" + ); + } + + private class MockCircuitBreaker extends CircuitBreaker { + + public MockCircuitBreaker(SolrConfig solrConfig) { + super(solrConfig); + } + + @Override + public boolean isTripped() { + // Always return true + return true; + } + + @Override + public String getDebugInfo() { + return "MockCircuitBreaker"; + } + } + + private class FakeMemoryPressureCircuitBreaker extends MemoryCircuitBreaker { + + public FakeMemoryPressureCircuitBreaker(SolrConfig solrConfig) { + super(solrConfig); + } + + @Override + protected long calculateLiveMemoryUsage() { + // Return a number large enough to trigger a pushback from the circuit breaker + return Long.MAX_VALUE; + } + } + + private class BuildingUpMemoryPressureCircuitBreaker extends MemoryCircuitBreaker { + private AtomicInteger count = new AtomicInteger(); + + public BuildingUpMemoryPressureCircuitBreaker(SolrConfig solrConfig) { + super(solrConfig); + } + + @Override + protected long calculateLiveMemoryUsage() { + if (count.getAndIncrement() >= 4) { + return Long.MAX_VALUE; + } + + return 5; // Random number guaranteed to not trip the circuit breaker + } + } +} diff --git a/solr/example/files/conf/solrconfig.xml b/solr/example/files/conf/solrconfig.xml index d16d4bf8557..a6cce10428d 100644 --- a/solr/example/files/conf/solrconfig.xml +++ b/solr/example/files/conf/solrconfig.xml @@ -546,6 +546,44 @@ + + + + false + + + 100 + + + 200 + + + + + + + + + + + + + + + + + + + + +