Merge branch 'pr/8795'
This commit is contained in:
commit
0f405e9710
|
@ -38,7 +38,7 @@ public class ChildMemoryCircuitBreaker implements CircuitBreaker {
|
||||||
private final AtomicLong trippedCount;
|
private final AtomicLong trippedCount;
|
||||||
private final ESLogger logger;
|
private final ESLogger logger;
|
||||||
private final HierarchyCircuitBreakerService parent;
|
private final HierarchyCircuitBreakerService parent;
|
||||||
private final Name name;
|
private final String name;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a circuit breaker that will break if the number of estimated
|
* Create a circuit breaker that will break if the number of estimated
|
||||||
|
@ -49,7 +49,7 @@ public class ChildMemoryCircuitBreaker implements CircuitBreaker {
|
||||||
* @param name the name of the breaker
|
* @param name the name of the breaker
|
||||||
*/
|
*/
|
||||||
public ChildMemoryCircuitBreaker(BreakerSettings settings, ESLogger logger,
|
public ChildMemoryCircuitBreaker(BreakerSettings settings, ESLogger logger,
|
||||||
HierarchyCircuitBreakerService parent, Name name) {
|
HierarchyCircuitBreakerService parent, String name) {
|
||||||
this(settings, null, logger, parent, name);
|
this(settings, null, logger, parent, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -64,7 +64,7 @@ public class ChildMemoryCircuitBreaker implements CircuitBreaker {
|
||||||
* @param oldBreaker the previous circuit breaker to inherit the used value from (starting offset)
|
* @param oldBreaker the previous circuit breaker to inherit the used value from (starting offset)
|
||||||
*/
|
*/
|
||||||
public ChildMemoryCircuitBreaker(BreakerSettings settings, ChildMemoryCircuitBreaker oldBreaker,
|
public ChildMemoryCircuitBreaker(BreakerSettings settings, ChildMemoryCircuitBreaker oldBreaker,
|
||||||
ESLogger logger, HierarchyCircuitBreakerService parent, Name name) {
|
ESLogger logger, HierarchyCircuitBreakerService parent, String name) {
|
||||||
this.name = name;
|
this.name = name;
|
||||||
this.settings = settings;
|
this.settings = settings;
|
||||||
this.memoryBytesLimit = settings.getLimit();
|
this.memoryBytesLimit = settings.getLimit();
|
||||||
|
@ -141,7 +141,7 @@ public class ChildMemoryCircuitBreaker implements CircuitBreaker {
|
||||||
newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead));
|
newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead));
|
||||||
}
|
}
|
||||||
if (memoryBytesLimit > 0 && newUsedWithOverhead > memoryBytesLimit) {
|
if (memoryBytesLimit > 0 && newUsedWithOverhead > memoryBytesLimit) {
|
||||||
logger.warn("[{}] New used memory {} [{}] from field [{}] would be larger than configured breaker: {} [{}], breaking",
|
logger.warn("[{}] New used memory {} [{}] for data of [{}] would be larger than configured breaker: {} [{}], breaking",
|
||||||
this.name,
|
this.name,
|
||||||
newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead), label,
|
newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead), label,
|
||||||
memoryBytesLimit, new ByteSizeValue(memoryBytesLimit));
|
memoryBytesLimit, new ByteSizeValue(memoryBytesLimit));
|
||||||
|
@ -220,7 +220,7 @@ public class ChildMemoryCircuitBreaker implements CircuitBreaker {
|
||||||
/**
|
/**
|
||||||
* @return the name of the breaker
|
* @return the name of the breaker
|
||||||
*/
|
*/
|
||||||
public Name getName() {
|
public String getName() {
|
||||||
return this.name;
|
return this.name;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,10 +20,7 @@
|
||||||
package org.elasticsearch.common.breaker;
|
package org.elasticsearch.common.breaker;
|
||||||
|
|
||||||
import org.elasticsearch.ElasticsearchIllegalArgumentException;
|
import org.elasticsearch.ElasticsearchIllegalArgumentException;
|
||||||
import org.elasticsearch.common.io.stream.StreamInput;
|
|
||||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -32,42 +29,9 @@ import java.util.Locale;
|
||||||
*/
|
*/
|
||||||
public interface CircuitBreaker {
|
public interface CircuitBreaker {
|
||||||
|
|
||||||
/**
|
public static final String PARENT = "parent";
|
||||||
* Enum used for specifying different types of circuit breakers
|
public static final String FIELDDATA = "fielddata";
|
||||||
*/
|
public static final String REQUEST = "request";
|
||||||
public static enum Name {
|
|
||||||
PARENT(0),
|
|
||||||
FIELDDATA(1),
|
|
||||||
REQUEST(2);
|
|
||||||
|
|
||||||
private int ordinal;
|
|
||||||
|
|
||||||
Name(int ordinal) {
|
|
||||||
this.ordinal = ordinal;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getSerializableValue() {
|
|
||||||
return this.ordinal;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Name readFrom(StreamInput in) throws IOException {
|
|
||||||
int value = in.readVInt();
|
|
||||||
switch (value) {
|
|
||||||
case 0:
|
|
||||||
return Name.PARENT;
|
|
||||||
case 1:
|
|
||||||
return Name.FIELDDATA;
|
|
||||||
case 2:
|
|
||||||
return Name.REQUEST;
|
|
||||||
default:
|
|
||||||
throw new ElasticsearchIllegalArgumentException("No CircuitBreaker with ordinal: " + value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void writeTo(Name name, StreamOutput out) throws IOException {
|
|
||||||
out.writeVInt(name.getSerializableValue());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static enum Type {
|
public static enum Type {
|
||||||
// A regular or child MemoryCircuitBreaker
|
// A regular or child MemoryCircuitBreaker
|
||||||
|
@ -135,5 +99,5 @@ public interface CircuitBreaker {
|
||||||
/**
|
/**
|
||||||
* @return the name of the breaker
|
* @return the name of the breaker
|
||||||
*/
|
*/
|
||||||
public Name getName();
|
public String getName();
|
||||||
}
|
}
|
||||||
|
|
|
@ -186,7 +186,7 @@ public class MemoryCircuitBreaker implements CircuitBreaker {
|
||||||
/**
|
/**
|
||||||
* @return the name of the breaker
|
* @return the name of the breaker
|
||||||
*/
|
*/
|
||||||
public Name getName() {
|
public String getName() {
|
||||||
return Name.FIELDDATA;
|
return FIELDDATA;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,9 +25,9 @@ package org.elasticsearch.common.breaker;
|
||||||
*/
|
*/
|
||||||
public class NoopCircuitBreaker implements CircuitBreaker {
|
public class NoopCircuitBreaker implements CircuitBreaker {
|
||||||
|
|
||||||
private final Name name;
|
private final String name;
|
||||||
|
|
||||||
public NoopCircuitBreaker(Name name) {
|
public NoopCircuitBreaker(String name) {
|
||||||
this.name = name;
|
this.name = name;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -67,7 +67,7 @@ public class NoopCircuitBreaker implements CircuitBreaker {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Name getName() {
|
public String getName() {
|
||||||
return this.name;
|
return this.name;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -384,7 +384,7 @@ public class BigArrays extends AbstractComponent {
|
||||||
*/
|
*/
|
||||||
void adjustBreaker(long delta) {
|
void adjustBreaker(long delta) {
|
||||||
if (this.breakerService != null) {
|
if (this.breakerService != null) {
|
||||||
CircuitBreaker breaker = this.breakerService.getBreaker(CircuitBreaker.Name.REQUEST);
|
CircuitBreaker breaker = this.breakerService.getBreaker(CircuitBreaker.REQUEST);
|
||||||
if (this.checkBreaker == true) {
|
if (this.checkBreaker == true) {
|
||||||
// checking breaker means potentially tripping, but it doesn't
|
// checking breaker means potentially tripping, but it doesn't
|
||||||
// have to if the delta is negative
|
// have to if the delta is negative
|
||||||
|
|
|
@ -53,7 +53,7 @@ public enum GlobalOrdinalsBuilder {
|
||||||
}
|
}
|
||||||
final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT);
|
final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT);
|
||||||
final long memorySizeInBytes = ordinalMap.ramBytesUsed();
|
final long memorySizeInBytes = ordinalMap.ramBytesUsed();
|
||||||
breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA).addWithoutBreaking(memorySizeInBytes);
|
breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(memorySizeInBytes);
|
||||||
|
|
||||||
if (logger.isDebugEnabled()) {
|
if (logger.isDebugEnabled()) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
|
|
|
@ -96,7 +96,7 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
|
||||||
Terms terms = reader.terms(getFieldNames().indexName());
|
Terms terms = reader.terms(getFieldNames().indexName());
|
||||||
AtomicNumericFieldData data = null;
|
AtomicNumericFieldData data = null;
|
||||||
// TODO: Use an actual estimator to estimate before loading.
|
// TODO: Use an actual estimator to estimate before loading.
|
||||||
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA));
|
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA));
|
||||||
if (terms == null) {
|
if (terms == null) {
|
||||||
data = AtomicDoubleFieldData.empty(reader.maxDoc());
|
data = AtomicDoubleFieldData.empty(reader.maxDoc());
|
||||||
estimator.afterLoad(null, data.ramBytesUsed());
|
estimator.afterLoad(null, data.ramBytesUsed());
|
||||||
|
|
|
@ -64,7 +64,7 @@ public class FSTBytesIndexFieldData extends AbstractIndexOrdinalsFieldData {
|
||||||
Terms terms = reader.terms(getFieldNames().indexName());
|
Terms terms = reader.terms(getFieldNames().indexName());
|
||||||
AtomicOrdinalsFieldData data = null;
|
AtomicOrdinalsFieldData data = null;
|
||||||
// TODO: Use an actual estimator to estimate before loading.
|
// TODO: Use an actual estimator to estimate before loading.
|
||||||
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA));
|
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA));
|
||||||
if (terms == null) {
|
if (terms == null) {
|
||||||
data = AbstractAtomicOrdinalsFieldData.empty();
|
data = AbstractAtomicOrdinalsFieldData.empty();
|
||||||
estimator.afterLoad(null, data.ramBytesUsed());
|
estimator.afterLoad(null, data.ramBytesUsed());
|
||||||
|
|
|
@ -94,7 +94,7 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<AtomicNumer
|
||||||
Terms terms = reader.terms(getFieldNames().indexName());
|
Terms terms = reader.terms(getFieldNames().indexName());
|
||||||
AtomicNumericFieldData data = null;
|
AtomicNumericFieldData data = null;
|
||||||
// TODO: Use an actual estimator to estimate before loading.
|
// TODO: Use an actual estimator to estimate before loading.
|
||||||
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA));
|
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA));
|
||||||
if (terms == null) {
|
if (terms == null) {
|
||||||
data = AtomicDoubleFieldData.empty(reader.maxDoc());
|
data = AtomicDoubleFieldData.empty(reader.maxDoc());
|
||||||
estimator.afterLoad(null, data.ramBytesUsed());
|
estimator.afterLoad(null, data.ramBytesUsed());
|
||||||
|
|
|
@ -83,7 +83,7 @@ public class GeoPointCompressedIndexFieldData extends AbstractIndexGeoPointField
|
||||||
Terms terms = reader.terms(getFieldNames().indexName());
|
Terms terms = reader.terms(getFieldNames().indexName());
|
||||||
AtomicGeoPointFieldData data = null;
|
AtomicGeoPointFieldData data = null;
|
||||||
// TODO: Use an actual estimator to estimate before loading.
|
// TODO: Use an actual estimator to estimate before loading.
|
||||||
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA));
|
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA));
|
||||||
if (terms == null) {
|
if (terms == null) {
|
||||||
data = AbstractAtomicGeoPointFieldData.empty(reader.maxDoc());
|
data = AbstractAtomicGeoPointFieldData.empty(reader.maxDoc());
|
||||||
estimator.afterLoad(null, data.ramBytesUsed());
|
estimator.afterLoad(null, data.ramBytesUsed());
|
||||||
|
|
|
@ -65,7 +65,7 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexGeoPointFiel
|
||||||
Terms terms = reader.terms(getFieldNames().indexName());
|
Terms terms = reader.terms(getFieldNames().indexName());
|
||||||
AtomicGeoPointFieldData data = null;
|
AtomicGeoPointFieldData data = null;
|
||||||
// TODO: Use an actual estimator to estimate before loading.
|
// TODO: Use an actual estimator to estimate before loading.
|
||||||
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA));
|
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA));
|
||||||
if (terms == null) {
|
if (terms == null) {
|
||||||
data = AbstractAtomicGeoPointFieldData.empty(reader.maxDoc());
|
data = AbstractAtomicGeoPointFieldData.empty(reader.maxDoc());
|
||||||
estimator.afterLoad(null, data.ramBytesUsed());
|
estimator.afterLoad(null, data.ramBytesUsed());
|
||||||
|
|
|
@ -114,7 +114,7 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
|
||||||
final LeafReader reader = context.reader();
|
final LeafReader reader = context.reader();
|
||||||
Terms terms = reader.terms(getFieldNames().indexName());
|
Terms terms = reader.terms(getFieldNames().indexName());
|
||||||
AtomicNumericFieldData data = null;
|
AtomicNumericFieldData data = null;
|
||||||
PackedArrayEstimator estimator = new PackedArrayEstimator(breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA), getNumericType(), getFieldNames().fullName());
|
PackedArrayEstimator estimator = new PackedArrayEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA), getNumericType(), getFieldNames().fullName());
|
||||||
if (terms == null) {
|
if (terms == null) {
|
||||||
data = AtomicLongFieldData.empty(reader.maxDoc());
|
data = AtomicLongFieldData.empty(reader.maxDoc());
|
||||||
estimator.adjustForNoTerms(data.ramBytesUsed());
|
estimator.adjustForNoTerms(data.ramBytesUsed());
|
||||||
|
|
|
@ -62,7 +62,7 @@ public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData {
|
||||||
LeafReader reader = context.reader();
|
LeafReader reader = context.reader();
|
||||||
AtomicOrdinalsFieldData data = null;
|
AtomicOrdinalsFieldData data = null;
|
||||||
|
|
||||||
PagedBytesEstimator estimator = new PagedBytesEstimator(context, breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA), getFieldNames().fullName());
|
PagedBytesEstimator estimator = new PagedBytesEstimator(context, breakerService.getBreaker(CircuitBreaker.FIELDDATA), getFieldNames().fullName());
|
||||||
Terms terms = reader.terms(getFieldNames().indexName());
|
Terms terms = reader.terms(getFieldNames().indexName());
|
||||||
if (terms == null) {
|
if (terms == null) {
|
||||||
data = AbstractAtomicOrdinalsFieldData.empty();
|
data = AbstractAtomicOrdinalsFieldData.empty();
|
||||||
|
|
|
@ -130,7 +130,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
|
||||||
new ParentChildIntersectTermsEnum(reader, UidFieldMapper.NAME, ParentFieldMapper.NAME),
|
new ParentChildIntersectTermsEnum(reader, UidFieldMapper.NAME, ParentFieldMapper.NAME),
|
||||||
parentTypes
|
parentTypes
|
||||||
);
|
);
|
||||||
ParentChildEstimator estimator = new ParentChildEstimator(breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA), termsEnum);
|
ParentChildEstimator estimator = new ParentChildEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA), termsEnum);
|
||||||
TermsEnum estimatedTermsEnum = estimator.beforeLoad(null);
|
TermsEnum estimatedTermsEnum = estimator.beforeLoad(null);
|
||||||
ObjectObjectOpenHashMap<String, TypeBuilder> typeBuilders = ObjectObjectOpenHashMap.newInstance();
|
ObjectObjectOpenHashMap<String, TypeBuilder> typeBuilders = ObjectObjectOpenHashMap.newInstance();
|
||||||
try {
|
try {
|
||||||
|
@ -344,7 +344,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
|
||||||
fielddata[i] = new GlobalAtomicFieldData(parentTypes, perType, i);
|
fielddata[i] = new GlobalAtomicFieldData(parentTypes, perType, i);
|
||||||
}
|
}
|
||||||
|
|
||||||
breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA).addWithoutBreaking(ramBytesUsed);
|
breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(ramBytesUsed);
|
||||||
if (logger.isDebugEnabled()) {
|
if (logger.isDebugEnabled()) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Global-ordinals[_parent] took {}",
|
"Global-ordinals[_parent] took {}",
|
||||||
|
|
|
@ -19,7 +19,6 @@
|
||||||
|
|
||||||
package org.elasticsearch.indices.breaker;
|
package org.elasticsearch.indices.breaker;
|
||||||
|
|
||||||
import org.elasticsearch.common.breaker.CircuitBreaker;
|
|
||||||
import org.elasticsearch.common.io.stream.StreamInput;
|
import org.elasticsearch.common.io.stream.StreamInput;
|
||||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||||
import org.elasticsearch.common.io.stream.Streamable;
|
import org.elasticsearch.common.io.stream.Streamable;
|
||||||
|
@ -48,9 +47,9 @@ public class AllCircuitBreakerStats implements Streamable, ToXContent {
|
||||||
return this.allStats;
|
return this.allStats;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CircuitBreakerStats getStats(CircuitBreaker.Name name) {
|
public CircuitBreakerStats getStats(String name) {
|
||||||
for (CircuitBreakerStats stats : allStats) {
|
for (CircuitBreakerStats stats : allStats) {
|
||||||
if (stats.getName() == name) {
|
if (stats.getName().equals(name)) {
|
||||||
return stats;
|
return stats;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,23 +27,23 @@ import org.elasticsearch.common.unit.ByteSizeValue;
|
||||||
*/
|
*/
|
||||||
public class BreakerSettings {
|
public class BreakerSettings {
|
||||||
|
|
||||||
private final CircuitBreaker.Name name;
|
private final String name;
|
||||||
private final long limitBytes;
|
private final long limitBytes;
|
||||||
private final double overhead;
|
private final double overhead;
|
||||||
private final CircuitBreaker.Type type;
|
private final CircuitBreaker.Type type;
|
||||||
|
|
||||||
public BreakerSettings(CircuitBreaker.Name name, long limitBytes, double overhead) {
|
public BreakerSettings(String name, long limitBytes, double overhead) {
|
||||||
this(name, limitBytes, overhead, CircuitBreaker.Type.MEMORY);
|
this(name, limitBytes, overhead, CircuitBreaker.Type.MEMORY);
|
||||||
}
|
}
|
||||||
|
|
||||||
public BreakerSettings(CircuitBreaker.Name name, long limitBytes, double overhead, CircuitBreaker.Type type) {
|
public BreakerSettings(String name, long limitBytes, double overhead, CircuitBreaker.Type type) {
|
||||||
this.name = name;
|
this.name = name;
|
||||||
this.limitBytes = limitBytes;
|
this.limitBytes = limitBytes;
|
||||||
this.overhead = overhead;
|
this.overhead = overhead;
|
||||||
this.type = type;
|
this.type = type;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CircuitBreaker.Name getName() {
|
public String getName() {
|
||||||
return this.name;
|
return this.name;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -61,7 +61,7 @@ public class BreakerSettings {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "[" + this.name.toString() +
|
return "[" + this.name +
|
||||||
",type=" + this.type.toString() +
|
",type=" + this.type.toString() +
|
||||||
",limit=" + this.limitBytes + "/" + new ByteSizeValue(this.limitBytes) +
|
",limit=" + this.limitBytes + "/" + new ByteSizeValue(this.limitBytes) +
|
||||||
",overhead=" + this.overhead + "]";
|
",overhead=" + this.overhead + "]";
|
||||||
|
|
|
@ -34,10 +34,17 @@ public abstract class CircuitBreakerService extends AbstractLifecycleComponent<C
|
||||||
super(settings);
|
super(settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Allows to register of a custom circuit breaker.
|
||||||
|
*
|
||||||
|
* @param breakerSettings
|
||||||
|
*/
|
||||||
|
public abstract void registerBreaker(BreakerSettings breakerSettings);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return the breaker that can be used to register estimates against
|
* @return the breaker that can be used to register estimates against
|
||||||
*/
|
*/
|
||||||
public abstract CircuitBreaker getBreaker(CircuitBreaker.Name type);
|
public abstract CircuitBreaker getBreaker(String name);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return stats about all breakers
|
* @return stats about all breakers
|
||||||
|
@ -47,7 +54,7 @@ public abstract class CircuitBreakerService extends AbstractLifecycleComponent<C
|
||||||
/**
|
/**
|
||||||
* @return stats about a specific breaker
|
* @return stats about a specific breaker
|
||||||
*/
|
*/
|
||||||
public abstract CircuitBreakerStats stats(CircuitBreaker.Name name);
|
public abstract CircuitBreakerStats stats(String name);
|
||||||
|
|
||||||
protected void doStart() throws ElasticsearchException {
|
protected void doStart() throws ElasticsearchException {
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,8 +19,6 @@
|
||||||
|
|
||||||
package org.elasticsearch.indices.breaker;
|
package org.elasticsearch.indices.breaker;
|
||||||
|
|
||||||
import org.elasticsearch.Version;
|
|
||||||
import org.elasticsearch.common.breaker.CircuitBreaker;
|
|
||||||
import org.elasticsearch.common.io.stream.StreamInput;
|
import org.elasticsearch.common.io.stream.StreamInput;
|
||||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||||
import org.elasticsearch.common.io.stream.Streamable;
|
import org.elasticsearch.common.io.stream.Streamable;
|
||||||
|
@ -37,7 +35,7 @@ import java.util.Locale;
|
||||||
*/
|
*/
|
||||||
public class CircuitBreakerStats implements Streamable, ToXContent {
|
public class CircuitBreakerStats implements Streamable, ToXContent {
|
||||||
|
|
||||||
private CircuitBreaker.Name name;
|
private String name;
|
||||||
private long limit;
|
private long limit;
|
||||||
private long estimated;
|
private long estimated;
|
||||||
private long trippedCount;
|
private long trippedCount;
|
||||||
|
@ -47,7 +45,7 @@ public class CircuitBreakerStats implements Streamable, ToXContent {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public CircuitBreakerStats(CircuitBreaker.Name name, long limit, long estimated, double overhead, long trippedCount) {
|
public CircuitBreakerStats(String name, long limit, long estimated, double overhead, long trippedCount) {
|
||||||
this.name = name;
|
this.name = name;
|
||||||
this.limit = limit;
|
this.limit = limit;
|
||||||
this.estimated = estimated;
|
this.estimated = estimated;
|
||||||
|
@ -55,7 +53,7 @@ public class CircuitBreakerStats implements Streamable, ToXContent {
|
||||||
this.overhead = overhead;
|
this.overhead = overhead;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CircuitBreaker.Name getName() {
|
public String getName() {
|
||||||
return this.name;
|
return this.name;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -87,7 +85,7 @@ public class CircuitBreakerStats implements Streamable, ToXContent {
|
||||||
estimated = in.readLong();
|
estimated = in.readLong();
|
||||||
overhead = in.readDouble();
|
overhead = in.readDouble();
|
||||||
this.trippedCount = in.readLong();
|
this.trippedCount = in.readLong();
|
||||||
this.name = CircuitBreaker.Name.readFrom(in);
|
this.name = in.readString();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -96,12 +94,12 @@ public class CircuitBreakerStats implements Streamable, ToXContent {
|
||||||
out.writeLong(estimated);
|
out.writeLong(estimated);
|
||||||
out.writeDouble(overhead);
|
out.writeDouble(overhead);
|
||||||
out.writeLong(trippedCount);
|
out.writeLong(trippedCount);
|
||||||
CircuitBreaker.Name.writeTo(name, out);
|
out.writeString(name);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
|
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
|
||||||
builder.startObject(name.toString().toLowerCase(Locale.ROOT));
|
builder.startObject(name.toLowerCase(Locale.ROOT));
|
||||||
builder.field(Fields.LIMIT, limit);
|
builder.field(Fields.LIMIT, limit);
|
||||||
builder.field(Fields.LIMIT_HUMAN, new ByteSizeValue(limit));
|
builder.field(Fields.LIMIT_HUMAN, new ByteSizeValue(limit));
|
||||||
builder.field(Fields.ESTIMATED, estimated);
|
builder.field(Fields.ESTIMATED, estimated);
|
||||||
|
@ -114,7 +112,7 @@ public class CircuitBreakerStats implements Streamable, ToXContent {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "[" + this.name.toString() +
|
return "[" + this.name +
|
||||||
",limit=" + this.limit + "/" + new ByteSizeValue(this.limit) +
|
",limit=" + this.limit + "/" + new ByteSizeValue(this.limit) +
|
||||||
",estimated=" + this.estimated + "/" + new ByteSizeValue(this.estimated) +
|
",estimated=" + this.estimated + "/" + new ByteSizeValue(this.estimated) +
|
||||||
",overhead=" + this.overhead + ",tripped=" + this.trippedCount + "]";
|
",overhead=" + this.overhead + ",tripped=" + this.trippedCount + "]";
|
||||||
|
|
|
@ -19,7 +19,6 @@
|
||||||
|
|
||||||
package org.elasticsearch.indices.breaker;
|
package org.elasticsearch.indices.breaker;
|
||||||
|
|
||||||
import com.google.common.collect.ImmutableMap;
|
|
||||||
import org.elasticsearch.ElasticsearchIllegalStateException;
|
import org.elasticsearch.ElasticsearchIllegalStateException;
|
||||||
import org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker;
|
import org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker;
|
||||||
import org.elasticsearch.common.breaker.CircuitBreaker;
|
import org.elasticsearch.common.breaker.CircuitBreaker;
|
||||||
|
@ -30,9 +29,9 @@ import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||||
import org.elasticsearch.node.settings.NodeSettingsService;
|
import org.elasticsearch.node.settings.NodeSettingsService;
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.ConcurrentMap;
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
import static com.google.common.collect.Lists.newArrayList;
|
import static com.google.common.collect.Lists.newArrayList;
|
||||||
|
@ -43,7 +42,7 @@ import static com.google.common.collect.Lists.newArrayList;
|
||||||
*/
|
*/
|
||||||
public class HierarchyCircuitBreakerService extends CircuitBreakerService {
|
public class HierarchyCircuitBreakerService extends CircuitBreakerService {
|
||||||
|
|
||||||
private volatile ImmutableMap<CircuitBreaker.Name, CircuitBreaker> breakers;
|
private final ConcurrentMap<String, CircuitBreaker> breakers = new ConcurrentHashMap();
|
||||||
|
|
||||||
// Old pre-1.4.0 backwards compatible settings
|
// Old pre-1.4.0 backwards compatible settings
|
||||||
public static final String OLD_CIRCUIT_BREAKER_MAX_BYTES_SETTING = "indices.fielddata.breaker.limit";
|
public static final String OLD_CIRCUIT_BREAKER_MAX_BYTES_SETTING = "indices.fielddata.breaker.limit";
|
||||||
|
@ -94,46 +93,26 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
|
||||||
compatibilityFielddataOverheadDefault = compatibilityFielddataOverhead;
|
compatibilityFielddataOverheadDefault = compatibilityFielddataOverhead;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.fielddataSettings = new BreakerSettings(CircuitBreaker.Name.FIELDDATA,
|
this.fielddataSettings = new BreakerSettings(CircuitBreaker.FIELDDATA,
|
||||||
settings.getAsMemory(FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, compatibilityFielddataLimitDefault).bytes(),
|
settings.getAsMemory(FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, compatibilityFielddataLimitDefault).bytes(),
|
||||||
settings.getAsDouble(FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING, compatibilityFielddataOverheadDefault),
|
settings.getAsDouble(FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING, compatibilityFielddataOverheadDefault),
|
||||||
CircuitBreaker.Type.parseValue(settings.get(FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING, DEFAULT_BREAKER_TYPE))
|
CircuitBreaker.Type.parseValue(settings.get(FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING, DEFAULT_BREAKER_TYPE))
|
||||||
);
|
);
|
||||||
|
|
||||||
this.requestSettings = new BreakerSettings(CircuitBreaker.Name.REQUEST,
|
this.requestSettings = new BreakerSettings(CircuitBreaker.REQUEST,
|
||||||
settings.getAsMemory(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, DEFAULT_REQUEST_BREAKER_LIMIT).bytes(),
|
settings.getAsMemory(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, DEFAULT_REQUEST_BREAKER_LIMIT).bytes(),
|
||||||
settings.getAsDouble(REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING, 1.0),
|
settings.getAsDouble(REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING, 1.0),
|
||||||
CircuitBreaker.Type.parseValue(settings.get(REQUEST_CIRCUIT_BREAKER_TYPE_SETTING, DEFAULT_BREAKER_TYPE))
|
CircuitBreaker.Type.parseValue(settings.get(REQUEST_CIRCUIT_BREAKER_TYPE_SETTING, DEFAULT_BREAKER_TYPE))
|
||||||
);
|
);
|
||||||
|
|
||||||
// Validate the configured settings
|
this.parentSettings = new BreakerSettings(CircuitBreaker.PARENT,
|
||||||
validateSettings(new BreakerSettings[] {this.requestSettings, this.fielddataSettings});
|
|
||||||
|
|
||||||
this.parentSettings = new BreakerSettings(CircuitBreaker.Name.PARENT,
|
|
||||||
settings.getAsMemory(TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, DEFAULT_TOTAL_CIRCUIT_BREAKER_LIMIT).bytes(), 1.0, CircuitBreaker.Type.PARENT);
|
settings.getAsMemory(TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, DEFAULT_TOTAL_CIRCUIT_BREAKER_LIMIT).bytes(), 1.0, CircuitBreaker.Type.PARENT);
|
||||||
if (logger.isTraceEnabled()) {
|
if (logger.isTraceEnabled()) {
|
||||||
logger.trace("parent circuit breaker with settings {}", this.parentSettings);
|
logger.trace("parent circuit breaker with settings {}", this.parentSettings);
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<CircuitBreaker.Name, CircuitBreaker> tempBreakers = new HashMap<>();
|
registerBreaker(this.requestSettings);
|
||||||
|
registerBreaker(this.fielddataSettings);
|
||||||
CircuitBreaker fielddataBreaker;
|
|
||||||
if (fielddataSettings.getType() == CircuitBreaker.Type.NOOP) {
|
|
||||||
fielddataBreaker = new NoopCircuitBreaker(CircuitBreaker.Name.FIELDDATA);
|
|
||||||
} else {
|
|
||||||
fielddataBreaker = new ChildMemoryCircuitBreaker(fielddataSettings, logger, this, CircuitBreaker.Name.FIELDDATA);
|
|
||||||
}
|
|
||||||
|
|
||||||
CircuitBreaker requestBreaker;
|
|
||||||
if (requestSettings.getType() == CircuitBreaker.Type.NOOP) {
|
|
||||||
requestBreaker = new NoopCircuitBreaker(CircuitBreaker.Name.REQUEST);
|
|
||||||
} else {
|
|
||||||
requestBreaker = new ChildMemoryCircuitBreaker(requestSettings, logger, this, CircuitBreaker.Name.REQUEST);
|
|
||||||
}
|
|
||||||
|
|
||||||
tempBreakers.put(CircuitBreaker.Name.FIELDDATA, fielddataBreaker);
|
|
||||||
tempBreakers.put(CircuitBreaker.Name.REQUEST, requestBreaker);
|
|
||||||
this.breakers = ImmutableMap.copyOf(tempBreakers);
|
|
||||||
|
|
||||||
nodeSettingsService.addListener(new ApplySettings());
|
nodeSettingsService.addListener(new ApplySettings());
|
||||||
}
|
}
|
||||||
|
@ -142,73 +121,43 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onRefreshSettings(Settings settings) {
|
public void onRefreshSettings(Settings settings) {
|
||||||
boolean changed = false;
|
|
||||||
|
|
||||||
// Fielddata settings
|
// Fielddata settings
|
||||||
BreakerSettings newFielddataSettings = HierarchyCircuitBreakerService.this.fielddataSettings;
|
|
||||||
ByteSizeValue newFielddataMax = settings.getAsMemory(FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, null);
|
ByteSizeValue newFielddataMax = settings.getAsMemory(FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, null);
|
||||||
Double newFielddataOverhead = settings.getAsDouble(FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING, null);
|
Double newFielddataOverhead = settings.getAsDouble(FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING, null);
|
||||||
if (newFielddataMax != null || newFielddataOverhead != null) {
|
if (newFielddataMax != null || newFielddataOverhead != null) {
|
||||||
changed = true;
|
|
||||||
long newFielddataLimitBytes = newFielddataMax == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getLimit() : newFielddataMax.bytes();
|
long newFielddataLimitBytes = newFielddataMax == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getLimit() : newFielddataMax.bytes();
|
||||||
newFielddataOverhead = newFielddataOverhead == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getOverhead() : newFielddataOverhead;
|
newFielddataOverhead = newFielddataOverhead == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getOverhead() : newFielddataOverhead;
|
||||||
|
|
||||||
newFielddataSettings = new BreakerSettings(CircuitBreaker.Name.FIELDDATA, newFielddataLimitBytes, newFielddataOverhead,
|
BreakerSettings newFielddataSettings = new BreakerSettings(CircuitBreaker.FIELDDATA, newFielddataLimitBytes, newFielddataOverhead,
|
||||||
HierarchyCircuitBreakerService.this.fielddataSettings.getType());
|
HierarchyCircuitBreakerService.this.fielddataSettings.getType());
|
||||||
|
registerBreaker(newFielddataSettings);
|
||||||
|
HierarchyCircuitBreakerService.this.fielddataSettings = newFielddataSettings;
|
||||||
|
logger.info("Updated breaker settings fielddata: {}", newFielddataSettings);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Request settings
|
// Request settings
|
||||||
BreakerSettings newRequestSettings = HierarchyCircuitBreakerService.this.requestSettings;
|
|
||||||
ByteSizeValue newRequestMax = settings.getAsMemory(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, null);
|
ByteSizeValue newRequestMax = settings.getAsMemory(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, null);
|
||||||
Double newRequestOverhead = settings.getAsDouble(REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING, null);
|
Double newRequestOverhead = settings.getAsDouble(REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING, null);
|
||||||
if (newRequestMax != null || newRequestOverhead != null) {
|
if (newRequestMax != null || newRequestOverhead != null) {
|
||||||
changed = true;
|
|
||||||
long newRequestLimitBytes = newRequestMax == null ? HierarchyCircuitBreakerService.this.requestSettings.getLimit() : newRequestMax.bytes();
|
long newRequestLimitBytes = newRequestMax == null ? HierarchyCircuitBreakerService.this.requestSettings.getLimit() : newRequestMax.bytes();
|
||||||
newRequestOverhead = newRequestOverhead == null ? HierarchyCircuitBreakerService.this.requestSettings.getOverhead() : newRequestOverhead;
|
newRequestOverhead = newRequestOverhead == null ? HierarchyCircuitBreakerService.this.requestSettings.getOverhead() : newRequestOverhead;
|
||||||
|
|
||||||
newRequestSettings = new BreakerSettings(CircuitBreaker.Name.REQUEST, newRequestLimitBytes, newRequestOverhead,
|
BreakerSettings newRequestSettings = new BreakerSettings(CircuitBreaker.REQUEST, newRequestLimitBytes, newRequestOverhead,
|
||||||
HierarchyCircuitBreakerService.this.requestSettings.getType());
|
HierarchyCircuitBreakerService.this.requestSettings.getType());
|
||||||
|
registerBreaker(newRequestSettings);
|
||||||
|
HierarchyCircuitBreakerService.this.requestSettings = newRequestSettings;
|
||||||
|
logger.info("Updated breaker settings request: {}", newRequestSettings);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parent settings
|
// Parent settings
|
||||||
BreakerSettings newParentSettings = HierarchyCircuitBreakerService.this.parentSettings;
|
|
||||||
long oldParentMax = HierarchyCircuitBreakerService.this.parentSettings.getLimit();
|
long oldParentMax = HierarchyCircuitBreakerService.this.parentSettings.getLimit();
|
||||||
ByteSizeValue newParentMax = settings.getAsMemory(TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, null);
|
ByteSizeValue newParentMax = settings.getAsMemory(TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, null);
|
||||||
if (newParentMax != null && (newParentMax.bytes() != oldParentMax)) {
|
if (newParentMax != null && (newParentMax.bytes() != oldParentMax)) {
|
||||||
changed = true;
|
BreakerSettings newParentSettings = new BreakerSettings(CircuitBreaker.PARENT, newParentMax.bytes(), 1.0, CircuitBreaker.Type.PARENT);
|
||||||
newParentSettings = new BreakerSettings(CircuitBreaker.Name.PARENT, newParentMax.bytes(), 1.0, CircuitBreaker.Type.PARENT);
|
validateSettings(new BreakerSettings[]{newParentSettings});
|
||||||
}
|
|
||||||
|
|
||||||
if (changed) {
|
|
||||||
// change all the things
|
|
||||||
validateSettings(new BreakerSettings[]{newFielddataSettings, newRequestSettings});
|
|
||||||
logger.info("Updating settings parent: {}, fielddata: {}, request: {}", newParentSettings, newFielddataSettings, newRequestSettings);
|
|
||||||
HierarchyCircuitBreakerService.this.parentSettings = newParentSettings;
|
HierarchyCircuitBreakerService.this.parentSettings = newParentSettings;
|
||||||
HierarchyCircuitBreakerService.this.fielddataSettings = newFielddataSettings;
|
logger.info("Updated breaker settings parent: {}", newParentSettings);
|
||||||
HierarchyCircuitBreakerService.this.requestSettings = newRequestSettings;
|
|
||||||
|
|
||||||
Map<CircuitBreaker.Name, CircuitBreaker> tempBreakers = new HashMap<>();
|
|
||||||
CircuitBreaker fielddataBreaker;
|
|
||||||
if (newFielddataSettings.getType() == CircuitBreaker.Type.NOOP) {
|
|
||||||
fielddataBreaker = new NoopCircuitBreaker(CircuitBreaker.Name.FIELDDATA);
|
|
||||||
} else {
|
|
||||||
fielddataBreaker = new ChildMemoryCircuitBreaker(newFielddataSettings,
|
|
||||||
(ChildMemoryCircuitBreaker) HierarchyCircuitBreakerService.this.breakers.get(CircuitBreaker.Name.FIELDDATA),
|
|
||||||
logger, HierarchyCircuitBreakerService.this, CircuitBreaker.Name.FIELDDATA);
|
|
||||||
}
|
|
||||||
|
|
||||||
CircuitBreaker requestBreaker;
|
|
||||||
if (newRequestSettings.getType() == CircuitBreaker.Type.NOOP) {
|
|
||||||
requestBreaker = new NoopCircuitBreaker(CircuitBreaker.Name.REQUEST);
|
|
||||||
} else {
|
|
||||||
requestBreaker = new ChildMemoryCircuitBreaker(newRequestSettings,
|
|
||||||
(ChildMemoryCircuitBreaker)HierarchyCircuitBreakerService.this.breakers.get(CircuitBreaker.Name.REQUEST),
|
|
||||||
logger, HierarchyCircuitBreakerService.this, CircuitBreaker.Name.REQUEST);
|
|
||||||
}
|
|
||||||
|
|
||||||
tempBreakers.put(CircuitBreaker.Name.FIELDDATA, fielddataBreaker);
|
|
||||||
tempBreakers.put(CircuitBreaker.Name.REQUEST, requestBreaker);
|
|
||||||
HierarchyCircuitBreakerService.this.breakers = ImmutableMap.copyOf(tempBreakers);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -231,7 +180,7 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public CircuitBreaker getBreaker(CircuitBreaker.Name name) {
|
public CircuitBreaker getBreaker(String name) {
|
||||||
return this.breakers.get(name);
|
return this.breakers.get(name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -246,13 +195,13 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
|
||||||
parentEstimated += breaker.getUsed();
|
parentEstimated += breaker.getUsed();
|
||||||
}
|
}
|
||||||
// Manually add the parent breaker settings since they aren't part of the breaker map
|
// Manually add the parent breaker settings since they aren't part of the breaker map
|
||||||
allStats.add(new CircuitBreakerStats(CircuitBreaker.Name.PARENT, parentSettings.getLimit(),
|
allStats.add(new CircuitBreakerStats(CircuitBreaker.PARENT, parentSettings.getLimit(),
|
||||||
parentEstimated, 1.0, parentTripCount.get()));
|
parentEstimated, 1.0, parentTripCount.get()));
|
||||||
return new AllCircuitBreakerStats(allStats.toArray(new CircuitBreakerStats[allStats.size()]));
|
return new AllCircuitBreakerStats(allStats.toArray(new CircuitBreakerStats[allStats.size()]));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public CircuitBreakerStats stats(CircuitBreaker.Name name) {
|
public CircuitBreakerStats stats(String name) {
|
||||||
CircuitBreaker breaker = this.breakers.get(name);
|
CircuitBreaker breaker = this.breakers.get(name);
|
||||||
return new CircuitBreakerStats(breaker.getName(), breaker.getLimit(), breaker.getUsed(), breaker.getOverhead(), breaker.getTrippedCount());
|
return new CircuitBreakerStats(breaker.getName(), breaker.getLimit(), breaker.getUsed(), breaker.getOverhead(), breaker.getTrippedCount());
|
||||||
}
|
}
|
||||||
|
@ -271,10 +220,45 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
|
||||||
long parentLimit = this.parentSettings.getLimit();
|
long parentLimit = this.parentSettings.getLimit();
|
||||||
if (totalUsed > parentLimit) {
|
if (totalUsed > parentLimit) {
|
||||||
this.parentTripCount.incrementAndGet();
|
this.parentTripCount.incrementAndGet();
|
||||||
throw new CircuitBreakingException("[PARENT] Data too large, data for [" +
|
throw new CircuitBreakingException("[parent] Data too large, data for [" +
|
||||||
label + "] would be larger than limit of [" +
|
label + "] would be larger than limit of [" +
|
||||||
parentLimit + "/" + new ByteSizeValue(parentLimit) + "]",
|
parentLimit + "/" + new ByteSizeValue(parentLimit) + "]",
|
||||||
totalUsed, parentLimit);
|
totalUsed, parentLimit);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Allows to register a custom circuit breaker.
|
||||||
|
* Warning: Will overwrite any existing custom breaker with the same name.
|
||||||
|
*
|
||||||
|
* @param breakerSettings
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void registerBreaker(BreakerSettings breakerSettings) {
|
||||||
|
// Validate the settings
|
||||||
|
validateSettings(new BreakerSettings[] {breakerSettings});
|
||||||
|
|
||||||
|
if (breakerSettings.getType() == CircuitBreaker.Type.NOOP) {
|
||||||
|
CircuitBreaker breaker = new NoopCircuitBreaker(breakerSettings.getName());
|
||||||
|
breakers.put(breakerSettings.getName(), breaker);
|
||||||
|
} else {
|
||||||
|
CircuitBreaker oldBreaker;
|
||||||
|
CircuitBreaker breaker = new ChildMemoryCircuitBreaker(breakerSettings,
|
||||||
|
logger, this, breakerSettings.getName());
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
oldBreaker = breakers.putIfAbsent(breakerSettings.getName(), breaker);
|
||||||
|
if (oldBreaker == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
breaker = new ChildMemoryCircuitBreaker(breakerSettings,
|
||||||
|
(ChildMemoryCircuitBreaker)oldBreaker, logger, this, breakerSettings.getName());
|
||||||
|
|
||||||
|
if (breakers.replace(breakerSettings.getName(), oldBreaker, breaker)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,24 +28,29 @@ import org.elasticsearch.common.settings.ImmutableSettings;
|
||||||
*/
|
*/
|
||||||
public class NoneCircuitBreakerService extends CircuitBreakerService {
|
public class NoneCircuitBreakerService extends CircuitBreakerService {
|
||||||
|
|
||||||
private final CircuitBreaker breaker = new NoopCircuitBreaker(CircuitBreaker.Name.FIELDDATA);
|
private final CircuitBreaker breaker = new NoopCircuitBreaker(CircuitBreaker.FIELDDATA);
|
||||||
|
|
||||||
public NoneCircuitBreakerService() {
|
public NoneCircuitBreakerService() {
|
||||||
super(ImmutableSettings.EMPTY);
|
super(ImmutableSettings.EMPTY);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public CircuitBreaker getBreaker(CircuitBreaker.Name name) {
|
public CircuitBreaker getBreaker(String name) {
|
||||||
return breaker;
|
return breaker;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public AllCircuitBreakerStats stats() {
|
public AllCircuitBreakerStats stats() {
|
||||||
return new AllCircuitBreakerStats(new CircuitBreakerStats[] {stats(CircuitBreaker.Name.FIELDDATA)});
|
return new AllCircuitBreakerStats(new CircuitBreakerStats[] {stats(CircuitBreaker.FIELDDATA)});
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public CircuitBreakerStats stats(CircuitBreaker.Name name) {
|
public CircuitBreakerStats stats(String name) {
|
||||||
return new CircuitBreakerStats(CircuitBreaker.Name.FIELDDATA, -1, -1, 0, 0);
|
return new CircuitBreakerStats(CircuitBreaker.FIELDDATA, -1, -1, 0, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void registerBreaker(BreakerSettings breakerSettings) {
|
||||||
|
// ignore
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -49,7 +49,7 @@ public class IndicesFieldDataCacheListener implements IndexFieldDataCache.Listen
|
||||||
@Override
|
@Override
|
||||||
public void onUnload(FieldMapper.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes) {
|
public void onUnload(FieldMapper.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes) {
|
||||||
assert sizeInBytes >= 0 : "When reducing circuit breaker, it should be adjusted with a number higher or equal to 0 and not [" + sizeInBytes + "]";
|
assert sizeInBytes >= 0 : "When reducing circuit breaker, it should be adjusted with a number higher or equal to 0 and not [" + sizeInBytes + "]";
|
||||||
circuitBreakerService.getBreaker(CircuitBreaker.Name.FIELDDATA).addWithoutBreaking(-sizeInBytes);
|
circuitBreakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(-sizeInBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -94,7 +94,7 @@ public class MemoryCircuitBreakerTests extends ElasticsearchTestCase {
|
||||||
final CircuitBreakerService service = new HierarchyCircuitBreakerService(ImmutableSettings.EMPTY, new NodeSettingsService(ImmutableSettings.EMPTY)) {
|
final CircuitBreakerService service = new HierarchyCircuitBreakerService(ImmutableSettings.EMPTY, new NodeSettingsService(ImmutableSettings.EMPTY)) {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public CircuitBreaker getBreaker(CircuitBreaker.Name type) {
|
public CircuitBreaker getBreaker(String name) {
|
||||||
return breakerRef.get();
|
return breakerRef.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -103,9 +103,9 @@ public class MemoryCircuitBreakerTests extends ElasticsearchTestCase {
|
||||||
// never trip
|
// never trip
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
final BreakerSettings settings = new BreakerSettings(CircuitBreaker.Name.REQUEST, (BYTES_PER_THREAD * NUM_THREADS) - 1, 1.0);
|
final BreakerSettings settings = new BreakerSettings(CircuitBreaker.REQUEST, (BYTES_PER_THREAD * NUM_THREADS) - 1, 1.0);
|
||||||
final ChildMemoryCircuitBreaker breaker = new ChildMemoryCircuitBreaker(settings, logger,
|
final ChildMemoryCircuitBreaker breaker = new ChildMemoryCircuitBreaker(settings, logger,
|
||||||
(HierarchyCircuitBreakerService)service, CircuitBreaker.Name.REQUEST);
|
(HierarchyCircuitBreakerService)service, CircuitBreaker.REQUEST);
|
||||||
breakerRef.set(breaker);
|
breakerRef.set(breaker);
|
||||||
|
|
||||||
for (int i = 0; i < NUM_THREADS; i++) {
|
for (int i = 0; i < NUM_THREADS; i++) {
|
||||||
|
@ -155,23 +155,23 @@ public class MemoryCircuitBreakerTests extends ElasticsearchTestCase {
|
||||||
final CircuitBreakerService service = new HierarchyCircuitBreakerService(ImmutableSettings.EMPTY, new NodeSettingsService(ImmutableSettings.EMPTY)) {
|
final CircuitBreakerService service = new HierarchyCircuitBreakerService(ImmutableSettings.EMPTY, new NodeSettingsService(ImmutableSettings.EMPTY)) {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public CircuitBreaker getBreaker(CircuitBreaker.Name type) {
|
public CircuitBreaker getBreaker(String name) {
|
||||||
return breakerRef.get();
|
return breakerRef.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void checkParentLimit(String label) throws CircuitBreakingException {
|
public void checkParentLimit(String label) throws CircuitBreakingException {
|
||||||
// Parent will trip right before regular breaker would trip
|
// Parent will trip right before regular breaker would trip
|
||||||
if (getBreaker(CircuitBreaker.Name.REQUEST).getUsed() > parentLimit) {
|
if (getBreaker(CircuitBreaker.REQUEST).getUsed() > parentLimit) {
|
||||||
parentTripped.incrementAndGet();
|
parentTripped.incrementAndGet();
|
||||||
logger.info("--> parent tripped");
|
logger.info("--> parent tripped");
|
||||||
throw new CircuitBreakingException("parent tripped");
|
throw new CircuitBreakingException("parent tripped");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
final BreakerSettings settings = new BreakerSettings(CircuitBreaker.Name.REQUEST, childLimit, 1.0);
|
final BreakerSettings settings = new BreakerSettings(CircuitBreaker.REQUEST, childLimit, 1.0);
|
||||||
final ChildMemoryCircuitBreaker breaker = new ChildMemoryCircuitBreaker(settings, logger,
|
final ChildMemoryCircuitBreaker breaker = new ChildMemoryCircuitBreaker(settings, logger,
|
||||||
(HierarchyCircuitBreakerService)service, CircuitBreaker.Name.REQUEST);
|
(HierarchyCircuitBreakerService)service, CircuitBreaker.REQUEST);
|
||||||
breakerRef.set(breaker);
|
breakerRef.set(breaker);
|
||||||
|
|
||||||
for (int i = 0; i < NUM_THREADS; i++) {
|
for (int i = 0; i < NUM_THREADS; i++) {
|
||||||
|
|
|
@ -347,7 +347,7 @@ public class BigArraysTests extends ElasticsearchSingleNodeTest {
|
||||||
} catch (InvocationTargetException e) {
|
} catch (InvocationTargetException e) {
|
||||||
assertTrue(e.getCause() instanceof CircuitBreakingException);
|
assertTrue(e.getCause() instanceof CircuitBreakingException);
|
||||||
}
|
}
|
||||||
assertEquals(0, hcbs.getBreaker(CircuitBreaker.Name.REQUEST).getUsed());
|
assertEquals(0, hcbs.getBreaker(CircuitBreaker.REQUEST).getUsed());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -373,9 +373,9 @@ public class BigArraysTests extends ElasticsearchSingleNodeTest {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
assertEquals(array.ramBytesUsed(), hcbs.getBreaker(CircuitBreaker.Name.REQUEST).getUsed());
|
assertEquals(array.ramBytesUsed(), hcbs.getBreaker(CircuitBreaker.REQUEST).getUsed());
|
||||||
array.close();
|
array.close();
|
||||||
assertEquals(0, hcbs.getBreaker(CircuitBreaker.Name.REQUEST).getUsed());
|
assertEquals(0, hcbs.getBreaker(CircuitBreaker.REQUEST).getUsed());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -27,8 +27,11 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
|
||||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||||
import org.elasticsearch.client.Client;
|
import org.elasticsearch.client.Client;
|
||||||
import org.elasticsearch.common.breaker.CircuitBreaker;
|
import org.elasticsearch.common.breaker.CircuitBreaker;
|
||||||
|
import org.elasticsearch.common.breaker.CircuitBreakingException;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.common.unit.TimeValue;
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.indices.breaker.BreakerSettings;
|
||||||
|
import org.elasticsearch.indices.breaker.CircuitBreakerService;
|
||||||
import org.elasticsearch.indices.breaker.CircuitBreakerStats;
|
import org.elasticsearch.indices.breaker.CircuitBreakerStats;
|
||||||
import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
|
import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
|
||||||
import org.elasticsearch.rest.RestStatus;
|
import org.elasticsearch.rest.RestStatus;
|
||||||
|
@ -94,10 +97,10 @@ public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest {
|
||||||
private boolean noopBreakerUsed() {
|
private boolean noopBreakerUsed() {
|
||||||
NodesStatsResponse stats = client().admin().cluster().prepareNodesStats().setBreaker(true).get();
|
NodesStatsResponse stats = client().admin().cluster().prepareNodesStats().setBreaker(true).get();
|
||||||
for (NodeStats nodeStats : stats) {
|
for (NodeStats nodeStats : stats) {
|
||||||
if (nodeStats.getBreaker().getStats(CircuitBreaker.Name.REQUEST).getLimit() == 0) {
|
if (nodeStats.getBreaker().getStats(CircuitBreaker.REQUEST).getLimit() == 0) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (nodeStats.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA).getLimit() == 0) {
|
if (nodeStats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getLimit() == 0) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -144,7 +147,7 @@ public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest {
|
||||||
NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().setBreaker(true).get();
|
NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().setBreaker(true).get();
|
||||||
int breaks = 0;
|
int breaks = 0;
|
||||||
for (NodeStats stat : stats.getNodes()) {
|
for (NodeStats stat : stats.getNodes()) {
|
||||||
CircuitBreakerStats breakerStats = stat.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA);
|
CircuitBreakerStats breakerStats = stat.getBreaker().getStats(CircuitBreaker.FIELDDATA);
|
||||||
breaks += breakerStats.getTrippedCount();
|
breaks += breakerStats.getTrippedCount();
|
||||||
}
|
}
|
||||||
assertThat(breaks, greaterThanOrEqualTo(1));
|
assertThat(breaks, greaterThanOrEqualTo(1));
|
||||||
|
@ -195,7 +198,7 @@ public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest {
|
||||||
NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().setBreaker(true).get();
|
NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().setBreaker(true).get();
|
||||||
int breaks = 0;
|
int breaks = 0;
|
||||||
for (NodeStats stat : stats.getNodes()) {
|
for (NodeStats stat : stats.getNodes()) {
|
||||||
CircuitBreakerStats breakerStats = stat.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA);
|
CircuitBreakerStats breakerStats = stat.getBreaker().getStats(CircuitBreaker.FIELDDATA);
|
||||||
breaks += breakerStats.getTrippedCount();
|
breaks += breakerStats.getTrippedCount();
|
||||||
}
|
}
|
||||||
assertThat(breaks, greaterThanOrEqualTo(1));
|
assertThat(breaks, greaterThanOrEqualTo(1));
|
||||||
|
@ -224,7 +227,7 @@ public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest {
|
||||||
|
|
||||||
// We need the request limit beforehand, just from a single node because the limit should always be the same
|
// We need the request limit beforehand, just from a single node because the limit should always be the same
|
||||||
long beforeReqLimit = client.admin().cluster().prepareNodesStats().setBreaker(true).get()
|
long beforeReqLimit = client.admin().cluster().prepareNodesStats().setBreaker(true).get()
|
||||||
.getNodes()[0].getBreaker().getStats(CircuitBreaker.Name.REQUEST).getLimit();
|
.getNodes()[0].getBreaker().getStats(CircuitBreaker.REQUEST).getLimit();
|
||||||
|
|
||||||
Settings resetSettings = settingsBuilder()
|
Settings resetSettings = settingsBuilder()
|
||||||
.put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, "10b")
|
.put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, "10b")
|
||||||
|
@ -237,7 +240,7 @@ public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest {
|
||||||
client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
|
client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
|
||||||
fail("should have thrown an exception");
|
fail("should have thrown an exception");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
String errMsg = "[FIELDDATA] Data too large, data for [test] would be larger than limit of [10/10b]";
|
String errMsg = "[fielddata] Data too large, data for [test] would be larger than limit of [10/10b]";
|
||||||
assertThat("Exception: " + ExceptionsHelper.unwrapCause(e) + " should contain a CircuitBreakingException",
|
assertThat("Exception: " + ExceptionsHelper.unwrapCause(e) + " should contain a CircuitBreakingException",
|
||||||
ExceptionsHelper.unwrapCause(e).getMessage().contains(errMsg), equalTo(true));
|
ExceptionsHelper.unwrapCause(e).getMessage().contains(errMsg), equalTo(true));
|
||||||
}
|
}
|
||||||
|
@ -259,7 +262,7 @@ public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest {
|
||||||
client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
|
client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
|
||||||
fail("should have thrown an exception");
|
fail("should have thrown an exception");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
String errMsg = "[PARENT] Data too large, data for [test] would be larger than limit of [15/15b]";
|
String errMsg = "[parent] Data too large, data for [test] would be larger than limit of [15/15b]";
|
||||||
assertThat("Exception: " + ExceptionsHelper.unwrapCause(e) + " should contain a CircuitBreakingException",
|
assertThat("Exception: " + ExceptionsHelper.unwrapCause(e) + " should contain a CircuitBreakingException",
|
||||||
ExceptionsHelper.unwrapCause(e).getMessage().contains(errMsg), equalTo(true));
|
ExceptionsHelper.unwrapCause(e).getMessage().contains(errMsg), equalTo(true));
|
||||||
}
|
}
|
||||||
|
@ -293,7 +296,7 @@ public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest {
|
||||||
client.prepareSearch("cb-test").setQuery(matchAllQuery()).addAggregation(cardinality("card").field("test")).get();
|
client.prepareSearch("cb-test").setQuery(matchAllQuery()).addAggregation(cardinality("card").field("test")).get();
|
||||||
fail("aggregation should have tripped the breaker");
|
fail("aggregation should have tripped the breaker");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
String errMsg = "CircuitBreakingException[[REQUEST] Data too large, data for [<reused_arrays>] would be larger than limit of [10/10b]]";
|
String errMsg = "CircuitBreakingException[[request] Data too large, data for [<reused_arrays>] would be larger than limit of [10/10b]]";
|
||||||
assertThat("Exception: " + ExceptionsHelper.unwrapCause(e) + " should contain a CircuitBreakingException",
|
assertThat("Exception: " + ExceptionsHelper.unwrapCause(e) + " should contain a CircuitBreakingException",
|
||||||
ExceptionsHelper.unwrapCause(e).getMessage().contains(errMsg), equalTo(true));
|
ExceptionsHelper.unwrapCause(e).getMessage().contains(errMsg), equalTo(true));
|
||||||
}
|
}
|
||||||
|
@ -309,10 +312,40 @@ public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest {
|
||||||
.clear().setBreaker(true).get(new TimeValue(15, TimeUnit.SECONDS));
|
.clear().setBreaker(true).get(new TimeValue(15, TimeUnit.SECONDS));
|
||||||
for (NodeStats nStats : resp.getNodes()) {
|
for (NodeStats nStats : resp.getNodes()) {
|
||||||
assertThat("fielddata breaker never reset back to 0",
|
assertThat("fielddata breaker never reset back to 0",
|
||||||
nStats.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA).getEstimated(),
|
nStats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(),
|
||||||
equalTo(0L));
|
equalTo(0L));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, 30, TimeUnit.SECONDS);
|
}, 30, TimeUnit.SECONDS);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCustomCircuitBreakerRegistration() throws Exception {
|
||||||
|
Iterable<CircuitBreakerService> serviceIter = internalCluster().getInstances(CircuitBreakerService.class);
|
||||||
|
|
||||||
|
final String breakerName = "customBreaker";
|
||||||
|
BreakerSettings breakerSettings = new BreakerSettings(breakerName, 8, 1.03);
|
||||||
|
CircuitBreaker breaker = null;
|
||||||
|
|
||||||
|
for (CircuitBreakerService s : serviceIter) {
|
||||||
|
s.registerBreaker(breakerSettings);
|
||||||
|
breaker = s.getBreaker(breakerSettings.getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (breaker != null) {
|
||||||
|
try {
|
||||||
|
breaker.addEstimateBytesAndMaybeBreak(16, "test");
|
||||||
|
} catch (CircuitBreakingException e) {
|
||||||
|
// ignore, we forced a circuit break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
NodesStatsResponse stats = client().admin().cluster().prepareNodesStats().clear().setBreaker(true).get();
|
||||||
|
int breaks = 0;
|
||||||
|
for (NodeStats stat : stats.getNodes()) {
|
||||||
|
CircuitBreakerStats breakerStats = stat.getBreaker().getStats(breakerName);
|
||||||
|
breaks += breakerStats.getTrippedCount();
|
||||||
|
}
|
||||||
|
assertThat(breaks, greaterThanOrEqualTo(1));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,11 +22,13 @@ package org.elasticsearch.indices.memory.breaker;
|
||||||
import org.elasticsearch.common.breaker.CircuitBreaker;
|
import org.elasticsearch.common.breaker.CircuitBreaker;
|
||||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||||
import org.elasticsearch.indices.breaker.BreakerSettings;
|
import org.elasticsearch.indices.breaker.BreakerSettings;
|
||||||
|
import org.elasticsearch.indices.breaker.CircuitBreakerService;
|
||||||
import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
|
import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
|
||||||
|
import org.elasticsearch.node.settings.NodeSettingsService;
|
||||||
import org.elasticsearch.test.ElasticsearchTestCase;
|
import org.elasticsearch.test.ElasticsearchTestCase;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import static org.hamcrest.Matchers.equalTo;
|
import static org.hamcrest.Matchers.*;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Unit tests for the circuit breaker
|
* Unit tests for the circuit breaker
|
||||||
|
@ -40,21 +42,21 @@ public class CircuitBreakerUnitTests extends ElasticsearchTestCase {
|
||||||
@Test
|
@Test
|
||||||
public void testBreakerSettingsValidationWithValidSettings() {
|
public void testBreakerSettingsValidationWithValidSettings() {
|
||||||
// parent: {:limit 70}, fd: {:limit 50}, request: {:limit 20}
|
// parent: {:limit 70}, fd: {:limit 50}, request: {:limit 20}
|
||||||
BreakerSettings fd = new BreakerSettings(CircuitBreaker.Name.FIELDDATA, pctBytes("50%"), 1.0);
|
BreakerSettings fd = new BreakerSettings(CircuitBreaker.FIELDDATA, pctBytes("50%"), 1.0);
|
||||||
BreakerSettings request = new BreakerSettings(CircuitBreaker.Name.REQUEST, pctBytes("20%"), 1.0);
|
BreakerSettings request = new BreakerSettings(CircuitBreaker.REQUEST, pctBytes("20%"), 1.0);
|
||||||
HierarchyCircuitBreakerService.validateSettings(new BreakerSettings[]{fd, request});
|
HierarchyCircuitBreakerService.validateSettings(new BreakerSettings[]{fd, request});
|
||||||
|
|
||||||
// parent: {:limit 70}, fd: {:limit 40}, request: {:limit 30}
|
// parent: {:limit 70}, fd: {:limit 40}, request: {:limit 30}
|
||||||
fd = new BreakerSettings(CircuitBreaker.Name.FIELDDATA, pctBytes("40%"), 1.0);
|
fd = new BreakerSettings(CircuitBreaker.FIELDDATA, pctBytes("40%"), 1.0);
|
||||||
request = new BreakerSettings(CircuitBreaker.Name.REQUEST, pctBytes("30%"), 1.0);
|
request = new BreakerSettings(CircuitBreaker.REQUEST, pctBytes("30%"), 1.0);
|
||||||
HierarchyCircuitBreakerService.validateSettings(new BreakerSettings[]{fd, request});
|
HierarchyCircuitBreakerService.validateSettings(new BreakerSettings[]{fd, request});
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testBreakerSettingsValidationNegativeOverhead() {
|
public void testBreakerSettingsValidationNegativeOverhead() {
|
||||||
// parent: {:limit 70}, fd: {:limit 50}, request: {:limit 20}
|
// parent: {:limit 70}, fd: {:limit 50}, request: {:limit 20}
|
||||||
BreakerSettings fd = new BreakerSettings(CircuitBreaker.Name.FIELDDATA, pctBytes("50%"), -0.1);
|
BreakerSettings fd = new BreakerSettings(CircuitBreaker.FIELDDATA, pctBytes("50%"), -0.1);
|
||||||
BreakerSettings request = new BreakerSettings(CircuitBreaker.Name.REQUEST, pctBytes("20%"), 1.0);
|
BreakerSettings request = new BreakerSettings(CircuitBreaker.REQUEST, pctBytes("20%"), 1.0);
|
||||||
try {
|
try {
|
||||||
HierarchyCircuitBreakerService.validateSettings(new BreakerSettings[]{fd, request});
|
HierarchyCircuitBreakerService.validateSettings(new BreakerSettings[]{fd, request});
|
||||||
fail("settings are invalid but validate settings did not throw an exception");
|
fail("settings are invalid but validate settings did not throw an exception");
|
||||||
|
@ -63,4 +65,18 @@ public class CircuitBreakerUnitTests extends ElasticsearchTestCase {
|
||||||
e.getMessage().contains("must be non-negative"), equalTo(true));
|
e.getMessage().contains("must be non-negative"), equalTo(true));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRegisterCustomBreaker() throws Exception {
|
||||||
|
CircuitBreakerService service = new HierarchyCircuitBreakerService(ImmutableSettings.EMPTY, new NodeSettingsService(ImmutableSettings.EMPTY));
|
||||||
|
String customName = "custom";
|
||||||
|
BreakerSettings settings = new BreakerSettings(customName, 20, 1.0);
|
||||||
|
service.registerBreaker(settings);
|
||||||
|
|
||||||
|
CircuitBreaker breaker = service.getBreaker(customName);
|
||||||
|
assertThat(breaker, notNullValue());
|
||||||
|
assertThat(breaker, instanceOf(CircuitBreaker.class));
|
||||||
|
assertThat(breaker.getName(), is(customName));
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -60,7 +60,7 @@ public class RandomExceptionCircuitBreakerTests extends ElasticsearchIntegration
|
||||||
public void testBreakerWithRandomExceptions() throws IOException, InterruptedException, ExecutionException {
|
public void testBreakerWithRandomExceptions() throws IOException, InterruptedException, ExecutionException {
|
||||||
for (NodeStats node : client().admin().cluster().prepareNodesStats()
|
for (NodeStats node : client().admin().cluster().prepareNodesStats()
|
||||||
.clear().setBreaker(true).execute().actionGet().getNodes()) {
|
.clear().setBreaker(true).execute().actionGet().getNodes()) {
|
||||||
assertThat("Breaker is not set to 0", node.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA).getEstimated(), equalTo(0L));
|
assertThat("Breaker is not set to 0", node.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
|
||||||
}
|
}
|
||||||
|
|
||||||
String mapping = XContentFactory.jsonBuilder()
|
String mapping = XContentFactory.jsonBuilder()
|
||||||
|
@ -146,7 +146,7 @@ public class RandomExceptionCircuitBreakerTests extends ElasticsearchIntegration
|
||||||
NodesStatsResponse resp = client().admin().cluster().prepareNodesStats()
|
NodesStatsResponse resp = client().admin().cluster().prepareNodesStats()
|
||||||
.clear().setBreaker(true).execute().actionGet();
|
.clear().setBreaker(true).execute().actionGet();
|
||||||
for (NodeStats stats : resp.getNodes()) {
|
for (NodeStats stats : resp.getNodes()) {
|
||||||
assertThat("Breaker is set to 0", stats.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA).getEstimated(), equalTo(0L));
|
assertThat("Breaker is set to 0", stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < numSearches; i++) {
|
for (int i = 0; i < numSearches; i++) {
|
||||||
|
@ -190,7 +190,7 @@ public class RandomExceptionCircuitBreakerTests extends ElasticsearchIntegration
|
||||||
.clear().setBreaker(true).execute().actionGet();
|
.clear().setBreaker(true).execute().actionGet();
|
||||||
for (NodeStats stats : nodeStats.getNodes()) {
|
for (NodeStats stats : nodeStats.getNodes()) {
|
||||||
assertThat("Breaker reset to 0 last search success: " + success + " mapping: " + mapping,
|
assertThat("Breaker reset to 0 last search success: " + success + " mapping: " + mapping,
|
||||||
stats.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA).getEstimated(), equalTo(0L));
|
stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -242,7 +242,7 @@ public class CompositeTestCluster extends TestCluster {
|
||||||
.clear().setBreaker(true).execute().actionGet();
|
.clear().setBreaker(true).execute().actionGet();
|
||||||
for (NodeStats stats : nodeStats.getNodes()) {
|
for (NodeStats stats : nodeStats.getNodes()) {
|
||||||
assertThat("Fielddata breaker not reset to 0 on node: " + stats.getNode(),
|
assertThat("Fielddata breaker not reset to 0 on node: " + stats.getNode(),
|
||||||
stats.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA).getEstimated(), equalTo(0L));
|
stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
|
||||||
}
|
}
|
||||||
// CompositeTestCluster does not check the request breaker,
|
// CompositeTestCluster does not check the request breaker,
|
||||||
// because checking it requires a network request, which in
|
// because checking it requires a network request, which in
|
||||||
|
|
|
@ -147,7 +147,7 @@ public final class ExternalTestCluster extends TestCluster {
|
||||||
.clear().setBreaker(true).setIndices(true).execute().actionGet();
|
.clear().setBreaker(true).setIndices(true).execute().actionGet();
|
||||||
for (NodeStats stats : nodeStats.getNodes()) {
|
for (NodeStats stats : nodeStats.getNodes()) {
|
||||||
assertThat("Fielddata breaker not reset to 0 on node: " + stats.getNode(),
|
assertThat("Fielddata breaker not reset to 0 on node: " + stats.getNode(),
|
||||||
stats.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA).getEstimated(), equalTo(0L));
|
stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
|
||||||
// ExternalTestCluster does not check the request breaker,
|
// ExternalTestCluster does not check the request breaker,
|
||||||
// because checking it requires a network request, which in
|
// because checking it requires a network request, which in
|
||||||
// turn increments the breaker, making it non-0
|
// turn increments the breaker, making it non-0
|
||||||
|
|
|
@ -1707,7 +1707,7 @@ public final class InternalTestCluster extends TestCluster {
|
||||||
|
|
||||||
final String name = nodeAndClient.name;
|
final String name = nodeAndClient.name;
|
||||||
final CircuitBreakerService breakerService = getInstanceFromNode(CircuitBreakerService.class, nodeAndClient.node);
|
final CircuitBreakerService breakerService = getInstanceFromNode(CircuitBreakerService.class, nodeAndClient.node);
|
||||||
CircuitBreaker fdBreaker = breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA);
|
CircuitBreaker fdBreaker = breakerService.getBreaker(CircuitBreaker.FIELDDATA);
|
||||||
assertThat("Fielddata breaker not reset to 0 on node: " + name, fdBreaker.getUsed(), equalTo(0L));
|
assertThat("Fielddata breaker not reset to 0 on node: " + name, fdBreaker.getUsed(), equalTo(0L));
|
||||||
// Anything that uses transport or HTTP can increase the
|
// Anything that uses transport or HTTP can increase the
|
||||||
// request breaker (because they use bigarrays), because of
|
// request breaker (because they use bigarrays), because of
|
||||||
|
@ -1721,7 +1721,7 @@ public final class InternalTestCluster extends TestCluster {
|
||||||
assertBusy(new Runnable() {
|
assertBusy(new Runnable() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
CircuitBreaker reqBreaker = breakerService.getBreaker(CircuitBreaker.Name.REQUEST);
|
CircuitBreaker reqBreaker = breakerService.getBreaker(CircuitBreaker.REQUEST);
|
||||||
assertThat("Request breaker not reset to 0 on node: " + name, reqBreaker.getUsed(), equalTo(0L));
|
assertThat("Request breaker not reset to 0 on node: " + name, reqBreaker.getUsed(), equalTo(0L));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
Loading…
Reference in New Issue