From ae8f625d73343c22ab2f96e5afb5dbd3a664de73 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 11 Jul 2019 19:11:42 +0200 Subject: [PATCH] Report usages old child breakers when breaking on real memory (#44221) This will help in investigations where the real memory circuit breaker is tripped to better understand on what the actual memory is used, i.e. whether it's a temporary thing (e.g. requests) in contrast to more permanently allocated memory (e.g. accounting). --- .../HierarchyCircuitBreakerService.java | 19 +++++++++---------- .../HierarchyCircuitBreakerServiceTests.java | 4 ++++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java index 48045e21925..5797843161c 100644 --- a/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java +++ b/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java @@ -325,17 +325,16 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { message.append("/"); message.append(new ByteSizeValue(newBytesReserved)); message.append("]"); - } else { - message.append(", usages ["); - message.append(String.join(", ", - this.breakers.entrySet().stream().map(e -> { - final CircuitBreaker breaker = e.getValue(); - final long breakerUsed = (long)(breaker.getUsed() * breaker.getOverhead()); - return e.getKey() + "=" + breakerUsed + "/" + new ByteSizeValue(breakerUsed); - }) - .collect(Collectors.toList()))); - message.append("]"); } + message.append(", usages ["); + message.append(String.join(", ", + this.breakers.entrySet().stream().map(e -> { + final CircuitBreaker breaker = e.getValue(); + final long breakerUsed = (long)(breaker.getUsed() * breaker.getOverhead()); + return e.getKey() + "=" + breakerUsed + "/" + new ByteSizeValue(breakerUsed); + }) + .collect(Collectors.toList()))); + message.append("]"); // derive durability of a tripped parent breaker depending on whether the majority of memory tracked by // child circuit breakers is categorized as transient or permanent. CircuitBreaker.Durability durability = memoryUsed.transientChildUsage >= memoryUsed.permanentChildUsage ? diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java index a4411930f92..9918b0c6e61 100644 --- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java @@ -246,6 +246,10 @@ public class HierarchyCircuitBreakerServiceTests extends ESTestCase { assertThat(exception.getMessage(), containsString("real usage: [181/181b], new bytes reserved: [" + (reservationInBytes * 2) + "/" + new ByteSizeValue(reservationInBytes * 2) + "]")); + final long requestCircuitBreakerUsed = (requestBreaker.getUsed() + reservationInBytes) * 2; + assertThat(exception.getMessage(), + containsString("usages [request=" + requestCircuitBreakerUsed + "/" + new ByteSizeValue(requestCircuitBreakerUsed) + + ", fielddata=0/0b, in_flight_requests=0/0b, accounting=0/0b]")); assertThat(exception.getDurability(), equalTo(CircuitBreaker.Durability.TRANSIENT)); assertEquals(0, requestBreaker.getTrippedCount()); assertEquals(1, service.stats().getStats(CircuitBreaker.PARENT).getTrippedCount());