Script Stats: Add compilation limit counter to stats (#26387)

In order to know, when the script compilation limit has kicked in,
this commit adds a counter in the script stats to expose that
information.

So far the only way to find out about this was to check the logs
or check out responses of individual requests.

core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java

@@ -205,6 +205,8 @@ public class RestNodesAction extends AbstractCatAction {
         table.addCell("script.compilations", "alias:scrcc,scriptCompilations;default:false;text-align:right;desc:script compilations");
         table.addCell("script.cache_evictions",
             "alias:scrce,scriptCacheEvictions;default:false;text-align:right;desc:script cache evictions");
+        table.addCell("script.compilation_limit_triggered", "alias:scrclt,scriptCacheCompilationLimitTriggered;default:false;" +
+                "text-align:right;desc:script cache compilation limit triggered");
 
         table.addCell("search.fetch_current", "alias:sfc,searchFetchCurrent;default:false;text-align:right;desc:current fetch phase ops");
         table.addCell("search.fetch_time", "alias:sfti,searchFetchTime;default:false;text-align:right;desc:time spent in fetch phase");
@@ -367,6 +369,7 @@ public class RestNodesAction extends AbstractCatAction {
             ScriptStats scriptStats = stats == null ? null : stats.getScriptStats();
             table.addCell(scriptStats == null ? null : scriptStats.getCompilations());
             table.addCell(scriptStats == null ? null : scriptStats.getCacheEvictions());
+            table.addCell(scriptStats == null ? null : scriptStats.getCompilationLimitTriggered());
 
             SearchStats searchStats = indicesStats == null ? null : indicesStats.getSearch();
             table.addCell(searchStats == null ? null : searchStats.getTotal().getFetchCurrent());

core/src/main/java/org/elasticsearch/script/ScriptMetrics.java

@@ -24,9 +24,10 @@ import org.elasticsearch.common.metrics.CounterMetric;
 public class ScriptMetrics {
     final CounterMetric compilationsMetric = new CounterMetric();
     final CounterMetric cacheEvictionsMetric = new CounterMetric();
+    final CounterMetric compilationLimitTriggered = new CounterMetric();
 
     public ScriptStats stats() {
-        return new ScriptStats(compilationsMetric.count(), cacheEvictionsMetric.count());
+        return new ScriptStats(compilationsMetric.count(), cacheEvictionsMetric.count(), compilationLimitTriggered.count());
     }
 
     public void onCompilation() {
@@ -36,4 +37,8 @@ public class ScriptMetrics {
     public void onCacheEviction() {
         cacheEvictionsMetric.inc();
     }
+
+    public void onCompilationLimit() {
+        compilationLimitTriggered.inc();
+    }
 }

core/src/main/java/org/elasticsearch/script/ScriptService.java

@@ -336,6 +336,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
         if (scriptsPerMinCounter >= 1) {
             scriptsPerMinCounter -= 1.0;
         } else {
+            scriptMetrics.onCompilationLimit();
             // Otherwise reject the request
             throw new CircuitBreakingException("[script] Too many dynamic script compilations within one minute, max: [" +
                             totalCompilesPerMinute + "/min]; please use on-disk, indexed, or scripts with parameters instead; " +

core/src/main/java/org/elasticsearch/script/ScriptStats.java

@@ -19,10 +19,10 @@
 
 package org.elasticsearch.script;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -31,21 +31,27 @@ import java.io.IOException;
 public class ScriptStats implements Writeable, ToXContentFragment {
     private final long compilations;
     private final long cacheEvictions;
+    private final long compilationLimitTriggered;
 
-    public ScriptStats(long compilations, long cacheEvictions) {
+    public ScriptStats(long compilations, long cacheEvictions, long compilationLimitTriggered) {
         this.compilations = compilations;
         this.cacheEvictions = cacheEvictions;
+        this.compilationLimitTriggered = compilationLimitTriggered;
     }
 
     public ScriptStats(StreamInput in) throws IOException {
         compilations = in.readVLong();
         cacheEvictions = in.readVLong();
+        compilationLimitTriggered = in.getVersion().onOrAfter(Version.V_7_0_0_alpha1) ? in.readVLong() : 0;
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(compilations);
         out.writeVLong(cacheEvictions);
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeVLong(compilationLimitTriggered);
+        }
     }
 
     public long getCompilations() {
@@ -56,11 +62,16 @@ public class ScriptStats implements Writeable, ToXContentFragment {
         return cacheEvictions;
     }
 
+    public long getCompilationLimitTriggered() {
+        return compilationLimitTriggered;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.SCRIPT_STATS);
         builder.field(Fields.COMPILATIONS, getCompilations());
         builder.field(Fields.CACHE_EVICTIONS, getCacheEvictions());
+        builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, getCompilationLimitTriggered());
         builder.endObject();
         return builder;
     }
@@ -69,5 +80,6 @@ public class ScriptStats implements Writeable, ToXContentFragment {
         static final String SCRIPT_STATS = "script";
         static final String COMPILATIONS = "compilations";
         static final String CACHE_EVICTIONS = "cache_evictions";
+        static final String COMPILATION_LIMIT_TRIGGERED = "compilation_limit_triggered";
     }
 }

core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java

@@ -381,7 +381,8 @@ public class NodeStatsTests extends ESTestCase {
             }
             allCircuitBreakerStats = new AllCircuitBreakerStats(circuitBreakerStatsArray);
         }
-        ScriptStats scriptStats = frequently() ? new ScriptStats(randomNonNegativeLong(), randomNonNegativeLong()) : null;
+        ScriptStats scriptStats = frequently() ?
+                new ScriptStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) : null;
         DiscoveryStats discoveryStats = frequently() ? new DiscoveryStats(randomBoolean() ? new PendingClusterStateStats(randomInt(),
                 randomInt(), randomInt()) : null) : null;
         IngestStats ingestStats = null;