Merge remote-tracking branch 'dakrone/allocation-explain-show-delay'

core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.shard.ShardId;
@@ -47,6 +48,7 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
     private final Map<DiscoveryNode, Decision> nodeToDecision;
     private final Map<DiscoveryNode, Float> nodeWeights;
     private final UnassignedInfo unassignedInfo;
+    private final long remainingDelayNanos;
 
     public ClusterAllocationExplanation(StreamInput in) throws IOException {
         this.shard = ShardId.readShardId(in);
@@ -73,17 +75,19 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
             ntw.put(dn, weight);
         }
         this.nodeWeights = ntw;
+        remainingDelayNanos = in.readVLong();
     }
 
     public ClusterAllocationExplanation(ShardId shard, boolean primary, @Nullable String assignedNodeId,
                                         UnassignedInfo unassignedInfo, Map<DiscoveryNode, Decision> nodeToDecision,
-                                        Map<DiscoveryNode, Float> nodeWeights) {
+                                        Map<DiscoveryNode, Float> nodeWeights, long remainingDelayNanos) {
         this.shard = shard;
         this.primary = primary;
         this.assignedNodeId = assignedNodeId;
         this.unassignedInfo = unassignedInfo;
         this.nodeToDecision = nodeToDecision == null ? Collections.emptyMap() : nodeToDecision;
         this.nodeWeights = nodeWeights == null ? Collections.emptyMap() : nodeWeights;
+        this.remainingDelayNanos = remainingDelayNanos;
     }
 
     public ShardId getShard() {
@@ -124,6 +128,11 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
         return this.nodeWeights;
     }
 
+    /** Return the remaining allocation delay for this shard in nanoseconds */
+    public long getRemainingDelayNanos() {
+        return this.remainingDelayNanos;
+    }
+
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(); {
             builder.startObject("shard"); {
@@ -141,6 +150,11 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
             // If we have unassigned info, show that
             if (unassignedInfo != null) {
                 unassignedInfo.toXContent(builder, params);
+                long delay = unassignedInfo.getLastComputedLeftDelayNanos();
+                builder.field("allocation_delay", TimeValue.timeValueNanos(delay));
+                builder.field("allocation_delay_ms", TimeValue.timeValueNanos(delay).millis());
+                builder.field("remaining_delay", TimeValue.timeValueNanos(remainingDelayNanos));
+                builder.field("remaining_delay_ms", TimeValue.timeValueNanos(remainingDelayNanos).millis());
             }
             builder.startObject("nodes");
             for (Map.Entry<DiscoveryNode, Float> entry : nodeWeights.entrySet()) {
@@ -194,5 +208,6 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
             entry.getKey().writeTo(out);
             out.writeFloat(entry.getValue());
         }
+        out.writeVLong(remainingDelayNanos);
     }
 }

core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java

@@ -139,8 +139,14 @@ public class TransportClusterAllocationExplainAction
                 nodeToDecision.put(discoNode, d);
             }
         }
+        long remainingDelayNanos = 0;
+        if (ui != null) {
+            final MetaData metadata = allocation.metaData();
+            final Settings indexSettings = metadata.index(shard.index()).getSettings();
+            remainingDelayNanos = ui.getRemainingDelay(System.nanoTime(), metadata.settings(), indexSettings);
+        }
         return new ClusterAllocationExplanation(shard.shardId(), shard.primary(), shard.currentNodeId(), ui, nodeToDecision,
-                shardAllocator.weighShard(allocation, shard));
+                shardAllocator.weighShard(allocation, shard), remainingDelayNanos);
     }
 
     @Override

core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java

@@ -231,20 +231,28 @@ public class UnassignedInfo implements ToXContent, Writeable<UnassignedInfo> {
         return lastComputedLeftDelayNanos;
     }
 
+    /**
+     * Calculates the delay left based on current time (in nanoseconds) and index/node settings.
+     *
+     * @return calculated delay in nanoseconds
+     */
+    public long getRemainingDelay(final long nanoTimeNow, final Settings settings, final Settings indexSettings) {
+        final long delayTimeoutNanos = getAllocationDelayTimeoutSettingNanos(settings, indexSettings);
+        if (delayTimeoutNanos == 0L) {
+            return 0L;
+        } else {
+            assert nanoTimeNow >= unassignedTimeNanos;
+            return Math.max(0L, delayTimeoutNanos - (nanoTimeNow - unassignedTimeNanos));
+        }
+    }
+
     /**
      * Updates delay left based on current time (in nanoseconds) and index/node settings.
      *
      * @return updated delay in nanoseconds
      */
-    public long updateDelay(long nanoTimeNow, Settings settings, Settings indexSettings) {
+    public long updateDelay(final long nanoTimeNow, final Settings settings, final Settings indexSettings) {
-        long delayTimeoutNanos = getAllocationDelayTimeoutSettingNanos(settings, indexSettings);
+        final long newComputedLeftDelayNanos = getRemainingDelay(nanoTimeNow, settings, indexSettings);
-        final long newComputedLeftDelayNanos;
-        if (delayTimeoutNanos == 0L) {
-            newComputedLeftDelayNanos = 0L;
-        } else {
-            assert nanoTimeNow >= unassignedTimeNanos;
-            newComputedLeftDelayNanos = Math.max(0L, delayTimeoutNanos - (nanoTimeNow - unassignedTimeNanos));
-        }
         lastComputedLeftDelayNanos = newComputedLeftDelayNanos;
         return newComputedLeftDelayNanos;
     }

core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.allocation;
+
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+/**
+ * Tests for the cluster allocation explanation
+ */
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public final class ClusterAllocationExplainIT extends ESIntegTestCase {
+    public void testDelayShards() throws Exception {
+        logger.info("--> starting 3 nodes");
+        List<String> nodes = internalCluster().startNodesAsync(3).get();
+
+        // Wait for all 3 nodes to be up
+        logger.info("--> waiting for 3 nodes to be up");
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get();
+                assertThat(resp.getNodes().length, equalTo(3));
+            }
+        });
+
+        logger.info("--> creating 'test' index");
+        prepareCreate("test").setSettings(Settings.settingsBuilder()
+                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "1m")
+                .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 5)
+                .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)).get();
+        ensureGreen("test");
+
+        logger.info("--> stopping a random node");
+        assertTrue(internalCluster().stopRandomDataNode());
+
+        ensureYellow("test");
+
+        ClusterAllocationExplainResponse resp = client().admin().cluster().prepareAllocationExplain().useAnyUnassignedShard().get();
+        ClusterAllocationExplanation cae = resp.getExplanation();
+        assertThat(cae.getShard().getIndexName(), equalTo("test"));
+        assertFalse(cae.isPrimary());
+        assertFalse(cae.isAssigned());
+        assertThat("expecting a remaining delay, got: " + cae.getRemainingDelayNanos(), cae.getRemainingDelayNanos(), greaterThan(0L));
+    }
+}

core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java

@@ -65,8 +65,9 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
             nodeToWeight.put(dn, randomFloat());
         }
 
+        long remainingDelay = randomIntBetween(0, 500);
         ClusterAllocationExplanation cae = new ClusterAllocationExplanation(shard, true, "assignedNode", null,
-                nodeToDecisions, nodeToWeight);
+                nodeToDecisions, nodeToWeight, remainingDelay);
         BytesStreamOutput out = new BytesStreamOutput();
         cae.writeTo(out);
         StreamInput in = StreamInput.wrap(out.bytes());
@@ -80,5 +81,6 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
             assertEquals(nodeToDecisions.get(entry.getKey()), entry.getValue());
         }
         assertEquals(nodeToWeight, cae2.getNodeWeights());
+        assertEquals(remainingDelay, cae2.getRemainingDelayNanos());
     }
 }