[Monitoring] Add Shard-level Details to Index Stats Collection (elastic/x-pack-elasticsearch#2817)

This adds details about the shards and the health of the index. By adding these stats directly to the document, the UI can avoid many aggregations and enable better searching and sorting against indices.

Original commit: elastic/x-pack-elasticsearch@f38ae5ce69

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollector.java

@@ -9,6 +9,9 @@ import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -71,13 +74,18 @@ public class IndexStatsCollector extends Collector {
 
         final long timestamp = timestamp();
         final String clusterUuid = clusterUUID();
+        final ClusterState clusterState = clusterService.state();
 
         // add the indices stats that we use to collect the index stats
         results.add(new IndicesStatsMonitoringDoc(clusterUuid, timestamp, interval, node, indicesStats));
 
         // collect each index stats document
-        for (IndexStats indexStats : indicesStats.getIndices().values()) {
+        for (final IndexStats indexStats : indicesStats.getIndices().values()) {
-            results.add(new IndexStatsMonitoringDoc(clusterUuid, timestamp, interval, node, indexStats));
+            final String index = indexStats.getIndex();
+            final IndexMetaData metaData = clusterState.metaData().index(index);
+            final IndexRoutingTable routingTable = clusterState.routingTable().index(index);
+
+            results.add(new IndexStatsMonitoringDoc(clusterUuid, timestamp, interval, node, indexStats, metaData, routingTable));
         }
 
         return Collections.unmodifiableCollection(results);

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDoc.java

@@ -7,6 +7,10 @@ package org.elasticsearch.xpack.monitoring.collector.indices;
 
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.cluster.health.ClusterIndexHealth;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
@@ -14,6 +18,7 @@ import org.elasticsearch.xpack.monitoring.exporter.FilteredMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
 import java.io.IOException;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.Set;
 
@@ -25,26 +30,77 @@ public class IndexStatsMonitoringDoc extends FilteredMonitoringDoc {
     public static final String TYPE = "index_stats";
 
     private final IndexStats indexStats;
+    private final IndexMetaData metaData;
+    private final IndexRoutingTable routingTable;
 
     IndexStatsMonitoringDoc(final String cluster,
                             final long timestamp,
                             final long intervalMillis,
                             final MonitoringDoc.Node node,
-                            final IndexStats indexStats) {
+                            @Nullable final IndexStats indexStats,
+                            final IndexMetaData metaData,
+                            final IndexRoutingTable routingTable) {
         super(cluster, timestamp, intervalMillis, node, MonitoredSystem.ES, TYPE, null, XCONTENT_FILTERS);
-        this.indexStats = Objects.requireNonNull(indexStats);
+        this.indexStats = indexStats;
+        this.metaData = Objects.requireNonNull(metaData);
+        this.routingTable = Objects.requireNonNull(routingTable);
     }
 
     IndexStats getIndexStats() {
         return indexStats;
     }
 
+    IndexMetaData getIndexMetaData() {
+        return metaData;
+    }
+
+    IndexRoutingTable getIndexRoutingTable() {
+        return routingTable;
+    }
+
     @Override
     protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        final ClusterIndexHealth health = new ClusterIndexHealth(metaData, routingTable);
+
         builder.startObject(TYPE);
         {
-            builder.field("index", indexStats.getIndex());
+            builder.field("index", metaData.getIndex().getName());
+            builder.field("uuid", metaData.getIndexUUID());
+            builder.field("created", metaData.getCreationDate());
+            builder.field("status", health.getStatus().name().toLowerCase(Locale.ROOT));
 
+            builder.startObject("version");
+            {
+                builder.field("created", metaData.getCreationVersion());
+                builder.field("upgraded", metaData.getUpgradedVersion());
+            }
+            builder.endObject();
+
+            builder.startObject("shards");
+            {
+                final int total = metaData.getTotalNumberOfShards();
+                final int primaries = metaData.getNumberOfShards();
+                final int activeTotal = health.getActiveShards();
+                final int activePrimaries = health.getActivePrimaryShards();
+                final int unassignedTotal = health.getUnassignedShards() + health.getInitializingShards();
+                final int unassignedPrimaries = primaries - health.getActivePrimaryShards();
+
+                builder.field("total", total);
+                builder.field("primaries", primaries);
+                builder.field("replicas", metaData.getNumberOfReplicas());
+                builder.field("active_total", activeTotal);
+                builder.field("active_primaries", activePrimaries);
+                builder.field("active_replicas", activeTotal - activePrimaries);
+                builder.field("unassigned_total", unassignedTotal);
+                builder.field("unassigned_primaries", unassignedPrimaries);
+                builder.field("unassigned_replicas", unassignedTotal - unassignedPrimaries);
+                builder.field("initializing", health.getInitializingShards());
+                builder.field("relocating", health.getRelocatingShards());
+            }
+            builder.endObject();
+
+            // when an index is completely red, then we don't get stats for it
+            if (indexStats != null) {
                 final CommonStats totalStats = indexStats.getTotal();
                 if (totalStats != null) {
                     builder.startObject("total");
@@ -63,11 +119,28 @@ public class IndexStatsMonitoringDoc extends FilteredMonitoringDoc {
                     builder.endObject();
                 }
             }
+        }
         builder.endObject();
     }
 
     public static final Set<String> XCONTENT_FILTERS =
         Sets.newHashSet("index_stats.index",
+                        "index_stats.uuid",
+                        "index_stats.created",
+                        "index_stats.status",
+                        "index_stats.version.created",
+                        "index_stats.version.upgraded",
+                        "index_stats.shards.total",
+                        "index_stats.shards.primaries",
+                        "index_stats.shards.replicas",
+                        "index_stats.shards.active_total",
+                        "index_stats.shards.active_primaries",
+                        "index_stats.shards.active_replicas",
+                        "index_stats.shards.unassigned_total",
+                        "index_stats.shards.unassigned_primaries",
+                        "index_stats.shards.unassigned_replicas",
+                        "index_stats.shards.initializing",
+                        "index_stats.shards.relocating",
                         "index_stats.primaries.docs.count",
                         "index_stats.primaries.fielddata.memory_size_in_bytes",
                         "index_stats.primaries.fielddata.evictions",

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollectorTests.java

@@ -13,6 +13,9 @@ import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
@@ -84,15 +87,29 @@ public class IndexStatsCollectorTests extends BaseCollectorTestCase {
         final String clusterUUID = UUID.randomUUID().toString();
         whenClusterStateWithUUID(clusterUUID);
 
+        final RoutingTable routingTable = mock(RoutingTable.class);
+        when(clusterState.routingTable()).thenReturn(routingTable);
+
         final MonitoringDoc.Node node = randomMonitoringNode(random());
 
-        final Map<String, IndexStats> indicesStats = new HashMap<>();
         final int indices = randomIntBetween(0, 10);
+        final Map<String, IndexStats> indicesStats = new HashMap<>(indices);
+        final Map<String, IndexMetaData> indicesMetaData = new HashMap<>(indices);
+        final Map<String, IndexRoutingTable> indicesRoutingTable = new HashMap<>(indices);
+
         for (int i = 0; i < indices; i++) {
-            String index = "_index_" + i;
+            final String index = "_index_" + i;
-            IndexStats indexStats = mock(IndexStats.class);
+            final IndexStats indexStats = mock(IndexStats.class);
-            when(indexStats.getIndex()).thenReturn(index);
+            final IndexMetaData indexMetaData = mock(IndexMetaData.class);
+            final IndexRoutingTable indexRoutingTable = mock(IndexRoutingTable.class);
+
             indicesStats.put(index, indexStats);
+            indicesMetaData.put(index, indexMetaData);
+            indicesRoutingTable.put(index, indexRoutingTable);
+
+            when(indexStats.getIndex()).thenReturn(index);
+            when(metaData.index(index)).thenReturn(indexMetaData);
+            when(routingTable.index(index)).thenReturn(indexRoutingTable);
         }
 
         final IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
@@ -121,7 +138,7 @@ public class IndexStatsCollectorTests extends BaseCollectorTestCase {
 
         assertEquals(1 + indices, results.size());
 
-        for (MonitoringDoc document : results) {
+        for (final MonitoringDoc document : results) {
             assertThat(document.getCluster(), equalTo(clusterUUID));
             assertThat(document.getTimestamp(), greaterThan(0L));
             assertThat(document.getIntervalMillis(), equalTo(interval));
@@ -135,8 +152,12 @@ public class IndexStatsCollectorTests extends BaseCollectorTestCase {
             } else {
                 assertThat(document.getType(), equalTo(IndexStatsMonitoringDoc.TYPE));
 
-                IndexStats indexStats = ((IndexStatsMonitoringDoc) document).getIndexStats();
+                final IndexStatsMonitoringDoc indexStatsDocument = (IndexStatsMonitoringDoc)document;
-                assertThat(indexStats, is(indicesStats.get(indexStats.getIndex())));
+                final String index = indexStatsDocument.getIndexStats().getIndex();
+
+                assertThat(indexStatsDocument.getIndexStats(), is(indicesStats.get(index)));
+                assertThat(indexStatsDocument.getIndexMetaData(), is(indicesMetaData.get(index)));
+                assertThat(indexStatsDocument.getIndexRoutingTable(), is(indicesRoutingTable.get(index)));
             }
         }
     }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java

@@ -5,12 +5,22 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.indices;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.cluster.health.ClusterIndexHealth;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
 import org.elasticsearch.index.cache.request.RequestCacheStats;
 import org.elasticsearch.index.engine.SegmentsStats;
@@ -19,15 +29,20 @@ import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.exporter.BaseFilteredMonitoringDocTestCase;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Date;
+import java.util.Locale;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
@@ -35,20 +50,35 @@ import static org.mockito.Mockito.when;
 
 public class IndexStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestCase<IndexStatsMonitoringDoc> {
 
+    private final Index index = new Index("logstash-2017.10.27", "aBcDeFg");
+    private final int primaries = randomIntBetween(1, 5);
+    private final int replicas = randomIntBetween(0, 2);
+    private final int total = primaries + (primaries * replicas);
+    private final int activePrimaries = randomInt(primaries);
+    private final int activeReplicas = randomInt(activePrimaries * replicas);
+    private final int initializing = randomInt(primaries - activePrimaries + Math.max(0, activePrimaries * replicas - activeReplicas));
+    // to simplify the test code, we only allow active primaries to relocate, rather than also include active replicas
+    private final int relocating = randomInt(activePrimaries);
 
     private IndexStats indexStats;
+    private IndexMetaData metaData;
+    private IndexRoutingTable routingTable;
+    private ClusterIndexHealth indexHealth;
 
     @Override
     @Before
     public void setUp() throws Exception {
         super.setUp();
         indexStats = mock(IndexStats.class);
+        metaData = mockIndexMetaData(index, primaries, replicas);
+        routingTable = mockIndexRoutingTable(index, primaries, replicas, activePrimaries, activeReplicas, initializing, relocating);
+        indexHealth = new ClusterIndexHealth(metaData, routingTable);
     }
 
     @Override
     protected IndexStatsMonitoringDoc createMonitoringDoc(String cluster, long timestamp, long interval, MonitoringDoc.Node node,
                                                           MonitoredSystem system, String type, String id) {
-        return new IndexStatsMonitoringDoc(cluster, timestamp, interval, node, indexStats);
+        return new IndexStatsMonitoringDoc(cluster, timestamp, interval, node, indexStats, metaData, routingTable);
     }
 
     @Override
@@ -58,6 +88,8 @@ public class IndexStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestC
         assertThat(document.getId(), nullValue());
 
         assertThat(document.getIndexStats(), is(indexStats));
+        assertThat(document.getIndexMetaData(), is(metaData));
+        assertThat(document.getIndexRoutingTable(), is(routingTable));
     }
 
     @Override
@@ -65,18 +97,32 @@ public class IndexStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestC
         return IndexStatsMonitoringDoc.XCONTENT_FILTERS;
     }
 
-    public void testConstructorIndexStatsMustNotBeNull() {
+    public void testConstructorIndexStatsCanBeNull() {
-        expectThrows(NullPointerException.class, () -> new IndexStatsMonitoringDoc(cluster, timestamp, interval, node, null));
+        new IndexStatsMonitoringDoc(cluster, timestamp, interval, node, null, metaData, routingTable);
+    }
+
+    public void testConstructorMetaDataMustNotBeNull() {
+        final IndexStats indexStats = randomFrom(this.indexStats, null);
+
+        expectThrows(NullPointerException.class,
+                     () -> new IndexStatsMonitoringDoc(cluster, timestamp, interval, node, indexStats, null, routingTable));
+    }
+
+    public void testConstructorRoutingTableMustNotBeNull() {
+        final IndexStats indexStats = randomFrom(this.indexStats, null);
+
+        expectThrows(NullPointerException.class,
+                     () -> new IndexStatsMonitoringDoc(cluster, timestamp, interval, node, indexStats, metaData, null));
     }
 
     @Override
     public void testToXContent() throws IOException {
         final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
-        when(indexStats.getIndex()).thenReturn("_index");
         when(indexStats.getTotal()).thenReturn(mockCommonStats());
         when(indexStats.getPrimaries()).thenReturn(mockCommonStats());
 
-        final IndexStatsMonitoringDoc document = new IndexStatsMonitoringDoc("_cluster", 1502266739402L, 1506593717631L, node, indexStats);
+        final IndexStatsMonitoringDoc document =
+                new IndexStatsMonitoringDoc("_cluster", 1502266739402L, 1506593717631L, node, indexStats, metaData, routingTable);
 
         final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
         assertEquals("{"
@@ -93,7 +139,7 @@ public class IndexStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestC
                        + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
                      + "},"
                      + "\"index_stats\":{"
-                       + "\"index\":\"_index\","
+                       + indexStatsSummary() + ","
                        + "\"total\":{"
                          + "\"docs\":{"
                            + "\"count\":1"
@@ -204,11 +250,19 @@ public class IndexStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestC
 
     public void testToXContentWithNullStats() throws IOException {
         final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
-        when(indexStats.getIndex()).thenReturn("_index");
+        final IndexStats indexStats;
+
+        if (randomBoolean()) {
+            indexStats = this.indexStats;
+
             when(indexStats.getTotal()).thenReturn(null);
             when(indexStats.getPrimaries()).thenReturn(null);
+        } else {
+            indexStats = null;
+        }
 
-        final IndexStatsMonitoringDoc document = new IndexStatsMonitoringDoc("_cluster", 1502266739402L, 1506593717631L, node, indexStats);
+        final IndexStatsMonitoringDoc document =
+                new IndexStatsMonitoringDoc("_cluster", 1502266739402L, 1506593717631L, node, indexStats, metaData, routingTable);
 
         final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
         assertEquals("{"
@@ -225,11 +279,36 @@ public class IndexStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestC
                        + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
                      + "},"
                      + "\"index_stats\":{"
-                       + "\"index\":\"_index\""
+                       + indexStatsSummary()
                       + "}"
                 + "}", xContent.utf8ToString());
     }
 
+    private String indexStatsSummary() {
+        // must append , if total / primaries stats are included
+        return "\"index\":\"" + index.getName() + "\"," +
+               "\"uuid\":\"" + index.getUUID() + "\"," +
+               "\"created\":" + metaData.getCreationDate() + "," +
+               "\"status\":\"" + indexHealth.getStatus().name().toLowerCase(Locale.ROOT) + "\"," +
+               "\"version\":{" +
+                 "\"created\":\"" + metaData.getCreationVersion() + "\"," +
+                 "\"upgraded\":\"" + metaData.getUpgradedVersion() + "\"" +
+               "}," +
+               "\"shards\":{" +
+                 "\"total\":" + total + "," +
+                 "\"primaries\":" + primaries + "," +
+                 "\"replicas\":" + replicas + "," +
+                 "\"active_total\":" + (activePrimaries + activeReplicas) + "," +
+                 "\"active_primaries\":" + activePrimaries + "," +
+                 "\"active_replicas\":" + activeReplicas + "," +
+                 "\"unassigned_total\":" + (total - (activePrimaries + activeReplicas)) + "," +
+                 "\"unassigned_primaries\":" + (primaries - activePrimaries) + "," +
+                 "\"unassigned_replicas\":" + (total - (activePrimaries + activeReplicas) - (primaries - activePrimaries)) + "," +
+                 "\"initializing\":" + initializing + "," +
+                 "\"relocating\":" + relocating +
+               "}";
+    }
+
     private static CommonStats mockCommonStats() {
         // This value is used in constructors of various stats objects,
         // when the value is not printed out in the final XContent.
@@ -270,4 +349,138 @@ public class IndexStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestC
 
         return commonStats;
     }
+
+    private static IndexMetaData mockIndexMetaData(final Index index,
+                                                   final int primaries, final int replicas) {
+        final Settings.Builder settings = Settings.builder();
+
+        settings.put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        settings.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, primaries);
+        settings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas);
+        settings.put(IndexMetaData.SETTING_VERSION_CREATED, MonitoringTemplateUtils.LAST_UPDATED_VERSION);
+        settings.put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.CURRENT.id);
+        settings.put(IndexMetaData.SETTING_CREATION_DATE, (new Date()).getTime());
+
+        return IndexMetaData.builder(index.getName()).settings(settings).build();
+    }
+
+    private static IndexRoutingTable mockIndexRoutingTable(final Index index,
+                                                           final int primaries, final int replicas,
+                                                           final int activePrimaries, final int activeReplicas,
+                                                           final int initializing, final int relocating) {
+        final int total = primaries + (primaries * replicas);
+        int unassignedTotal = total - (activePrimaries + activeReplicas);
+        int unassignedPrimaries = primaries - activePrimaries;
+        int unassignedReplicas = unassignedTotal - unassignedPrimaries;
+        int activePrimariesRemaining = activePrimaries;
+        int activeReplicasRemaining = activeReplicas;
+        int initializingTotal = initializing; // we count initializing as a special type of unassigned!
+        int relocatingTotal = relocating;
+
+        assertThat("more initializing shards than unassigned", unassignedTotal, greaterThanOrEqualTo(initializingTotal));
+        // we only relocate primaries to simplify this method -- replicas can be relocating
+        assertThat("more relocating shards than active primaries", activePrimaries, greaterThanOrEqualTo(relocatingTotal));
+
+        final IndexRoutingTable.Builder builder = IndexRoutingTable.builder(index);
+
+        for (int i = 0; i < primaries; ++i) {
+            final ShardId shardId = new ShardId(index, i);
+            final IndexShardRoutingTable.Builder shard = new IndexShardRoutingTable.Builder(shardId);
+            final int primariesLeft = primaries - i - 1;
+
+            // randomly mark unassigned shards
+            if (activePrimariesRemaining == 0 || (activePrimariesRemaining < primariesLeft && randomBoolean())) {
+                --unassignedTotal;
+                --unassignedPrimaries;
+
+                final UnassignedInfo unassignedInfo =
+                        new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), randomAlphaOfLength(3));
+                final String nodeId;
+                final ShardRoutingState state;
+
+                if (initializingTotal > 0) {
+                    --initializingTotal;
+
+                    nodeId = "abc";
+                    state = ShardRoutingState.INITIALIZING;
+                } else {
+                    nodeId = null;
+                    state = ShardRoutingState.UNASSIGNED;
+                }
+
+                shard.addShard(TestShardRouting.newShardRouting(shardId, nodeId, null, true, state, unassignedInfo));
+
+                // mark all as unassigned
+                for (int j = 0; j < replicas; ++j) {
+                    --unassignedTotal;
+                    --unassignedReplicas;
+
+                    shard.addShard(TestShardRouting.newShardRouting(shardId, null, false, ShardRoutingState.UNASSIGNED));
+                }
+            // primary should be allocated, but replicas can still be unassigned
+            } else {
+                --activePrimariesRemaining;
+
+                final String relocatingNodeId;
+                final ShardRoutingState state;
+
+                if (relocatingTotal > activePrimariesRemaining || (relocatingTotal > 0 && randomBoolean())) {
+                    --relocatingTotal;
+
+                    relocatingNodeId = "def";
+                    state = ShardRoutingState.RELOCATING;
+                } else {
+                    relocatingNodeId = null;
+                    state = ShardRoutingState.STARTED;
+                }
+
+                // Primary shard is STARTED (active)
+                shard.addShard(TestShardRouting.newShardRouting(shardId, "abc", relocatingNodeId, true, state));
+
+                for (int j = 0; j < replicas; ++j) {
+                    final int replicasForActivePrimariesLeft = replicas - j - 1 + activePrimariesRemaining * replicas;
+
+                    if (activeReplicasRemaining == 0 || (activeReplicasRemaining < replicasForActivePrimariesLeft && randomBoolean())) {
+                        --unassignedTotal;
+                        --unassignedReplicas;
+
+                        final String replicaNodeId;
+                        final ShardRoutingState replicaState;
+
+                        // first case means that we MUST assign it because it's this unassigned shard
+                        if (initializingTotal > 0) {
+                            --initializingTotal;
+
+                            replicaNodeId = "abc" + j;
+                            replicaState = ShardRoutingState.INITIALIZING;
+                        } else {
+                            replicaNodeId = null;
+                            replicaState = ShardRoutingState.UNASSIGNED;
+                        }
+
+                        shard.addShard(TestShardRouting.newShardRouting(shardId, replicaNodeId, false, replicaState));
+                    } else {
+                        --activeReplicasRemaining;
+
+                        // Replica shard is STARTED (active)
+                        shard.addShard(TestShardRouting.newShardRouting(shardId, "abc" + j, false, ShardRoutingState.STARTED));
+                    }
+                }
+            }
+
+            builder.addIndexShard(shard.build());
+        }
+
+        // sanity checks
+        assertThat("unassigned shards miscounted", unassignedTotal, is(0));
+        assertThat("unassigned primary shards miscounted", unassignedPrimaries, is(0));
+        assertThat("unassigned replica shards miscounted", unassignedReplicas, is(0));
+        assertThat("initializing shards miscounted", initializingTotal, is(0));
+        assertThat("relocating shards miscounted", relocatingTotal, is(0));
+        assertThat("active primaries miscounted", activePrimariesRemaining, is(0));
+        assertThat("active replicas miscounted", activeReplicasRemaining, is(0));
+
+        return builder.build();
+    }
+
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java

@@ -397,11 +397,21 @@ public class MonitoringIT extends ESRestTestCase {
         final Map<String, Object> source = (Map<String, Object>) document.get("_source");
         assertEquals(6, source.size());
 
+        // particular field values checked in the index stats tests
         final Map<String, Object> indexStats = (Map<String, Object>) source.get(IndexStatsMonitoringDoc.TYPE);
-        assertEquals(3, indexStats.size());
+        assertEquals(8, indexStats.size());
+        assertThat((String) indexStats.get("index"), not(isEmptyOrNullString()));
+        assertThat((String) indexStats.get("uuid"), not(isEmptyOrNullString()));
+        assertThat((Long) indexStats.get("created"), notNullValue());
+        assertThat((String) indexStats.get("status"), not(isEmptyOrNullString()));
+        assertThat(indexStats.get("version"), notNullValue());
+        final Map<String, Object> version = (Map<String, Object>) indexStats.get("version");
+        assertEquals(2, version.size());
+        assertThat(indexStats.get("shards"), notNullValue());
+        final Map<String, Object> shards = (Map<String, Object>) indexStats.get("shards");
+        assertEquals(11, shards.size());
         assertThat(indexStats.get("primaries"), notNullValue());
         assertThat(indexStats.get("total"), notNullValue());
-        assertThat((String) indexStats.get("index"), not(isEmptyOrNullString()));
 
         IndexStatsMonitoringDoc.XCONTENT_FILTERS.forEach(filter ->
                 assertThat(filter + " must not be null in the monitoring document", extractValue(filter, source), notNullValue()));