[ML] Don't serialise the detector_index field to cluster state (elastic/x-pack-elasticsearch#1628)

Because:

1. It's pointless, as new detector_index values are assigned when an
   analysis_config is parsed
2. It creates a backwards compatibility issue when upgrading from v5.4

Original commit: elastic/x-pack-elasticsearch@2f61aa457e

plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java

@@ -31,6 +31,7 @@ import org.elasticsearch.xpack.ml.job.config.JobState;
 import org.elasticsearch.xpack.ml.job.config.JobTaskStatus;
 import org.elasticsearch.xpack.ml.job.messages.Messages;
 import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.utils.ToXContentParams;
 import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask;
 
@@ -139,8 +140,10 @@ public class MlMetadata implements MetaData.Custom {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        mapValuesToXContent(JOBS_FIELD, jobs, builder, params);
-        mapValuesToXContent(DATAFEEDS_FIELD, datafeeds, builder, params);
+        DelegatingMapParams extendedParams =
+                new DelegatingMapParams(Collections.singletonMap(ToXContentParams.FOR_CLUSTER_STATE, "true"), params);
+        mapValuesToXContent(JOBS_FIELD, jobs, builder, extendedParams);
+        mapValuesToXContent(DATAFEEDS_FIELD, datafeeds, builder, extendedParams);
         return builder;
     }
 

plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfig.java

@@ -353,7 +353,11 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable {
         if (summaryCountFieldName != null) {
             builder.field(SUMMARY_COUNT_FIELD_NAME.getPreferredName(), summaryCountFieldName);
         }
-        builder.field(DETECTORS.getPreferredName(), detectors);
+        builder.startArray(DETECTORS.getPreferredName());
+        for (Detector detector: detectors) {
+            detector.toXContent(builder, params);
+        }
+        builder.endArray();
         builder.field(INFLUENCERS.getPreferredName(), influencers);
         if (overlappingBuckets != null) {
             builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets);

plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java

@@ -18,6 +18,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.ml.job.messages.Messages;
 import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.utils.ToXContentParams;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -382,7 +383,9 @@ public class Detector extends ToXContentToBytes implements Writeable {
         }
         builder.field(DETECTOR_RULES_FIELD.getPreferredName(), detectorRules);
         // negative means "unknown", which should only happen for a 5.4 job
-        if (detectorIndex >= 0) {
+        if (detectorIndex >= 0
+                // no point writing this to cluster state, as the indexes will get reassigned on reload anyway
+                && params.paramAsBoolean(ToXContentParams.FOR_CLUSTER_STATE, false) == false) {
             builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex);
         }
         builder.endObject();

plugin/src/main/java/org/elasticsearch/xpack/ml/utils/ToXContentParams.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.utils;
+
+
+/**
+ * Parameters used by machine learning for controlling X Content serialisation.
+ */
+public final class ToXContentParams {
+
+    /**
+     * Parameter to indicate whether we are serialising to X Content for cluster state output.
+     */
+    public static final String FOR_CLUSTER_STATE = "for_cluster_state";
+
+    private ToXContentParams() {
+    }
+}