[ML] Delete one type at a time when deleting model snapshots (elastic/x-pack-elasticsearch#1637)

This avoids log spam about being unable to create new mappings in indices
that are set to only allow one type.  (It doesn't actually have any effect
on the deletion, which was working before despite the failure to create new
mappings for the legacy types referenced by the delete request.)

relates elastic/x-pack-elasticsearch#1634

Original commit: elastic/x-pack-elasticsearch@061ce7acf1

plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java

@@ -53,26 +53,60 @@ public class JobDataDeleter {
         }
 
         String stateIndexName = AnomalyDetectorsIndex.jobStateIndexName();
+
+        // TODO: remove in 7.0
+        ActionListener<BulkResponse> docDeleteListener = ActionListener.wrap(
+                response -> {
+                    // if the doc delete worked then don't bother trying the old types
+                    if (response.hasFailures() == false) {
+                        listener.onResponse(response);
+                        return;
+                    }
                     BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
                     for (ModelSnapshot modelSnapshot : modelSnapshots) {
-            for (String stateDocId : modelSnapshot.stateDocumentIds()) {
-                bulkRequestBuilder.add(client.prepareDelete(stateIndexName, ElasticsearchMappings.DOC_TYPE, stateDocId));
-            }
-            // TODO: remove in 7.0
                         for (String stateDocId : modelSnapshot.legacyStateDocumentIds()) {
                             bulkRequestBuilder.add(client.prepareDelete(stateIndexName, ModelState.TYPE, stateDocId));
                         }
 
-            bulkRequestBuilder.add(client.prepareDelete(AnomalyDetectorsIndex.jobResultsAliasedName(modelSnapshot.getJobId()),
-                    ElasticsearchMappings.DOC_TYPE, ModelSnapshot.documentId(modelSnapshot)));
-            // TODO: remove in 7.0
                         bulkRequestBuilder.add(client.prepareDelete(AnomalyDetectorsIndex.jobResultsAliasedName(modelSnapshot.getJobId()),
                                 ModelSnapshot.TYPE.getPreferredName(), ModelSnapshot.v54DocumentId(modelSnapshot)));
                     }
 
                     bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
                     try {
-            bulkRequestBuilder.execute(listener);
+                        bulkRequestBuilder.execute(ActionListener.wrap(
+                                listener::onResponse,
+                                // ignore problems relating to single type indices - if we're running against a single type
+                                // index then it must be type doc, so just return the response from deleting that type
+                                e -> {
+                                    if (e instanceof IllegalArgumentException
+                                            && e.getMessage().contains("as the final mapping would have more than 1 type")) {
+                                        listener.onResponse(response);
+                                    }
+                                    listener.onFailure(e);
+                                }
+                        ));
+                    } catch (Exception e) {
+                        listener.onFailure(e);
+                    }
+                },
+                listener::onFailure
+        );
+
+        BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
+        for (ModelSnapshot modelSnapshot : modelSnapshots) {
+            for (String stateDocId : modelSnapshot.stateDocumentIds()) {
+                bulkRequestBuilder.add(client.prepareDelete(stateIndexName, ElasticsearchMappings.DOC_TYPE, stateDocId));
+            }
+
+            bulkRequestBuilder.add(client.prepareDelete(AnomalyDetectorsIndex.jobResultsAliasedName(modelSnapshot.getJobId()),
+                    ElasticsearchMappings.DOC_TYPE, ModelSnapshot.documentId(modelSnapshot)));
+        }
+
+        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        try {
+            // TODO: change docDeleteListener to listener in 7.0
+            bulkRequestBuilder.execute(docDeleteListener);
         } catch (Exception e) {
             listener.onFailure(e);
         }