[TEST] Improve ML security tests (elastic/x-pack-elasticsearch#2417)

The changes made for elastic/x-pack-elasticsearch#2369 showed that the ML security tests were seriously
weakened by the decision to grant many "minimal" privileges to all users
involved in the tests.  A better solution is to override the auth header
such that a superuser runs setup actions and assertions that work by
querying raw documents in ways that an end user wouldn't.  Then the ML
endpoints can be called with the privileges provided by the ML roles and
nothing else.

Original commit: elastic/x-pack-elasticsearch@4de42d9e54

plugin/src/main/java/org/elasticsearch/xpack/ml/action/DeleteModelSnapshotAction.java

@@ -15,7 +15,6 @@ import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -133,19 +132,16 @@ public class DeleteModelSnapshotAction extends Action<DeleteModelSnapshotAction.
 
         private final InternalClient client;
         private final JobProvider jobProvider;
-        private final JobManager jobManager;
         private final ClusterService clusterService;
         private final Auditor auditor;
 
         @Inject
         public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool,
                                ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                               JobProvider jobProvider, JobManager jobManager, ClusterService clusterService,
+                               JobProvider jobProvider, ClusterService clusterService, InternalClient client, Auditor auditor) {
-                               InternalClient client, Auditor auditor) {
             super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new);
             this.client = client;
             this.jobProvider = jobProvider;
-            this.jobManager = jobManager;
             this.clusterService = clusterService;
             this.auditor = auditor;
         }

plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: custom-all-test-1
         body:  >
@@ -11,6 +13,8 @@ setup:
             "data_description": {}
           }
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: custom-all-test-2
         body:  >
@@ -24,6 +28,8 @@ setup:
 
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-custom-all-test-1
         type:   doc
@@ -48,6 +54,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-custom-all-test-2
         type:   doc
@@ -62,6 +70,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: [.ml-anomalies-shared]
 

plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: datafeeds-crud-1
         body:  >
@@ -17,6 +19,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: datafeeds-crud-2
         body:  >

plugin/src/test/resources/rest-api-spec/test/ml/delete_job_force.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: force-delete-job
         body:  >
@@ -44,7 +46,7 @@ setup:
   - match: { count: 0 }
 
 ---
-"Test can't force delete an inexistent job":
+"Test cannot force delete a non-existent job":
 
   - do:
       catch: /resource_not_found_exception/

plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: delete-model-snapshot
         body:  >
@@ -18,14 +20,20 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.open_job:
         job_id: delete-model-snapshot
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.close_job:
         job_id: delete-model-snapshot
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-delete-model-snapshot
         type:   doc
@@ -42,6 +50,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-state
         type:   doc
@@ -52,6 +62,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-state
         type:   doc
@@ -62,6 +74,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-delete-model-snapshot
         type:   doc
@@ -78,14 +92,20 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-state
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-delete-model-snapshot
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.update_job:
         job_id: delete-model-snapshot
         body:  >
@@ -115,6 +135,8 @@ setup:
   - match: { count: 2 }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-state
         type: doc
@@ -128,10 +150,14 @@ setup:
   - match: { acknowledged: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-delete-model-snapshot
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-state
 
@@ -142,6 +168,8 @@ setup:
   - match: { model_snapshots.0.snapshot_id: "active-snapshot"}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-state
         type: doc

plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml

@@ -2,6 +2,8 @@
 setup:
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-meta
         type: doc
@@ -14,6 +16,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_filter:
         filter_id: filter-foo
         body:  >
@@ -22,6 +26,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_filter:
         filter_id: filter-foo2
         body:  >
@@ -30,6 +36,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh: {}
 
 ---

plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.create:
         index: index-1
         body:
@@ -13,6 +15,8 @@ setup:
                   type: date
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.create:
         index: index-2
         body:
@@ -26,6 +30,8 @@ setup:
                   type: date
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: get-datafeed-stats-1
         body:  >
@@ -43,6 +49,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: get-datafeed-stats-2
         body:  >
@@ -58,6 +66,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_datafeed:
         datafeed_id: datafeed-1
         body: >
@@ -68,6 +78,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_datafeed:
         datafeed_id: datafeed-2
         body: >

plugin/src/test/resources/rest-api-spec/test/ml/get_datafeeds.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: get-datafeed-1
         body:  >
@@ -17,6 +19,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: get-datafeed-2
         body:  >
@@ -32,6 +36,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_datafeed:
         datafeed_id: datafeed-1
         body: >
@@ -42,6 +48,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_datafeed:
         datafeed_id: datafeed-2
         body: >

plugin/src/test/resources/rest-api-spec/test/ml/get_model_snapshots.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: get-model-snapshots
         body:  >
@@ -14,6 +16,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-get-model-snapshots
         type:   doc
@@ -27,6 +31,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-get-model-snapshots
         type:   doc
@@ -40,6 +46,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-get-model-snapshots
 

plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml

@@ -85,29 +85,41 @@
   - match: { closed: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: ".ml-state"
   - is_true: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: ".ml-anomalies-index-layout-job"
   - is_true: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh: {}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-state
   - gt: {count: 0}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-shared
   - match: {count: 6}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-index-layout-job
         body:
@@ -120,6 +132,8 @@
   - match: {count: 3}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-shared
         body:
@@ -132,6 +146,8 @@
   - match: {count: 3}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-index-layout-job2
         body:
@@ -144,6 +160,8 @@
   - match: {count: 3}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-shared
         body:
@@ -155,6 +173,8 @@
   - match: {count: 3}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-state
         type: doc
@@ -163,12 +183,14 @@
           key: value
 
   - do:
-       index:
+      headers:
-         index: .ml-state
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-         type: doc
+      index:
-         id: index-layout-job2_categorizer_state#2
+        index: .ml-state
-         body:
+        type: doc
-           key: value
+        id: index-layout-job2_categorizer_state#2
+        body:
+          key: value
 
   - do:
       xpack.ml.delete_job:
@@ -176,16 +198,22 @@
   - match: { acknowledged: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: ".ml-anomalies-shared"
   - is_true: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-state
   - match: {count: 4}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-shared
         body:
@@ -197,12 +225,16 @@
   - match: {count: 0}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-shared
   - match: {count: 3}
 
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-index-layout-job2
         body:
@@ -215,6 +247,8 @@
   - match: {count: 3}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-shared
         body:
@@ -232,44 +266,60 @@
   - match: { acknowledged: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: ".ml-anomalies-shared"
   - is_true: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: ".ml-anomalies-index-layout-job"
   - is_false: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: ".ml-anomalies-index-layout-job2"
   - is_false: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         type: doc
         index: .ml-state
   - match: {count: 0}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         type: doc
         index: .ml-state
   - match: {count: 0}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         type: doc
         index: .ml-state
   - match: {count: 0}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-state
   - match: {count: 0}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-shared
   - match: {count: 0}
@@ -309,14 +359,20 @@
           {"airline":"JZA","responsetime":"990.4628","sourcetype":"farequote","time":"1403481700"}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.create:
         index: foo
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.create:
         index: .ml-anomalies-foo
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: foo
         type: doc
@@ -324,6 +380,8 @@
           key: value
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-anomalies-foo
         type: doc
@@ -331,6 +389,8 @@
           key: value
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-anomalies-foo
         type: doc
@@ -354,44 +414,62 @@
   - match: { acknowledged: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: ".ml-anomalies-shared"
   - is_true: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: ".ml-anomalies-foo"
   - is_true: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: ".ml-state"
   - is_true: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: "foo"
   - is_true: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-state
   - match: {count: 0}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-shared
   - match: {count: 0}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh: {}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: foo
   - match: {count: 1}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-foo
   - match: {count: 2}
@@ -415,6 +493,8 @@
   - match: { job_id: "index-layout-quantiles-job" }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-state
         type: doc
@@ -423,6 +503,8 @@
           state: quantile-state
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh: {}
 
   - do:
@@ -431,6 +513,8 @@
   - match: { acknowledged: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-state
   - match: {count: 0}
@@ -455,6 +539,8 @@
   - match: { job_id: "index-layout-state-job" }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-shared
         type:   doc
@@ -469,6 +555,8 @@
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-state
         type: doc
@@ -477,6 +565,8 @@
           state: new-model-state
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-state
         type: doc
@@ -485,6 +575,8 @@
           state: more-new-model-state
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-state
         type: doc
@@ -493,6 +585,8 @@
           state: new-categorizer-state
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-state
         type: doc
@@ -501,6 +595,8 @@
           state: more-new-categorizer-state
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh: {}
 
   - do:
@@ -509,11 +605,15 @@
   - match: { acknowledged: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-anomalies-shared
   - match: {count: 0}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-state
   - match: {count: 0}
@@ -522,6 +622,8 @@
 "Test force close does not create state":
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.create:
         index: .ml-state
 
@@ -560,14 +662,20 @@
   - match: { closed: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.exists:
         index: ".ml-state"
   - is_true: ''
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh: {}
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       count:
         index: .ml-state
   - match: {count: 0}

plugin/src/test/resources/rest-api-spec/test/ml/job_groups.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: test-job-groups-foo-1
         body:  >
@@ -12,6 +14,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: test-job-groups-foo-2
         body:  >
@@ -24,6 +28,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: test-job-groups-bar-1
         body:  >
@@ -36,6 +42,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: test-job-groups-bar-2
         body:  >
@@ -48,6 +56,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: test-job-groups-nogroup
         body:  >

plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml

@@ -449,6 +449,8 @@
   - match: { flushed: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       cluster.state:
         metric: [ metadata ]
         filter_path: metadata.persistent_tasks
@@ -460,6 +462,8 @@
   - match: { closed: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       cluster.state:
         metric: [ metadata ]
         filter_path: metadata.persistent_tasks
@@ -665,6 +669,8 @@
   - match: { flushed: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       cluster.state:
         metric: [ metadata ]
         filter_path: metadata.persistent_tasks
@@ -677,6 +683,8 @@
   - match: { closed: true }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       cluster.state:
         metric: [ metadata ]
         filter_path: metadata.persistent_tasks

plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: jobs-get-1
         body:  >
@@ -18,6 +20,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: jobs-get-2
         body:  >

plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_buckets.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: jobs-get-result-buckets
         body:  >
@@ -14,6 +16,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-jobs-get-result-buckets
         type:   doc
@@ -28,6 +32,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-jobs-get-result-buckets
         type:   doc
@@ -42,6 +48,8 @@ setup:
             "is_interim": true
           }
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-jobs-get-result-buckets
         type:   doc
@@ -56,6 +64,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-jobs-get-result-buckets
 

plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_categories.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: jobs-get-result-categories
         body:  >
@@ -14,18 +16,24 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-jobs-get-result-categories
         type:   doc
         id:     jobs-get-result-categories-1
         body:   { "job_id": "jobs-get-result-categories", "category_id": 1 }
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-jobs-get-result-categories
         type:   doc
         id:     jobs-get-result-categories-2
         body:   { "job_id": "jobs-get-result-categories", "category_id": 2 }
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-unrelated
         type:   doc
@@ -33,6 +41,8 @@ setup:
         body:   { "job_id": "unrelated", "category_id": 1 }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-jobs-get-result-categories
 

plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: get-influencers-test
         body:  >
@@ -14,6 +16,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-get-influencers-test
         type:   doc
@@ -30,6 +34,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-get-influencers-test
         type:   doc
@@ -47,6 +53,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-get-influencers-test
         type:   doc
@@ -63,6 +71,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-get-influencers-test
 

plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: jobs-get-result-records
         body:  >
@@ -14,6 +16,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-jobs-get-result-records
         type:   doc
@@ -28,21 +32,25 @@ setup:
           }
 
   - do:
-        index:
+      headers:
-          index:  .ml-anomalies-jobs-get-result-records
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-          type:   doc
+      index:
-          id:     jobs-get-result-records_1464825600000_1_2
+        index:  .ml-anomalies-jobs-get-result-records
-          body:
+        type:   doc
-            {
+        id:     jobs-get-result-records_1464825600000_1_2
-              "job_id": "jobs-get-result-records",
+        body:
-              "result_type": "record",
+          {
-              "timestamp": "2016-06-02T00:00:00Z",
+            "job_id": "jobs-get-result-records",
-              "record_score": 80.0,
+            "result_type": "record",
-              "bucket_span": 1,
+            "timestamp": "2016-06-02T00:00:00Z",
-              "is_interim": true
+            "record_score": 80.0,
-            }
+            "bucket_span": 1,
+            "is_interim": true
+          }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-jobs-get-result-records
 

plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: job-stats-test
         body:  >
@@ -18,10 +20,14 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.open_job:
         job_id: job-stats-test
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: jobs-get-stats-datafeed-job
         body:  >
@@ -39,10 +45,14 @@ setup:
             }
           }
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.open_job:
         job_id: jobs-get-stats-datafeed-job
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_datafeed:
         datafeed_id: datafeed-1
         body:  >
@@ -196,6 +206,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh: {}
 
   # This is testing that the documents with v5.4 IDs are fetched.
@@ -203,6 +215,8 @@ setup:
   # for another type into the single type indices. Type isn't used
   # in the query so the test is valid
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-anomalies-shared
         type: doc
@@ -225,6 +239,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index: .ml-anomalies-shared
         type: doc
@@ -243,6 +259,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: [.ml-anomalies-shared]
 

plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml

@@ -17,6 +17,8 @@
   - match: { job_id: "ml-anomalies-default-mappings-job" }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-shared
         type:   doc
@@ -27,10 +29,14 @@
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-shared
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.get_field_mapping:
         index: .ml-anomalies-shared
         type: doc
@@ -56,10 +62,14 @@
   - match: { job_id: "ml-anomalies-shared-mappings-job1" }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-shared
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.get_mapping:
         index: .ml-anomalies-shared
   - is_true: \.ml-anomalies-shared.mappings.doc._meta.version
@@ -80,10 +90,14 @@
   - match: { job_id: "ml-anomalies-shared-mappings-job2" }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-shared
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.get_mapping:
         index: .ml-anomalies-shared
   - is_true: \.ml-anomalies-shared.mappings.doc._meta.version

plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: post-data-job
         body:  >
@@ -18,6 +20,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: post-data-closed-job
         body:  >
@@ -33,6 +37,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.open_job:
         job_id: post-data-job
 

plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: revert-model-snapshot
         body:  >
@@ -18,14 +20,20 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.open_job:
         job_id: revert-model-snapshot
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.close_job:
         job_id: revert-model-snapshot
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-revert-model-snapshot
         type:   doc
@@ -51,6 +59,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-revert-model-snapshot
         type:   doc
@@ -76,6 +86,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-revert-model-snapshot
         type:   doc
@@ -89,6 +101,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-revert-model-snapshot
         type:   doc
@@ -102,6 +116,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-revert-model-snapshot
         type:   doc
@@ -115,7 +131,9 @@ setup:
           }
 
   - do:
-       index:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      index:
         index:  .ml-anomalies-revert-model-snapshot
         type:   doc
         id:     "revert-model-snapshot_1464825600000_1_1"
@@ -128,7 +146,9 @@ setup:
           }
 
   - do:
-       index:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      index:
          index:  .ml-anomalies-revert-model-snapshot
          type:   doc
          id:     "revert-model-snapshot_1462060800000_1_2"
@@ -141,6 +161,8 @@ setup:
            }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-revert-model-snapshot
         type:   doc
@@ -156,6 +178,8 @@ setup:
                 }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-revert-model-snapshot
         type:   doc
@@ -172,10 +196,14 @@ setup:
             }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-revert-model-snapshot
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-state
 
@@ -228,6 +256,8 @@ setup:
         delete_intervening_results: true
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-revert-model-snapshot
 

plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml

@@ -16,6 +16,8 @@ setup:
                   type: float
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: start-stop-datafeed-job
         body:  >
@@ -34,6 +36,8 @@ setup:
             }
           }
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_datafeed:
         datafeed_id: start-stop-datafeed-datafeed-1
         body:  >

plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml

@@ -1,5 +1,7 @@
 setup:
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       xpack.ml.put_job:
         job_id: update-model-snapshot
         body:  >
@@ -14,6 +16,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-update-model-snapshot
         type:   doc
@@ -28,6 +32,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       index:
         index:  .ml-anomalies-update-model-snapshot
         type:   doc
@@ -43,6 +49,8 @@ setup:
           }
 
   - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
         index: .ml-anomalies-update-model-snapshot
 
@@ -69,10 +77,6 @@ setup:
   - match: { model.retain: false }
   - match: { model.description: "new_description" }
 
-  - do:
-      indices.refresh:
-        index: .ml-anomalies-update-model-snapshot
-
   - do:
       xpack.ml.get_model_snapshots:
         job_id: "update-model-snapshot"
@@ -81,6 +85,7 @@ setup:
   - match: { count: 1 }
   - match: { model_snapshots.0.snapshot_id: "snapshot-1" }
   - match: { model_snapshots.0.timestamp: 1464825600000 }
+  - match: { model_snapshots.0.description: "new_description" }
 
 ---
 "Test duplicate descriptions are allowed":
@@ -93,10 +98,6 @@ setup:
             "description": "snapshot 2 description"
           }
 
-  - do:
-      indices.refresh:
-        index: .ml-anomalies-update-model-snapshot
-
   - do:
       xpack.ml.get_model_snapshots:
         job_id: "update-model-snapshot"

qa/smoke-test-ml-with-security/build.gradle

@@ -17,10 +17,12 @@ integTestRunner {
   systemProperty 'tests.rest.blacklist', [
     // Remove tests that are expected to throw an exception, because we cannot then
     // know whether to expect an authorization exception or a validation exception
+    'ml/custom_all_field/Test querying custom all field',
     'ml/datafeeds_crud/Test delete datafeed with missing id',
     'ml/datafeeds_crud/Test put datafeed referring to missing job_id',
     'ml/datafeeds_crud/Test put datafeed with invalid query',
     'ml/datafeeds_crud/Test update datafeed with missing id',
+    'ml/delete_job_force/Test cannot force delete a non-existent job',
     'ml/delete_model_snapshot/Test delete snapshot missing snapshotId',
     'ml/delete_model_snapshot/Test delete snapshot missing job_id',
     'ml/delete_model_snapshot/Test delete with in-use model',
@@ -46,6 +48,11 @@ integTestRunner {
     'ml/jobs_get_result_categories/Test with invalid param combinations via body',
     'ml/jobs_get_stats/Test get job stats given missing job',
     'ml/jobs_get_stats/Test no exception on get job stats with missing index',
+    'ml/job_groups/Test put job with empty group',
+    'ml/job_groups/Test put job with group that matches an job id',
+    'ml/job_groups/Test put job with group that matches its id',
+    'ml/job_groups/Test put job with id that matches an existing group',
+    'ml/job_groups/Test put job with invalid group',
     'ml/post_data/Test Flush data with invalid parameters',
     'ml/post_data/Test flushing and posting a closed job',
     'ml/post_data/Test open and close with non-existent job id',
@@ -68,7 +75,7 @@ integTestCluster {
   plugin ':x-pack-elasticsearch:plugin'
   extraConfigFile 'x-pack/roles.yml', 'roles.yml'
   setupCommand 'setupTestAdminUser',
-               'bin/x-pack/users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
+               'bin/x-pack/users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
   setupCommand 'setupMlAdminUser',
                'bin/x-pack/users', 'useradd', 'ml_admin', '-p', 'x-pack-test-password', '-r', 'minimal,machine_learning_admin'
   setupCommand 'setupMlUserUser',
@@ -79,7 +86,7 @@ integTestCluster {
     File tmpFile = new File(node.cwd, 'wait.success')
     ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
             dest: tmpFile.toString(),
-            username: 'test_admin',
+            username: 'x_pack_rest_user',
             password: 'x-pack-test-password',
             ignoreerrors: true,
             retries: 10)

qa/smoke-test-ml-with-security/roles.yml

@@ -1,15 +1,15 @@
 minimal:
   cluster:
-    - cluster:monitor/health
+    # This is always required because the REST client uses it to find the version of
+    # Elasticsearch it's talking to
     - cluster:monitor/main
-    - cluster:monitor/state
   indices:
-    - names: '*'
+    # Give all users involved in these tests access to the indices where the data to
+    # be analyzed is stored, because the ML roles alone do not provide access to
+    # non-ML indices
+    - names: [ 'airline-data', 'index-foo', 'unavailable-data' ]
       privileges:
         - indices:admin/create
-        - indices:admin/exists
-        - indices:admin/get
-        - indices:admin/mapping/put
         - indices:admin/refresh
         - indices:data/read/field_caps
         - indices:data/read/search

qa/smoke-test-ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityIT.java

@@ -13,6 +13,7 @@ import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
@@ -36,8 +37,7 @@ import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordTok
 
 public class MlWithSecurityIT extends ESClientYamlSuiteTestCase {
 
-    private static final String TEST_ADMIN_USERNAME = "test_admin";
+    private static final String TEST_ADMIN_USERNAME = "x_pack_rest_user";
-    private static final String TEST_ADMIN_PASSWORD = "x-pack-test-password";
 
     @After
     public void clearMlState() throws Exception {
@@ -99,7 +99,7 @@ public class MlWithSecurityIT extends ESClientYamlSuiteTestCase {
                                            Map<String, String> params,
                                            List<Map<String, Object>> bodies) throws IOException {
         Map<String, String> headers = Collections.singletonMap("Authorization",
-                basicAuthHeaderValue(TEST_ADMIN_USERNAME, new SecureString(TEST_ADMIN_PASSWORD.toCharArray())));
+                basicAuthHeaderValue(TEST_ADMIN_USERNAME, SecuritySettingsSource.TEST_PASSWORD_SECURE_STRING));
         return getAdminExecutionContext().callApi(apiName, params, bodies, headers);
     }
 
@@ -123,7 +123,7 @@ public class MlWithSecurityIT extends ESClientYamlSuiteTestCase {
 
     @Override
     protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, new SecureString(TEST_ADMIN_PASSWORD.toCharArray()));
+        String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, SecuritySettingsSource.TEST_PASSWORD_SECURE_STRING);
         return Settings.builder()
             .put(ThreadContext.PREFIX + ".Authorization", token)
             .build();

qa/smoke-test-ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityInsufficientRoleIT.java

@@ -28,10 +28,15 @@ public class MlWithSecurityInsufficientRoleIT extends MlWithSecurityIT {
             super.test();
             fail("should have failed because of missing role");
         } catch (AssertionError ae) {
-            assertThat(ae.getMessage(),
+            // Some tests assert on searches of wildcarded ML indices rather than on ML endpoints.  For these we expect no hits.
-                    either(containsString("action [cluster:monitor/xpack/ml")).or(containsString("action [cluster:admin/xpack/ml")));
+            if (ae.getMessage().contains("hits.total didn't match expected value")) {
-            assertThat(ae.getMessage(), containsString("returned [403 Forbidden]"));
+                assertThat(ae.getMessage(), containsString("but was [0]"));
-            assertThat(ae.getMessage(), containsString("is unauthorized for user [no_ml]"));
+            } else {
+                assertThat(ae.getMessage(),
+                        either(containsString("action [cluster:monitor/xpack/ml")).or(containsString("action [cluster:admin/xpack/ml")));
+                assertThat(ae.getMessage(), containsString("returned [403 Forbidden]"));
+                assertThat(ae.getMessage(), containsString("is unauthorized for user [no_ml]"));
+            }
         }
     }
 

qa/smoke-test-ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityUserRoleIT.java

@@ -30,10 +30,11 @@ public class MlWithSecurityUserRoleIT extends MlWithSecurityIT {
         try {
             super.test();
 
-            // We should have got here if and only if the test consisted entirely of GETs
+            // We should have got here if and only if the only ML endpoints in the test were GETs
             for (ExecutableSection section : testCandidate.getTestSection().getExecutableSections()) {
                 if (section instanceof DoSection) {
-                    if (((DoSection) section).getApiCallSection().getApi().startsWith("xpack.ml.get_") == false) {
+                    if (((DoSection) section).getApiCallSection().getApi().startsWith("xpack.ml.") &&
+                            ((DoSection) section).getApiCallSection().getApi().startsWith("xpack.ml.get_") == false) {
                         fail("should have failed because of missing role");
                     }
                 }