[TEST] Enforce skip headers when needed (#34735)

The java yaml test runner supports sending request headers, yet not all clients support headers. This commit makes sure that we enforce adding a skip section with feature "headers" whenever headers are used in a do section as part of a test. That decreases the chance for new tests to break client builds due to the missing skip section. Closes #34650
2025-02-17 02:14:54 +00:00 · 2018-10-31 13:07:02 +01:00 · 2018-10-31 13:07:02 +01:00 · 674225aaa1
commit 674225aaa1
parent 00dc2ba36f
46 changed files with 157 additions and 59 deletions
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
@ -82,6 +82,9 @@

 ---
 "Find a task result record from the old cluster":
+  - skip:
+      features: headers
+
  - do:
      search:
        index: .tasks
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
@ -86,7 +86,6 @@ public class ClientYamlTestSuite {

        SetupSection setupSection = SetupSection.parseIfNext(parser);
        TeardownSection teardownSection = TeardownSection.parseIfNext(parser);
-
        Set<ClientYamlTestSection> testSections = new TreeSet<>();
        while(true) {
            //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed
@ -179,6 +178,14 @@ public class ClientYamlTestSuite {
                "without a corresponding [\"skip\": \"features\": \"contains\"] so runners that do not support the " +
                "[contains] assertion can skip the test at line [" + section.getLocation().lineNumber + "]"));

+        errors = Stream.concat(errors, sections.stream().filter(section -> section instanceof DoSection)
+            .map(section -> (DoSection) section)
+            .filter(section -> false == section.getApiCallSection().getHeaders().isEmpty())
+            .filter(section -> false == hasSkipFeature("headers", testSection, setupSection, teardownSection))
+            .map(section -> "attempted to add a [do] with a [headers] section without a corresponding "
+                + "[\"skip\": \"features\": \"headers\"] so runners that do not support the [headers] section can skip the test at " +
+                "line [" + section.getLocation().lineNumber + "]"));
+
        return errors;
    }

--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Map;

 import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@ -403,11 +404,24 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
        doSection.setExpectedWarningHeaders(singletonList("foo"));
        doSection.setApiCallSection(new ApiCallSection("test"));
-        ClientYamlTestSuite testSuite = createTestSuite(doSection);
+        ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertEquals("api/name:\nattempted to add a [do] with a [warnings] section without a corresponding " +
+        assertThat(e.getMessage(), containsString("api/name:\nattempted to add a [do] with a [warnings] section without a corresponding " +
            "[\"skip\": \"features\": \"warnings\"] so runners that do not support the [warnings] section can skip the test " +
-            "at line [" + lineNumber + "]", e.getMessage());
+            "at line [" + lineNumber + "]"));
+    }
+
+    public void testAddingDoWithHeaderWithoutSkipHeaders() {
+        int lineNumber = between(1, 10000);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        ApiCallSection apiCallSection = new ApiCallSection("test");
+        apiCallSection.addHeaders(Collections.singletonMap("header", "value"));
+        doSection.setApiCallSection(apiCallSection);
+        ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
+        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
+        assertThat(e.getMessage(), containsString("api/name:\nattempted to add a [do] with a [headers] section without a corresponding " +
+            "[\"skip\": \"features\": \"headers\"] so runners that do not support the [headers] section can skip the test at line ["
+            + lineNumber + "]"));
    }

    public void testAddingDoWithNodeSelectorWithoutSkipNodeSelector() {
@ -416,11 +430,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
        ApiCallSection apiCall = new ApiCallSection("test");
        apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
        doSection.setApiCallSection(apiCall);
-        ClientYamlTestSuite testSuite = createTestSuite(doSection);
+        ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertEquals("api/name:\nattempted to add a [do] with a [node_selector] section without a corresponding"
-            + " [\"skip\": \"features\": \"node_selector\"] so runners that do not support the [node_selector] section can skip the test at"
-            + " line [" + lineNumber + "]", e.getMessage());
+        assertThat(e.getMessage(), containsString("api/name:\nattempted to add a [do] with a [node_selector] section without a " +
+            "corresponding [\"skip\": \"features\": \"node_selector\"] so runners that do not support the [node_selector] section can " +
+            "skip the test at line [" + lineNumber + "]"));
    }

    public void testAddingContainsWithoutSkipContains() {
@ -429,11 +443,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
            new XContentLocation(lineNumber, 0),
            randomAlphaOfLength(randomIntBetween(3, 30)),
            randomDouble());
-        ClientYamlTestSuite testSuite = createTestSuite(containsAssertion);
+        ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, containsAssertion);
        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertEquals("api/name:\nattempted to add a [contains] assertion without a corresponding " +
+        assertThat(e.getMessage(), containsString("api/name:\nattempted to add a [contains] assertion without a corresponding " +
            "[\"skip\": \"features\": \"contains\"] so runners that do not support the [contains] assertion " +
-            "can skip the test at line [" + lineNumber + "]", e.getMessage());
+            "can skip the test at line [" + lineNumber + "]"));
    }

    public void testMultipleValidationErrors() {
@ -477,44 +491,13 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
            e.getMessage());
    }

-    private static ClientYamlTestSuite createTestSuite(ExecutableSection executableSection) {
-        final SetupSection setupSection;
-        final TeardownSection teardownSection;
-        final ClientYamlTestSection clientYamlTestSection;
-        switch(randomIntBetween(0, 2)) {
-            case 0:
-                setupSection = new SetupSection(SkipSection.EMPTY, Collections.singletonList(executableSection));
-                teardownSection = TeardownSection.EMPTY;
-                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
-                    SkipSection.EMPTY, Collections.emptyList());
-                break;
-            case 1:
-                setupSection = SetupSection.EMPTY;
-                teardownSection = new TeardownSection(SkipSection.EMPTY, Collections.singletonList(executableSection));
-                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
-                    SkipSection.EMPTY, Collections.emptyList());
-                break;
-            case 2:
-                setupSection = SetupSection.EMPTY;
-                teardownSection = TeardownSection.EMPTY;
-                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
-                    SkipSection.EMPTY, Collections.singletonList(executableSection));
-                break;
-            default:
-                throw new UnsupportedOperationException();
-        }
-
-        return new ClientYamlTestSuite("api", "name", setupSection, teardownSection,
-            Collections.singletonList(clientYamlTestSection));
-    }
-
    public void testAddingDoWithWarningWithSkip() {
        int lineNumber = between(1, 10000);
        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
        doSection.setExpectedWarningHeaders(singletonList("foo"));
        doSection.setApiCallSection(new ApiCallSection("test"));
        SkipSection skipSection = new SkipSection(null, singletonList("warnings"), null);
-        createTestSuiteAndValidate(skipSection, doSection);
+        createTestSuite(skipSection, doSection).validate();
    }

    public void testAddingDoWithNodeSelectorWithSkip() {
@ -524,7 +507,17 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
        ApiCallSection apiCall = new ApiCallSection("test");
        apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
        doSection.setApiCallSection(apiCall);
-        createTestSuiteAndValidate(skipSection, doSection);
+        createTestSuite(skipSection, doSection).validate();
+    }
+
+    public void testAddingDoWithHeadersWithSkip() {
+        int lineNumber = between(1, 10000);
+        SkipSection skipSection = new SkipSection(null, singletonList("headers"), null);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        ApiCallSection apiCallSection = new ApiCallSection("test");
+        apiCallSection.addHeaders(singletonMap("foo", "bar"));
+        doSection.setApiCallSection(apiCallSection);
+        createTestSuite(skipSection, doSection).validate();
    }

    public void testAddingContainsWithSkip() {
@ -534,10 +527,10 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
            new XContentLocation(lineNumber, 0),
            randomAlphaOfLength(randomIntBetween(3, 30)),
            randomDouble());
-        createTestSuiteAndValidate(skipSection, containsAssertion);
+        createTestSuite(skipSection, containsAssertion).validate();
    }

-    private static void createTestSuiteAndValidate(SkipSection skipSection, ExecutableSection executableSection) {
+    private static ClientYamlTestSuite createTestSuite(SkipSection skipSection, ExecutableSection executableSection) {
        final SetupSection setupSection;
        final TeardownSection teardownSection;
        final ClientYamlTestSection clientYamlTestSection;
@ -571,13 +564,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
                teardownSection = new TeardownSection(skipSection, Collections.singletonList(executableSection));
                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
                    SkipSection.EMPTY, randomBoolean() ? Collections.emptyList() : Collections.singletonList(executableSection));
-
                break;
            default:
                throw new UnsupportedOperationException();
        }
-        ClientYamlTestSuite clientYamlTestSuite = new ClientYamlTestSuite("api", "name", setupSection, teardownSection,
+        return new ClientYamlTestSuite("api", "name", setupSection, teardownSection,
            Collections.singletonList(clientYamlTestSection));
-        clientYamlTestSuite.validate();
    }
 }
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_forecast.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_forecast.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_job_force.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_job_force.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
@ -1,6 +1,7 @@
 ---
 setup:
-
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml
@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Test NDJSON file structure analysis without overrides":
  - do:
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeeds.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeeds.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_model_snapshots.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_model_snapshots.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Test CRUD on two jobs in shared index":

--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_groups.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_groups.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
@ -399,6 +399,8 @@

 ---
 "Test cannot decrease model_memory_limit below current usage":
+  - skip:
+      features: headers
  - do:
      xpack.ml.put_job:
        job_id: jobs-crud-model-memory-limit-decrease
@ -527,7 +529,8 @@

 ---
 "Test close job":
-
+  - skip:
+      features: headers
  - do:
      xpack.ml.put_job:
        job_id: jobs-crud-close-job
@ -762,7 +765,8 @@

 ---
 "Test force close job":
-
+  - skip:
+      features: headers
  - do:
      xpack.ml.put_job:
        job_id: jobs-crud-force-close-job
@ -929,7 +933,8 @@

 ---
 "Test cannot create job with existing result document":
-
+  - skip:
+      features: headers
  - do:
      headers:
        Content-Type: application/json
@ -1068,7 +1073,8 @@

 ---
 "Test max model memory limit":
-
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@ -1315,7 +1321,8 @@

 ---
 "Test open job when persistent task allocation disabled":
-
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_buckets.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_buckets.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_categories.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_categories.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml
@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Test new fields are mapped as keyword":

--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      indices.create:
        index: airline-data
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/40_get_user_privs.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/40_get_user_privs.yml
@ -204,7 +204,7 @@ teardown:
 "Test get_user_privileges for single role":
  - skip:
          reason: "contains is a newly added assertion"
-          features: contains  
+          features: contains
  - do:
      headers: { Authorization: "Basic dGVzdC0xOjEyMzQ1Njc4" } # test-1
      xpack.security.get_user_privileges: {}
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      indices.create:
        index: foo
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      indices.create:
        index: foo
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      indices.create:
        index: foo
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      indices.create:
        index: foo
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      indices.create:
        index: foo
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      indices.create:
        index: foo
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/security_tests.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/security_tests.yml
@ -6,8 +6,6 @@ setup:
      cluster.health:
          wait_for_status: yellow

-
-
 ---
 teardown:
  - do:
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      indices.create:
        index: foo
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/stop_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/stop_job.yml
@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
  - do:
      indices.create:
        index: foo
--- a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/10_reindex.yml
+++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/10_reindex.yml
@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Reindex as same user works":

--- a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml
+++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml
@ -163,6 +163,8 @@

 ---
 "Using a script to write to an index to which you don't have access is forbidden even if you read as a superuser":
+  - skip:
+      features: headers
  - do:
      index:
        index:        source
--- a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/20_update_by_query.yml
+++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/20_update_by_query.yml
@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Update_by_query as same user works":

--- a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/30_delete_by_query.yml
+++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/30_delete_by_query.yml
@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Delete_by_query as same user works":

--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_security.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_security.yml
@ -1,5 +1,7 @@
 ---
 "Verify native store security actions":
+  - skip:
+      features: headers
  # create native user and role
  - do:
     xpack.security.put_user:
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_security.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_security.yml
@ -1,5 +1,7 @@
 ---
 "Verify user and role in upgraded cluster":
+  - skip:
+      features: headers
  - do:
      headers:
        Authorization: "Basic bmF0aXZlX3VzZXI6eC1wYWNrLXRlc3QtcGFzc3dvcmQ="
--- a/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/10_insufficient_privs.yml
+++ b/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/10_insufficient_privs.yml
@ -1,5 +1,7 @@
 ---
 "Test watcher is protected by security":
+  - skip:
+      features: headers
  - do:
      headers: { es-security-runas-user: powerless_user }
      catch: forbidden
--- a/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/20_test_run_as_execute_watch.yml
+++ b/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/20_test_run_as_execute_watch.yml
@ -28,6 +28,8 @@ teardown:

 ---
 "Test watch search input is run as user who added the watch":
+  - skip:
+      features: headers
  - do:
      xpack.watcher.put_watch:
        id: "my_watch"
@ -81,6 +83,8 @@ teardown:

 ---
 "Test watch is runas user properly recorded":
+  - skip:
+      features: headers
  - do:
      xpack.watcher.put_watch:
        id: "my_watch"
@ -133,7 +137,8 @@ teardown:

 ---
 "Test watch search input does not work against index user is not allowed to read":
-
+  - skip:
+      features: headers
  - do:
      # by impersonating this request as powerless user we cannot query the my_test_index
      # headers: { es-security-runas-user: powerless_user }
@ -290,6 +295,8 @@ teardown:

 ---
 "Test watch index action requires permission to write to an index":
+  - skip:
+      features: headers
  - do:
      xpack.watcher.put_watch:
        id: "my_watch"
@ -339,6 +346,8 @@ teardown:

 ---
 "Test watch index action does not work without permissions":
+  - skip:
+      features: headers
  - do:
      xpack.watcher.put_watch:
        id: "my_watch"