Enforce a [skip] when using [contains] (#34840)

Be friendly to other runners

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java

@@ -99,6 +99,13 @@ public class ClientYamlTestSection implements Comparable<ClientYamlTestSection>
                     + doSection.getLocation().lineNumber + "]");
             }
         }
+        if (executableSection instanceof ContainsAssertion) {
+            if (false == skipSection.getFeatures().contains("contains")) {
+                throw new IllegalArgumentException("Attempted to add a [contains] assertion without a corresponding "
+                    + "[skip: \"features\": \"contains\"] so runners that do not support the [contains] assertion " +
+                    "can skip the test at line [" + executableSection.getLocation().lineNumber + "]");
+            }
+        }
         this.executableSections.add(executableSection);
     }
 

test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java

@@ -328,4 +328,27 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa
         assertThat(testSection.getExecutableSections().size(), equalTo(3));
     }
 
+    public void testAddingContainsWithoutSkip() {
+        int lineNumber = between(1, 10000);
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
+        if (randomBoolean()) {
+            section.setSkipSection(new SkipSection(null, singletonList("yaml"), null));
+        } else {
+            section.setSkipSection(SkipSection.EMPTY);
+        }
+        Exception e = expectThrows(
+            IllegalArgumentException.class,
+            () -> section.addExecutableSection(
+                new ContainsAssertion(
+                    new XContentLocation(lineNumber, 0),
+                    randomAlphaOfLength(randomIntBetween(3, 30)),
+                    randomDouble()
+                )
+            )
+        );
+        assertEquals("Attempted to add a [contains] assertion without a corresponding " +
+            "[skip: \"features\": \"contains\"] so runners that do not support the [contains] assertion " +
+            "can skip the test at line [" + lineNumber + "]", e.getMessage());
+    }
+
 }