Build a plugin for testing docs

This makes it much easier to apply to other projects.

Fixes to doc tests infrastructure:
* Fix comparing lists. Was totally broken.
* Fix order of actual vs expected parameters.
* Allow multiple `// TESTRESPONSE` lines with substitutions to join
into one big list of subtitutions. This makes lets the docs look
tidier.
* Exclude build from snippet scanning
* Allow subclasses of ESRestTestCase access to the admin execution context

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.doc
+
+import org.elasticsearch.gradle.test.RestTestPlugin
+import org.gradle.api.Project
+import org.gradle.api.Task
+
+/**
+ * Sets up tests for documentation.
+ */
+public class DocsTestPlugin extends RestTestPlugin {
+
+    @Override
+    public void apply(Project project) {
+        super.apply(project)
+        Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)
+        listSnippets.group 'Docs'
+        listSnippets.description 'List each snippet'
+        listSnippets.perSnippet { println(it.toString()) }
+
+        Task listConsoleCandidates = project.tasks.create(
+                'listConsoleCandidates', SnippetsTask)
+        listConsoleCandidates.group 'Docs'
+        listConsoleCandidates.description
+                'List snippets that probably should be marked // CONSOLE'
+        listConsoleCandidates.perSnippet {
+            if (
+                       it.autoSense    // Already marked, nothing to do
+                    || it.testResponse // It is a response
+                ) {
+                return
+            }
+            List<String> languages = [
+                // These languages should almost always be marked autosense
+                'js', 'json',
+                // These are often curl commands that should be converted but
+                // are probably false positives
+                'sh', 'shell',
+            ]
+            if (false == languages.contains(it.language)) {
+                return
+            }
+            println(it.toString())
+        }
+
+        project.tasks.create('buildRestTests', RestTestsFromSnippetsTask)
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -17,9 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.gradle
+package org.elasticsearch.gradle.doc
 
-import org.elasticsearch.gradle.SnippetsTask.Snippet
+import org.elasticsearch.gradle.doc.SnippetsTask.Snippet
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.OutputDirectory

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.gradle
+package org.elasticsearch.gradle.doc
 
 import org.gradle.api.DefaultTask
 import org.gradle.api.InvalidUserDataException
@@ -55,6 +55,8 @@ public class SnippetsTask extends DefaultTask {
     ConfigurableFileTree docs = project.fileTree(project.projectDir) {
         // No snippets in the build file
         exclude 'build.gradle'
+        // That is where the snippets go, not where they come from!
+        exclude 'build'
     }
 
     @TaskAction
@@ -166,7 +168,9 @@ public class SnippetsTask extends DefaultTask {
                     }
                     snippet.testResponse = true
                     if (matcher.group(2) != null) {
-                        substitutions = []
+                        if (substitutions == null) {
+                            substitutions = []
+                        }
                         String loc = "$file:$lineNumber"
                         parse(loc, matcher.group(2), /$SUBSTITUTION ?/) {
                             substitutions.add([it.group(1), it.group(2)])

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docs-test.properties

@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.doc.DocsTestPlugin

docs/build.gradle

@@ -17,50 +17,27 @@
  * under the License.
  */
 
-import org.elasticsearch.gradle.SnippetsTask
+apply plugin: 'elasticsearch.docs-test'
-import org.elasticsearch.gradle.SnippetsTask.Snippet
-import org.elasticsearch.gradle.RestTestsFromSnippetsTask
 
-apply plugin: 'elasticsearch.rest-test'
+integTest {
-
+  cluster {
-task listSnippets(type: SnippetsTask) {
+    setting 'script.inline', 'true'
-  group 'Docs'
-  description 'List each snippet'
-  perSnippet { println(it) }
-}
-
-task listAutoSenseCandidates(type: SnippetsTask) {
-  group 'Docs'
-  description 'List snippets that probably should be marked // CONSOLE'
-  perSnippet {
-    if (
-             it.autoSense    // Already marked, nothing to do
-          || it.testResponse // Only commands are autosense
-        ) {
-      return
-    }
-    List<String> languages = [
-      'js', 'json', // These languages should almost always be marked autosense
-      'sh', 'shell', // These are often curl commands that should be converted
-    ]
-    if (false == languages.contains(it.language)) {
-      return
-    }
-    println(it)
   }
 }
 
-task buildRestTests(type: RestTestsFromSnippetsTask) {
+buildRestTests.docs = fileTree(projectDir) {
-  docs = fileTree(project.projectDir) {
+  // No snippets in here!
-    // No snippets in here!
+  exclude 'build.gradle'
-    exclude 'build.gradle'
+  // That is where the snippets go, not where they come from!
-    // Remove plugins because they aren't installed during this test. Yet?
+  exclude 'build'
-    exclude 'plugins'
+  // Remove plugins because they aren't installed during this test. Yet?
-    // This file simply doesn't pass yet. We should figure out how to fix it.
+  exclude 'plugins'
-    exclude 'reference/modules/snapshots.asciidoc'
+  // This file simply doesn't pass yet. We should figure out how to fix it.
-  }
+  exclude 'reference/modules/snapshots.asciidoc'
-  Closure setupTwitter = { String name, int count ->
+}
-    setups[name] = '''
+
+Closure setupTwitter = { String name, int count ->
+  buildRestTests.setups[name] = '''
   - do:
       bulk:
         index: twitter
@@ -68,17 +45,10 @@ task buildRestTests(type: RestTestsFromSnippetsTask) {
         refresh: true
         body: |'''
     for (int i = 0; i < count; i++) {
-      setups[name] += """
+      buildRestTests.setups[name] += """
           {"index":{}}
           {"msg": "some message with the number $i", "date": $i}"""
     }
   }
-  setupTwitter('twitter', 5)
+setupTwitter('twitter', 5)
-  setupTwitter('big_twitter', 120)
+setupTwitter('big_twitter', 120)
-}
-
-integTest {
-  cluster {
-    setting 'script.inline', 'true'
-  }
-}

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -249,6 +249,10 @@ public abstract class ESRestTestCase extends ESTestCase {
         adminExecutionContext = new RestTestExecutionContext(restSpec);
     }
 
+    protected RestTestExecutionContext getAdminExecutionContext() {
+        return adminExecutionContext;
+    }
+
     private static void validateSpec(RestSpec restSpec) {
         boolean validateSpec = RandomizedTest.systemPropertyAsBoolean(REST_TESTS_VALIDATE_SPEC, true);
         if (validateSpec) {

test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java

@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.TreeMap;
 
 import org.elasticsearch.common.Nullable;
@@ -69,7 +70,7 @@ public class ResponseBodyAssertion extends Assertion {
             actual = new TreeMap<>(actual);
             expected = new TreeMap<>(expected);
             for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
-                compare(expectedEntry.getKey(), expectedEntry.getValue(), actual.remove(expectedEntry.getKey()));
+                compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue());
             }
             for (Map.Entry<String, Object> unmatchedEntry : actual.entrySet()) {
                 field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]");
@@ -80,6 +81,7 @@ public class ResponseBodyAssertion extends Assertion {
             int i = 0;
             while (i < actual.size() && i < expected.size()) {
                 compare(i, actual.get(i), expected.get(i));
+                i++;
             }
             if (actual.size() == expected.size()) {
                 return;
@@ -92,7 +94,7 @@ public class ResponseBodyAssertion extends Assertion {
             message.append("received [").append(actual.size() - i).append("] more entries than expected\n");
         }
 
-        private void compare(Object field, Object expected, @Nullable Object actual) {
+        private void compare(Object field, @Nullable Object actual, Object expected) {
             if (expected instanceof Map) {
                 if (actual == null) {
                     field(field, "expected map but not found");
@@ -108,10 +110,11 @@ public class ResponseBodyAssertion extends Assertion {
                 Map<String, Object> actualMap = (Map<String, Object>) actual;
                 if (expectedMap.isEmpty() && actualMap.isEmpty()) {
                     field(field, "same [empty map]");
+                    return;
                 }
                 field(field, null);
                 indent += 1;
-                compareMaps(expectedMap, actualMap);
+                compareMaps(actualMap, expectedMap);
                 indent -= 1;
                 return;
             }
@@ -134,7 +137,7 @@ public class ResponseBodyAssertion extends Assertion {
                 }
                 field(field, null);
                 indent += 1;
-                compareLists(expectedList, actualList);
+                compareLists(actualList, expectedList);
                 indent -= 1;
                 return;
             }
@@ -142,7 +145,7 @@ public class ResponseBodyAssertion extends Assertion {
                 field(field, "expected [" + expected + "] but not found");
                 return;
             }
-            if (expected.equals(actual)) {
+            if (Objects.equals(expected, actual)) {
                 field(field, "same [" + expected + "]");
                 return;
             }