diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index c9f5668a326..ba7311fee6f 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -127,7 +127,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 current.println(setup)
             }
 
-            body(test)
+            body(test, false)
         }
 
         private void response(Snippet response) {
@@ -136,7 +136,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         }
 
         void emitDo(String method, String pathAndQuery,
-                String body, String catchPart) {
+                String body, String catchPart, boolean inSetup) {
             def (String path, String query) = pathAndQuery.tokenize('?')
             current.println("  - do:")
             if (catchPart != null) {
@@ -160,6 +160,19 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 current.println("        body: |")
                 body.eachLine { current.println("          $it") }
             }
+            /* Catch any shard failures. These only cause a non-200 response if
+             * no shard succeeds. But we need to fail the tests on all of these
+             * because they mean invalid syntax or broken queries or something
+             * else that we don't want to teach people to do. The REST test
+             * framework doesn't allow us to has assertions in the setup
+             * section so we have to skip it there. We also have to skip _cat
+             * actions because they don't return json so we can't is_false
+             * them. That is ok because they don't have this
+             * partial-success-is-success thing.
+             */
+            if (false == inSetup && false == path.startsWith('_cat')) {
+                current.println("  - is_false: _shards.failures")
+            }
         }
 
         private void setup(Snippet setup) {
@@ -169,7 +182,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             setupCurrent(setup)
             current.println('---')
             current.println("setup:")
-            body(setup)
+            body(setup, true)
             // always wait for yellow before anything is executed
             current.println(
                     "  - do:\n" +
@@ -179,7 +192,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                     "        wait_for_status: \"yellow\"")
         }
 
-        private void body(Snippet snippet) {
+        private void body(Snippet snippet, boolean inSetup) {
             parse("$snippet", snippet.contents, SYNTAX) { matcher, last ->
                 if (matcher.group("comment") != null) {
                     // Comment
@@ -193,7 +206,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                     // Leading '/'s break the generated paths
                     pathAndQuery = pathAndQuery.substring(1)
                 }
-                emitDo(method, pathAndQuery, body, catchPart)
+                emitDo(method, pathAndQuery, body, catchPart, inSetup)
             }
         }
 
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 20b301bda31..2c753e09039 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -133,7 +133,7 @@ GET hockey/_search
       "order": "asc",
       "script": {
         "lang": "painless",
-        "inline": "doc['first'].value + ' ' + doc['last'].value"
+        "inline": "doc['first.keyword'].value + ' ' + doc['last.keyword'].value"
       }
     }
   }