Merge branch 'master' into zen2

Vagrantfile

@@ -115,11 +115,6 @@ Vagrant.configure(2) do |config|
   'opensuse-42'.tap do |box|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/opensuse-42-x86_64'
-
-      # https://github.com/elastic/elasticsearch/issues/30295
-      config.vm.provider 'virtualbox' do |vbox|
-        vbox.customize ['storagectl', :id, '--name', 'SATA Controller', '--hostiocache', 'on']
-      end
       suse_common config, box
     end
   end

build.gradle

@@ -87,8 +87,15 @@ subprojects {
           }
         }
       }
+      repositories {
+        maven {
+          name = 'localTest'
+          url = "${rootProject.buildDir}/local-test-repo"
+        }
+      }
     }
   }
+
   plugins.withType(BuildPlugin).whenPluginAdded {
     project.licenseFile = project.rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
     project.noticeFile = project.rootProject.file('NOTICE.txt')
@@ -228,6 +235,7 @@ subprojects {
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
     "org.elasticsearch.client:test:${version}": ':client:test',
     "org.elasticsearch.client:transport:${version}": ':client:transport',
+    "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip',

buildSrc/build.gradle

@@ -162,11 +162,24 @@ if (project != rootProject) {
   // it's fine as we run them as part of :buildSrc
   test.enabled = false
   task integTest(type: Test) {
+    // integration test requires the local testing repo for example plugin builds
+    dependsOn project.rootProject.allprojects.collect {
+      it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'}
+    }
     exclude "**/*Tests.class"
     include "**/*IT.class"
     testClassesDirs = sourceSets.test.output.classesDirs
     classpath = sourceSets.test.runtimeClasspath
     inputs.dir(file("src/testKit"))
+    // tell BuildExamplePluginsIT where to find the example plugins
+    systemProperty (
+            'test.build-tools.plugin.examples',
+            files(
+                    project(':example-plugins').subprojects.collect { it.projectDir }
+            ).asPath,
+    )
+    systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo"
+    systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1]
   }
   check.dependsOn(integTest)
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -87,6 +87,8 @@ class BuildPlugin implements Plugin<Project> {
         project.pluginManager.apply('nebula.info-scm')
         project.pluginManager.apply('nebula.info-jar')
 
+        project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
+
         globalBuildInfo(project)
         configureRepositories(project)
         configureConfigurations(project)
@@ -101,6 +103,7 @@ class BuildPlugin implements Plugin<Project> {
         configureDependenciesInfo(project)
     }
 
+
     /** Performs checks on the build environment and prints information about the build environment. */
     static void globalBuildInfo(Project project) {
         if (project.rootProject.ext.has('buildChecksDone') == false) {
@@ -116,12 +119,14 @@ class BuildPlugin implements Plugin<Project> {
 
             final Map<Integer, String> javaVersions = [:]
             for (int version = 7; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) {
-                javaVersions.put(version, findJavaHome(version));
+                if(System.getenv(getJavaHomeEnvVarName(version.toString())) != null) {
+                    javaVersions.put(version, findJavaHome(version.toString()));
+                }
             }
 
             String javaVendor = System.getProperty('java.vendor')
-            String javaVersion = System.getProperty('java.version')
-            String gradleJavaVersionDetails = "${javaVendor} ${javaVersion}" +
+            String gradleJavaVersion = System.getProperty('java.version')
+            String gradleJavaVersionDetails = "${javaVendor} ${gradleJavaVersion}" +
                 " [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]"
 
             String compilerJavaVersionDetails = gradleJavaVersionDetails
@@ -144,33 +149,33 @@ class BuildPlugin implements Plugin<Project> {
             // Build debugging info
             println '======================================='
             println 'Elasticsearch Build Hamster says Hello!'
-            println '======================================='
             println "  Gradle Version        : ${project.gradle.gradleVersion}"
             println "  OS Info               : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})"
             if (gradleJavaVersionDetails != compilerJavaVersionDetails || gradleJavaVersionDetails != runtimeJavaVersionDetails) {
-                println "  JDK Version (gradle)  : ${gradleJavaVersionDetails}"
-                println "  JAVA_HOME (gradle)    : ${gradleJavaHome}"
-                println "  JDK Version (compile) : ${compilerJavaVersionDetails}"
-                println "  JAVA_HOME (compile)   : ${compilerJavaHome}"
-                println "  JDK Version (runtime) : ${runtimeJavaVersionDetails}"
-                println "  JAVA_HOME (runtime)   : ${runtimeJavaHome}"
+                println "  Compiler JDK Version  : ${getPaddedMajorVersion(compilerJavaVersionEnum)} (${compilerJavaVersionDetails})"
+                println "  Compiler java.home    : ${compilerJavaHome}"
+                println "  Runtime JDK Version   : ${getPaddedMajorVersion(runtimeJavaVersionEnum)} (${runtimeJavaVersionDetails})"
+                println "  Runtime java.home     : ${runtimeJavaHome}"
+                println "  Gradle JDK Version    : ${getPaddedMajorVersion(JavaVersion.toVersion(gradleJavaVersion))} (${gradleJavaVersionDetails})"
+                println "  Gradle java.home      : ${gradleJavaHome}"
             } else {
-                println "  JDK Version           : ${gradleJavaVersionDetails}"
+                println "  JDK Version           : ${getPaddedMajorVersion(JavaVersion.toVersion(gradleJavaVersion))} (${gradleJavaVersionDetails})"
                 println "  JAVA_HOME             : ${gradleJavaHome}"
             }
             println "  Random Testing Seed   : ${project.testSeed}"
+            println '======================================='
 
             // enforce Java version
             if (compilerJavaVersionEnum < minimumCompilerVersion) {
                 final String message =
-                        "the environment variable JAVA_HOME must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
+                        "the compiler java.home must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
                                 " but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]"
                 throw new GradleException(message)
             }
 
             if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
                 final String message =
-                        "the environment variable RUNTIME_JAVA_HOME must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
+                        "the runtime java.home must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
                                 " but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]"
                 throw new GradleException(message)
             }
@@ -205,6 +210,7 @@ class BuildPlugin implements Plugin<Project> {
             project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
             project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
             project.rootProject.ext.inFipsJvm = inFipsJvm
+            project.rootProject.ext.gradleJavaVersion = JavaVersion.toVersion(gradleJavaVersion)
         }
 
         project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
@@ -217,11 +223,20 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
         project.ext.javaVersions = project.rootProject.ext.javaVersions
         project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
+        project.ext.gradleJavaVersion = project.rootProject.ext.gradleJavaVersion
+    }
+
+    private static String getPaddedMajorVersion(JavaVersion compilerJavaVersionEnum) {
+        compilerJavaVersionEnum.getMajorVersion().toString().padLeft(2)
     }
 
     private static String findCompilerJavaHome() {
-        final String javaHome = System.getenv('JAVA_HOME')
-        if (javaHome == null) {
+        final String compilerJavaHome = System.getenv('JAVA_HOME')
+        final String compilerJavaProperty = System.getProperty('compiler.java')
+        if (compilerJavaProperty != null) {
+            compilerJavaHome = findJavaHome(compilerJavaProperty)
+        }
+        if (compilerJavaHome == null) {
             if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
                 // IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
                 return Jvm.current().javaHome
@@ -233,11 +248,24 @@ class BuildPlugin implements Plugin<Project> {
                 )
             }
         }
-        return javaHome
+        return compilerJavaHome
     }
 
-    private static String findJavaHome(int version) {
-        return System.getenv('JAVA' + version + '_HOME')
+    private static String findJavaHome(String version) {
+        String versionedVarName = getJavaHomeEnvVarName(version)
+        String versionedJavaHome = System.getenv(versionedVarName);
+        if (versionedJavaHome == null) {
+            throw new GradleException(
+                    "$versionedVarName must be set to build Elasticsearch. " +
+                            "Note that if the variable was just set you might have to run `./gradlew --stop` for " +
+                            "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
+            )
+        }
+        return versionedJavaHome
+    }
+
+    private static String getJavaHomeEnvVarName(String version) {
+        return 'JAVA' + version + '_HOME'
     }
 
     /** Add a check before gradle execution phase which ensures java home for the given java version is set. */
@@ -271,7 +299,10 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     private static String findRuntimeJavaHome(final String compilerJavaHome) {
-        assert compilerJavaHome != null
+        String runtimeJavaProperty = System.getProperty("runtime.java")
+        if (runtimeJavaProperty != null) {
+            return findJavaHome(runtimeJavaProperty)
+        }
         return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome
     }
 
@@ -405,6 +436,10 @@ class BuildPlugin implements Plugin<Project> {
             repos.mavenLocal()
         }
         repos.mavenCentral()
+        repos.maven {
+            name "elastic"
+            url "https://artifacts.elastic.co/maven"
+        }
         String luceneVersion = VersionProperties.lucene
         if (luceneVersion.contains('-snapshot')) {
             // extract the revision number from the version with a regex matcher
@@ -519,7 +554,7 @@ class BuildPlugin implements Plugin<Project> {
                 project.publishing {
                     publications {
                         nebula(MavenPublication) {
-                            artifact project.tasks.shadowJar
+                            artifacts = [ project.tasks.shadowJar ]
                             artifactId = project.archivesBaseName
                             /*
                             * Configure the pom to include the "shadow" as compile dependencies
@@ -549,7 +584,6 @@ class BuildPlugin implements Plugin<Project> {
                 }
             }
         }
-
     }
 
     /** Adds compiler settings to the project */
@@ -764,6 +798,14 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
             systemProperty 'jna.nosys', 'true'
+            // TODO: remove this deprecation compatibility setting for 7.0
+            systemProperty 'es.aggregations.enable_scripted_metric_agg_param', 'false'
+            systemProperty 'compiler.java', project.ext.compilerJavaVersion.getMajorVersion()
+            if (project.ext.inFipsJvm) {
+                systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion() + "FIPS"
+            } else {
+                systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion()
+            }
             // TODO: remove setting logging level via system property
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {
@@ -777,11 +819,19 @@ class BuildPlugin implements Plugin<Project> {
                     systemProperty property.getKey(), property.getValue()
                 }
             }
+
+            // TODO: remove this once joda time is removed from scripting in 7.0
+            systemProperty 'es.scripting.use_java_time', 'true'
+
+            // TODO: remove this once ctx isn't added to update script params in 7.0
+            systemProperty 'es.scripting.update.ctx_in_params', 'false'
+
             // Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
             if (project.inFipsJvm) {
                 systemProperty 'javax.net.ssl.trustStorePassword', 'password'
                 systemProperty 'javax.net.ssl.keyStorePassword', 'password'
             }
+
             boolean assertionsEnabled = Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))
             enableSystemAssertions assertionsEnabled
             enableAssertions assertionsEnabled

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -284,6 +284,10 @@ public class SnippetsTask extends DefaultTask {
                     contents.append(line).append('\n')
                     return
                 }
+                // Allow line continuations for console snippets within lists
+                if (snippet != null && line.trim() == '+') {
+                    return
+                }
                 // Just finished
                 emit()
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -25,7 +25,6 @@ import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
 import org.gradle.api.InvalidUserDataException
-import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.XmlProvider
@@ -39,7 +38,6 @@ import java.nio.file.Path
 import java.nio.file.StandardCopyOption
 import java.util.regex.Matcher
 import java.util.regex.Pattern
-
 /**
  * Encapsulates build configuration for an Elasticsearch plugin.
  */

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy

@@ -20,6 +20,7 @@ package org.elasticsearch.gradle.plugin
 
 import org.gradle.api.Project
 import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFile
 
 /**
  * A container for plugin properties that will be written to the plugin descriptor, for easy
@@ -55,18 +56,39 @@ class PluginPropertiesExtension {
     boolean requiresKeystore = false
 
     /** A license file that should be included in the built plugin zip. */
-    @Input
-    File licenseFile = null
+    private File licenseFile = null
 
     /**
      * A notice file that should be included in the built plugin zip. This will be
      * extended with notices from the {@code licenses/} directory.
      */
-    @Input
-    File noticeFile = null
+    private File noticeFile = null
+
+    Project project = null
 
     PluginPropertiesExtension(Project project) {
         name = project.name
         version = project.version
+        this.project = project
+    }
+
+    @InputFile
+    File getLicenseFile() {
+        return licenseFile
+    }
+
+    void setLicenseFile(File licenseFile) {
+        project.ext.licenseFile = licenseFile
+        this.licenseFile = licenseFile
+    }
+
+    @InputFile
+    File getNoticeFile() {
+        return noticeFile
+    }
+
+    void setNoticeFile(File noticeFile) {
+        project.ext.noticeFile = noticeFile
+        this.noticeFile = noticeFile
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -23,7 +23,6 @@ import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Task
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.OutputFile
-
 /**
  * Creates a plugin descriptor.
  */

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -20,10 +20,15 @@ package org.elasticsearch.gradle.precommit
 
 import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
+import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
+import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.file.FileCollection
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.plugins.quality.Checkstyle
+import org.gradle.api.tasks.JavaExec
+import org.gradle.api.tasks.StopExecutionException
 
 /**
  * Validation tasks which should be run before committing. These run before tests.
@@ -40,7 +45,11 @@ class PrecommitTasks {
             project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
             project.tasks.create('filepermissions', FilePermissionsTask.class),
             project.tasks.create('jarHell', JarHellTask.class),
-            project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)]
+            project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)
+        ]
+
+        // Configure it but don't add it as a dependency yet
+        configureForbiddenApisCli(project)
 
         // tasks with just tests don't need dependency licenses, so this flag makes adding
         // the task optional
@@ -93,12 +102,66 @@ class PrecommitTasks {
                 signaturesURLs = project.forbiddenApis.signaturesURLs +
                     [ getClass().getResource('/forbidden/es-server-signatures.txt') ]
             }
+            // forbidden apis doesn't support Java 11, so stop at 10
+            String targetMajorVersion = (project.compilerJavaVersion.compareTo(JavaVersion.VERSION_1_10) > 0 ?
+                    JavaVersion.VERSION_1_10 :
+                    project.compilerJavaVersion).getMajorVersion()
+            targetCompatibility = Integer.parseInt(targetMajorVersion) >= 9 ?targetMajorVersion : "1.${targetMajorVersion}"
         }
         Task forbiddenApis = project.tasks.findByName('forbiddenApis')
         forbiddenApis.group = "" // clear group, so this does not show up under verification tasks
+
         return forbiddenApis
     }
 
+    private static Task configureForbiddenApisCli(Project project) {
+        project.configurations.create("forbiddenApisCliJar")
+        project.dependencies {
+            forbiddenApisCliJar 'de.thetaphi:forbiddenapis:2.5'
+        }
+        Task forbiddenApisCli = project.tasks.create('forbiddenApisCli')
+
+        project.sourceSets.forEach { sourceSet ->
+            forbiddenApisCli.dependsOn(
+                project.tasks.create(sourceSet.getTaskName('forbiddenApisCli', null), JavaExec) {
+                    ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
+                    dependsOn(buildResources)
+                    classpath = project.files(
+                            project.configurations.forbiddenApisCliJar,
+                            sourceSet.compileClasspath,
+                            sourceSet.runtimeClasspath
+                    )
+                    main = 'de.thetaphi.forbiddenapis.cli.CliMain'
+                    executable = "${project.runtimeJavaHome}/bin/java"
+                    args "-b", 'jdk-unsafe-1.8'
+                    args "-b", 'jdk-deprecated-1.8'
+                    args "-b", 'jdk-non-portable'
+                    args "-b", 'jdk-system-out'
+                    args "-f", buildResources.copy("forbidden/jdk-signatures.txt")
+                    args "-f", buildResources.copy("forbidden/es-all-signatures.txt")
+                    args "--suppressannotation", '**.SuppressForbidden'
+                    if (sourceSet.name == 'test') {
+                        args "-f", buildResources.copy("forbidden/es-test-signatures.txt")
+                        args "-f", buildResources.copy("forbidden/http-signatures.txt")
+                    } else {
+                        args "-f", buildResources.copy("forbidden/es-server-signatures.txt")
+                    }
+                    dependsOn sourceSet.classesTaskName
+                    doFirst {
+                        // Forbidden APIs expects only existing dirs, and requires at least one
+                        FileCollection existingOutputs = sourceSet.output.classesDirs
+                                .filter { it.exists() }
+                        if (existingOutputs.isEmpty()) {
+                            throw new StopExecutionException("${sourceSet.name} has no outputs")
+                        }
+                        existingOutputs.forEach { args "-d", it }
+                    }
+                }
+            )
+        }
+        return forbiddenApisCli
+    }
+
     private static Task configureCheckstyle(Project project) {
         // Always copy the checkstyle configuration files to 'buildDir/checkstyle' since the resources could be located in a jar
         // file. If the resources are located in a jar, Gradle will fail when it tries to turn the URL into a file

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -177,6 +177,12 @@ class NodeInfo {
             javaVersion = 8
         } else if (nodeVersion.onOrAfter("6.2.0") && nodeVersion.before("6.3.0")) {
             javaVersion = 9
+        } else if (project.inFipsJvm && nodeVersion.onOrAfter("6.3.0") && nodeVersion.before("6.4.0")) {
+            /*
+             * Elasticsearch versions before 6.4.0 cannot be run in a FIPS-140 JVM. If we're running
+             * bwc tests in a FIPS-140 JVM, ensure that the pre v6.4.0 nodes use a Java 10 JVM instead.
+             */
+            javaVersion = 10
         }
 
         args.addAll("-E", "node.portsfile=true")

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -31,6 +31,7 @@ import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskState
+import org.gradle.plugins.ide.idea.IdeaPlugin
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
@@ -243,10 +244,12 @@ public class RestIntegTestTask extends DefaultTask {
                 }
             }
         }
-        project.idea {
-            module {
-                if (scopes.TEST != null) {
-                    scopes.TEST.plus.add(project.configurations.restSpec)
+        if (project.plugins.hasPlugin(IdeaPlugin)) {
+            project.idea {
+                module {
+                    if (scopes.TEST != null) {
+                        scopes.TEST.plus.add(project.configurations.restSpec)
+                    }
                 }
             }
         }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -22,15 +22,14 @@ package org.elasticsearch.gradle.test
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
 import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
-import org.gradle.api.Task
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.tasks.compile.JavaCompile
-
 /**
  * Configures the build to compile tests against Elasticsearch's test framework
  * and run REST tests. Use BuildPlugin if you want to build main code as well
@@ -48,6 +47,7 @@ public class StandaloneRestTestPlugin implements Plugin<Project> {
         project.pluginManager.apply(JavaBasePlugin)
         project.pluginManager.apply(RandomizedTestingPlugin)
 
+        project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
         BuildPlugin.globalBuildInfo(project)
         BuildPlugin.configureRepositories(project)
 

buildSrc/src/main/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTask.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.file.DirectoryProperty;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.SkipWhenEmpty;
+import org.gradle.api.tasks.StopExecutionException;
+import org.gradle.api.tasks.TaskAction;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Export Elasticsearch build resources to configurable paths
+ * <p>
+ * Wil overwrite existing files and create missing directories.
+ * Useful for resources that that need to be passed to other processes trough the filesystem or otherwise can't be
+ * consumed from the classpath.
+ */
+public class ExportElasticsearchBuildResourcesTask extends DefaultTask {
+
+    private final Logger logger = Logging.getLogger(ExportElasticsearchBuildResourcesTask.class);
+
+    private final Set<String> resources = new HashSet<>();
+
+    private DirectoryProperty outputDir;
+
+    public ExportElasticsearchBuildResourcesTask() {
+        outputDir = getProject().getLayout().directoryProperty(
+            getProject().getLayout().getBuildDirectory().dir("build-tools-exported")
+        );
+    }
+
+    @OutputDirectory
+    public DirectoryProperty getOutputDir() {
+        return outputDir;
+    }
+
+    @Input
+    @SkipWhenEmpty
+    public Set<String> getResources() {
+        return Collections.unmodifiableSet(resources);
+    }
+
+    @Classpath
+    public String getResourcesClasspath() {
+        // This will make sure the task is not considered up to date if the resources are changed.
+        logger.info("Classpath: {}", System.getProperty("java.class.path"));
+        return System.getProperty("java.class.path");
+    }
+
+    public void setOutputDir(DirectoryProperty outputDir) {
+        this.outputDir = outputDir;
+    }
+
+    public File copy(String resource) {
+        if (getState().getExecuted() || getState().getExecuting()) {
+            throw new GradleException("buildResources can't be configured after the task ran. " +
+                "Make sure task is not used after configuration time"
+            );
+        }
+        resources.add(resource);
+        return outputDir.file(resource).get().getAsFile();
+    }
+
+    @TaskAction
+    public void doExport() {
+        if (resources.isEmpty()) {
+            throw new StopExecutionException();
+        }
+        resources.stream().parallel()
+            .forEach(resourcePath -> {
+                Path destination = outputDir.get().file(resourcePath).getAsFile().toPath();
+                try (InputStream is = getClass().getClassLoader().getResourceAsStream(resourcePath)) {
+                    Files.createDirectories(destination.getParent());
+                    if (is == null) {
+                        throw new GradleException("Can't export `" + resourcePath + "` from build-tools: not found");
+                    }
+                    Files.copy(is, destination);
+                } catch (IOException e) {
+                    throw new GradleException("Can't write resource `" + resourcePath + "` to " + destination, e);
+                }
+            });
+    }
+
+}

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -686,6 +686,7 @@
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionScriptEngine.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]MoreExpressionTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]StoredExpressionTests.java" checks="LineLength" />
+  <suppress files="modules[/\\]lang-painless[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]painless[/\\]ContextExampleTests.java" checks="LineLength" />
   <suppress files="modules[/\\]reindex[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]reindex[/\\]TransportUpdateByQueryAction.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuCollationTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuFoldingTokenFilterFactory.java" checks="LineLength" />

buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java

@@ -0,0 +1,164 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.apache.commons.io.FileUtils;
+import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
+import org.gradle.testkit.runner.GradleRunner;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class BuildExamplePluginsIT extends GradleIntegrationTestCase {
+
+    private static List<File> EXAMPLE_PLUGINS = Collections.unmodifiableList(
+        Arrays.stream(
+            Objects.requireNonNull(System.getProperty("test.build-tools.plugin.examples"))
+                .split(File.pathSeparator)
+        ).map(File::new).collect(Collectors.toList())
+    );
+
+    @Rule
+    public TemporaryFolder tmpDir = new TemporaryFolder();
+
+    public final File examplePlugin;
+
+    public BuildExamplePluginsIT(File examplePlugin) {
+        this.examplePlugin = examplePlugin;
+    }
+
+    @BeforeClass
+    public static void assertProjectsExist() {
+        assertEquals(
+            EXAMPLE_PLUGINS,
+            EXAMPLE_PLUGINS.stream().filter(File::exists).collect(Collectors.toList())
+        );
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        return EXAMPLE_PLUGINS
+            .stream()
+            .map(each -> new Object[] {each})
+            .collect(Collectors.toList());
+    }
+
+    public void testCurrentExamplePlugin() throws IOException {
+        FileUtils.copyDirectory(examplePlugin, tmpDir.getRoot());
+        // just get rid of deprecation warnings
+        Files.write(
+            getTempPath("settings.gradle"),
+            "enableFeaturePreview('STABLE_PUBLISHING')\n".getBytes(StandardCharsets.UTF_8)
+        );
+
+        adaptBuildScriptForTest();
+
+        Files.write(
+            tmpDir.newFile("NOTICE.txt").toPath(),
+            "dummy test notice".getBytes(StandardCharsets.UTF_8)
+        );
+
+        GradleRunner.create()
+            .withProjectDir(tmpDir.getRoot())
+            .withArguments("clean", "check", "-s", "-i", "--warning-mode=all", "--scan")
+            .withPluginClasspath()
+            .build();
+    }
+
+    private void adaptBuildScriptForTest() throws IOException {
+        // Add the local repo as a build script URL so we can pull in build-tools and apply the plugin under test
+        // + is ok because we have no other repo and just want to pick up latest
+        writeBuildScript(
+            "buildscript {\n" +
+                "    repositories {\n" +
+                "        maven {\n" +
+                "            url = '" + getLocalTestRepoPath() + "'\n" +
+                "        }\n" +
+                "    }\n" +
+                "    dependencies {\n" +
+                "        classpath \"org.elasticsearch.gradle:build-tools:+\"\n" +
+                "    }\n" +
+                "}\n"
+        );
+        // get the original file
+        Files.readAllLines(getTempPath("build.gradle"), StandardCharsets.UTF_8)
+            .stream()
+            .map(line -> line + "\n")
+            .forEach(this::writeBuildScript);
+        // Add a repositories section to be able to resolve dependencies
+        String luceneSnapshotRepo = "";
+        String luceneSnapshotRevision = System.getProperty("test.lucene-snapshot-revision");
+        if (luceneSnapshotRepo != null) {
+            luceneSnapshotRepo =  "  maven {\n" +
+                "    url \"http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + luceneSnapshotRevision + "\"\n" +
+                "  }\n";
+        }
+        writeBuildScript("\n" +
+                "repositories {\n" +
+                "  maven {\n" +
+                "    url \"" + getLocalTestRepoPath()  + "\"\n" +
+                "  }\n" +
+                luceneSnapshotRepo +
+                "}\n"
+        );
+        Files.delete(getTempPath("build.gradle"));
+        Files.move(getTempPath("build.gradle.new"), getTempPath("build.gradle"));
+        System.err.print("Generated build script is:");
+        Files.readAllLines(getTempPath("build.gradle")).forEach(System.err::println);
+    }
+
+    private Path getTempPath(String fileName) {
+        return new File(tmpDir.getRoot(), fileName).toPath();
+    }
+
+    private Path writeBuildScript(String script) {
+        try {
+            Path path = getTempPath("build.gradle.new");
+            return Files.write(
+                path,
+                script.getBytes(StandardCharsets.UTF_8),
+                Files.exists(path) ? StandardOpenOption.APPEND : StandardOpenOption.CREATE_NEW
+            );
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private String getLocalTestRepoPath() {
+        String property = System.getProperty("test.local-test-repo-path");
+        Objects.requireNonNull(property, "test.local-test-repo-path not passed to tests");
+        File file = new File(property);
+        assertTrue("Expected " + property + " to exist, but it did not!", file.exists());
+        return file.getAbsolutePath();
+    }
+
+}

buildSrc/src/test/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java

@@ -0,0 +1,91 @@
+package org.elasticsearch.gradle;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+
+
+public class ExportElasticsearchBuildResourcesTaskIT extends GradleIntegrationTestCase {
+
+    public static final String PROJECT_NAME = "elasticsearch-build-resources";
+
+    public void testUpToDateWithSourcesConfigured() {
+        GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("clean", "-s")
+            .withPluginClasspath()
+            .build();
+
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("buildResources", "-s", "-i")
+            .withPluginClasspath()
+            .build();
+        assertTaskSuccessfull(result, ":buildResources");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
+
+        result = GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("buildResources", "-s", "-i")
+            .withPluginClasspath()
+            .build();
+        assertTaskUpToDate(result, ":buildResources");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
+    }
+
+    public void testImplicitTaskDependencyCopy() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("clean", "sampleCopyAll", "-s", "-i")
+            .withPluginClasspath()
+            .build();
+
+        assertTaskSuccessfull(result, ":buildResources");
+        assertTaskSuccessfull(result, ":sampleCopyAll");
+        assertBuildFileExists(result, PROJECT_NAME, "sampleCopyAll/checkstyle.xml");
+        // This is a side effect of compile time reference
+        assertBuildFileExists(result, PROJECT_NAME, "sampleCopyAll/checkstyle_suppressions.xml");
+    }
+
+    public void testImplicitTaskDependencyInputFileOfOther() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("clean", "sample", "-s", "-i")
+            .withPluginClasspath()
+            .build();
+
+        assertTaskSuccessfull(result, ":sample");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
+    }
+
+    public void testIncorrectUsage() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir(PROJECT_NAME))
+            .withArguments("noConfigAfterExecution", "-s", "-i")
+            .withPluginClasspath()
+            .buildAndFail();
+        assertOutputContains("buildResources can't be configured after the task ran");
+    }
+}

buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java

@@ -1,8 +1,13 @@
 package org.elasticsearch.gradle.test;
 
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.BuildTask;
 import org.gradle.testkit.runner.GradleRunner;
+import org.gradle.testkit.runner.TaskOutcome;
 
 import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -15,7 +20,7 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
             throw new RuntimeException("Could not find resources dir for integration tests. " +
                 "Note that these tests can only be ran by Gradle and are not currently supported by the IDE");
         }
-        return new File(root, name);
+        return new File(root, name).getAbsoluteFile();
     }
 
     protected GradleRunner getGradleRunner(String sampleProject) {
@@ -61,4 +66,47 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
         }
     }
 
+    protected void assertTaskSuccessfull(BuildResult result, String taskName) {
+        BuildTask task = result.task(taskName);
+        if (task == null) {
+            fail("Expected task `" + taskName + "` to be successful, but it did not run");
+        }
+        assertEquals(
+            "Expected task to be successful but it was: " + task.getOutcome() +
+                "\n\nOutput is:\n" + result.getOutput() ,
+            TaskOutcome.SUCCESS,
+            task.getOutcome()
+        );
+    }
+
+    protected void assertTaskUpToDate(BuildResult result, String taskName) {
+        BuildTask task = result.task(taskName);
+        if (task == null) {
+            fail("Expected task `" + taskName + "` to be up-to-date, but it did not run");
+        }
+        assertEquals(
+            "Expected task to be up to date but it was: " + task.getOutcome() +
+                "\n\nOutput is:\n" + result.getOutput() ,
+            TaskOutcome.UP_TO_DATE,
+            task.getOutcome()
+        );
+    }
+
+    protected void assertBuildFileExists(BuildResult result, String projectName, String path) {
+        Path absPath = getBuildDir(projectName).toPath().resolve(path);
+        assertTrue(
+            result.getOutput() + "\n\nExpected `" + absPath + "` to exists but it did not" +
+                "\n\nOutput is:\n" + result.getOutput(),
+            Files.exists(absPath)
+        );
+    }
+
+    protected void assertBuildFileDoesNotExists(BuildResult result, String projectName, String path) {
+        Path absPath = getBuildDir(projectName).toPath().resolve(path);
+        assertFalse(
+            result.getOutput() + "\n\nExpected `" + absPath + "` bo to exists but it did" +
+                "\n\nOutput is:\n" + result.getOutput(),
+            Files.exists(absPath)
+        );
+    }
 }

buildSrc/src/testKit/elasticsearch-build-resources/build.gradle

@@ -0,0 +1,38 @@
+plugins {
+    id 'elasticsearch.build'
+}
+
+ext.licenseFile = file("$buildDir/dummy/license")
+ext.noticeFile = file("$buildDir/dummy/notice")
+
+buildResources {
+    copy 'checkstyle.xml'
+}
+
+task sampleCopyAll(type: Sync) {
+    /** Note: no explicit dependency. This works with tasks that use the Provider API a.k.a "Lazy Configuration" **/
+    from buildResources
+    into "$buildDir/sampleCopyAll"
+}
+
+task sample {
+    // This does not work, task dependencies can't be providers
+    // dependsOn buildResources.resource('minimumRuntimeVersion')
+    // Nor does this, despite https://github.com/gradle/gradle/issues/3811
+    // dependsOn buildResources.outputDir
+    // for now it's just
+    dependsOn buildResources
+    // we have to refference it at configuration time in order to be picked up
+    ext.checkstyle_suppressions = buildResources.copy('checkstyle_suppressions.xml')
+    doLast {
+        println "This task is using ${file(checkstyle_suppressions)}"
+    }
+}
+
+task noConfigAfterExecution {
+    dependsOn buildResources
+    doLast {
+        println "This should cause an error because we are refferencing " +
+                "${buildResources.copy('checkstyle_suppressions.xml')} after the `buildResources` task has ran."
+    }
+}

buildSrc/version.properties

@@ -1,13 +1,15 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.5.0-snapshot-608f0277b0
+lucene            = 7.5.0-snapshot-13b9e28f9d
 
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.8.10
+jackson           = 2.8.11
 snakeyaml         = 1.17
+icu4j             = 62.1
+supercsv          = 2.4.0
 # when updating log4j, please update also docs/java-api/index.asciidoc
-log4j             = 2.9.1
+log4j             = 2.11.1
 slf4j             = 1.6.2
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle

client/rest-high-level/build.gradle

@@ -30,6 +30,14 @@ apply plugin: 'com.github.johnrengelman.shadow'
 group = 'org.elasticsearch.client'
 archivesBaseName = 'elasticsearch-rest-high-level-client'
 
+publishing {
+  publications {
+    nebula {
+      artifactId = archivesBaseName
+    }
+  }
+}
+
 //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions)
 Task copyRestSpec = RestIntegTestTask.createCopyRestSpecTask(project, Providers.FALSE)
 test.dependsOn(copyRestSpec)

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -21,18 +21,15 @@ package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
-import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest;
@@ -45,7 +42,6 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRespon
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -55,15 +51,14 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
-import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeResponse;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -93,9 +88,9 @@ public final class IndicesClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public DeleteIndexResponse delete(DeleteIndexRequest deleteIndexRequest, RequestOptions options) throws IOException {
+    public AcknowledgedResponse delete(DeleteIndexRequest deleteIndexRequest, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(deleteIndexRequest, RequestConverters::deleteIndex, options,
-                DeleteIndexResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -106,9 +101,9 @@ public final class IndicesClient {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void deleteAsync(DeleteIndexRequest deleteIndexRequest, RequestOptions options, ActionListener<DeleteIndexResponse> listener) {
+    public void deleteAsync(DeleteIndexRequest deleteIndexRequest, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(deleteIndexRequest, RequestConverters::deleteIndex, options,
-                DeleteIndexResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**
@@ -147,9 +142,9 @@ public final class IndicesClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public PutMappingResponse putMapping(PutMappingRequest putMappingRequest, RequestOptions options) throws IOException {
+    public AcknowledgedResponse putMapping(PutMappingRequest putMappingRequest, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(putMappingRequest, RequestConverters::putMapping, options,
-                PutMappingResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -160,9 +155,10 @@ public final class IndicesClient {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void putMappingAsync(PutMappingRequest putMappingRequest, RequestOptions options,  ActionListener<PutMappingResponse> listener) {
+    public void putMappingAsync(PutMappingRequest putMappingRequest, RequestOptions options,
+                                ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(putMappingRequest, RequestConverters::putMapping, options,
-                PutMappingResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**
@@ -231,9 +227,9 @@ public final class IndicesClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public IndicesAliasesResponse updateAliases(IndicesAliasesRequest indicesAliasesRequest, RequestOptions options) throws IOException {
+    public AcknowledgedResponse updateAliases(IndicesAliasesRequest indicesAliasesRequest, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(indicesAliasesRequest, RequestConverters::updateAliases, options,
-                IndicesAliasesResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -245,9 +241,9 @@ public final class IndicesClient {
      * @param listener the listener to be notified upon request completion
      */
     public void updateAliasesAsync(IndicesAliasesRequest indicesAliasesRequest, RequestOptions options,
-                                   ActionListener<IndicesAliasesResponse> listener) {
+                                   ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(indicesAliasesRequest, RequestConverters::updateAliases, options,
-                IndicesAliasesResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**
@@ -286,9 +282,9 @@ public final class IndicesClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public CloseIndexResponse close(CloseIndexRequest closeIndexRequest, RequestOptions options) throws IOException {
+    public AcknowledgedResponse close(CloseIndexRequest closeIndexRequest, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(closeIndexRequest, RequestConverters::closeIndex, options,
-                CloseIndexResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -299,9 +295,9 @@ public final class IndicesClient {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void closeAsync(CloseIndexRequest closeIndexRequest, RequestOptions options, ActionListener<CloseIndexResponse> listener) {
+    public void closeAsync(CloseIndexRequest closeIndexRequest, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(closeIndexRequest, RequestConverters::closeIndex, options,
-                CloseIndexResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
 
@@ -706,9 +702,9 @@ public final class IndicesClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public UpdateSettingsResponse putSettings(UpdateSettingsRequest updateSettingsRequest, RequestOptions options) throws IOException {
+    public AcknowledgedResponse putSettings(UpdateSettingsRequest updateSettingsRequest, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(updateSettingsRequest, RequestConverters::indexPutSettings, options,
-                UpdateSettingsResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -720,9 +716,9 @@ public final class IndicesClient {
      * @param listener the listener to be notified upon request completion
      */
     public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, RequestOptions options,
-                                 ActionListener<UpdateSettingsResponse> listener) {
+                                 ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(updateSettingsRequest, RequestConverters::indexPutSettings, options,
-                UpdateSettingsResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**
@@ -734,10 +730,10 @@ public final class IndicesClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public PutIndexTemplateResponse putTemplate(PutIndexTemplateRequest putIndexTemplateRequest,
-                                                RequestOptions options) throws IOException {
+    public AcknowledgedResponse putTemplate(PutIndexTemplateRequest putIndexTemplateRequest,
+                                            RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, RequestConverters::putTemplate, options,
-            PutIndexTemplateResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -749,9 +745,9 @@ public final class IndicesClient {
      * @param listener the listener to be notified upon request completion
      */
     public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, RequestOptions options,
-                                 ActionListener<PutIndexTemplateResponse> listener) {
+                                 ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, RequestConverters::putTemplate, options,
-            PutIndexTemplateResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**

client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java

@@ -26,7 +26,7 @@ import org.elasticsearch.action.ingest.GetPipelineResponse;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 
 import java.io.IOException;
 
@@ -54,9 +54,9 @@ public final class IngestClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public WritePipelineResponse putPipeline(PutPipelineRequest request, RequestOptions options) throws IOException {
+    public AcknowledgedResponse putPipeline(PutPipelineRequest request, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::putPipeline, options,
-            WritePipelineResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -67,9 +67,9 @@ public final class IngestClient {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void putPipelineAsync(PutPipelineRequest request, RequestOptions options, ActionListener<WritePipelineResponse> listener) {
+    public void putPipelineAsync(PutPipelineRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::putPipeline, options,
-            WritePipelineResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**
@@ -109,9 +109,9 @@ public final class IngestClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public WritePipelineResponse deletePipeline(DeletePipelineRequest request, RequestOptions options) throws IOException {
+    public AcknowledgedResponse deletePipeline(DeletePipelineRequest request, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::deletePipeline, options,
-            WritePipelineResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -123,9 +123,9 @@ public final class IngestClient {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void deletePipelineAsync(DeletePipelineRequest request, RequestOptions options, ActionListener<WritePipelineResponse> listener) {
+    public void deletePipelineAsync(DeletePipelineRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::deletePipeline, options,
-            WritePipelineResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**

client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseClient.java

@@ -19,11 +19,27 @@
 
 package org.elasticsearch.client;
 
+import org.apache.http.HttpEntity;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.GetLicenseResponse;
 import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 
 import static java.util.Collections.emptySet;
 
@@ -34,7 +50,7 @@ import static java.util.Collections.emptySet;
  * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html">
  * X-Pack Licensing APIs on elastic.co</a> for more information.
  */
-public class LicenseClient {
+public final class LicenseClient {
 
     private final RestHighLevelClient restHighLevelClient;
 
@@ -54,7 +70,7 @@ public class LicenseClient {
     }
 
     /**
-     * Asynchronously updates license for the cluster cluster.
+     * Asynchronously updates license for the cluster.
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
@@ -63,4 +79,79 @@ public class LicenseClient {
             PutLicenseResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Returns the current license for the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetLicenseResponse getLicense(GetLicenseRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequest(request, RequestConverters::getLicense, options,
+            response -> new GetLicenseResponse(convertResponseToJson(response)), emptySet());
+    }
+
+    /**
+     * Asynchronously returns the current license for the cluster cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getLicenseAsync(GetLicenseRequest request, RequestOptions options, ActionListener<GetLicenseResponse> listener) {
+        restHighLevelClient.performRequestAsync(request, RequestConverters::getLicense, options,
+            response -> new GetLicenseResponse(convertResponseToJson(response)), listener, emptySet());
+    }
+
+    /**
+     * Deletes license from the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse deleteLicense(DeleteLicenseRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::deleteLicense, options,
+            AcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously deletes license from the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void deleteLicenseAsync(DeleteLicenseRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::deleteLicense, options,
+            AcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Converts an entire response into a json string
+     *
+     * This is useful for responses that we don't parse on the client side, but instead work as string
+     * such as in case of the license JSON
+     */
+    static String convertResponseToJson(Response response) throws IOException {
+        HttpEntity entity = response.getEntity();
+        if (entity == null) {
+            throw new IllegalStateException("Response body expected but not returned");
+        }
+        if (entity.getContentType() == null) {
+            throw new IllegalStateException("Elasticsearch didn't return the [Content-Type] header, unable to parse response body");
+        }
+        XContentType xContentType = XContentType.fromMediaTypeOrFormat(entity.getContentType().getValue());
+        if (xContentType == null) {
+            throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType().getValue());
+        }
+        if (xContentType == XContentType.JSON) {
+            // No changes is required
+            return Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8));
+        } else {
+            // Need to convert into JSON
+            try (InputStream stream = response.getEntity().getContent();
+                 XContentParser parser = XContentFactory.xContent(xContentType).createParser(NamedXContentRegistry.EMPTY,
+                     DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) {
+                parser.nextToken();
+                XContentBuilder builder = XContentFactory.jsonBuilder();
+                builder.copyCurrentStructure(parser);
+                return Strings.toString(builder);
+            }
+        }
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.elasticsearch.client.RequestConverters.EndpointBuilder;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
+import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
+import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
+
+import java.io.IOException;
+
+import static org.elasticsearch.client.RequestConverters.REQUEST_BODY_CONTENT_TYPE;
+import static org.elasticsearch.client.RequestConverters.createEntity;
+
+final class MLRequestConverters {
+
+    private MLRequestConverters() {}
+
+    static Request putJob(PutJobRequest putJobRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+                .addPathPartAsIs("_xpack")
+                .addPathPartAsIs("ml")
+                .addPathPartAsIs("anomaly_detectors")
+                .addPathPart(putJobRequest.getJob().getId())
+                .build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(putJobRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
+    static Request openJob(OpenJobRequest openJobRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+                .addPathPartAsIs("_xpack")
+                .addPathPartAsIs("ml")
+                .addPathPartAsIs("anomaly_detectors")
+                .addPathPart(openJobRequest.getJobId())
+                .addPathPartAsIs("_open")
+                .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        request.setJsonEntity(openJobRequest.toString());
+        return request;
+    }
+
+    static Request deleteJob(DeleteJobRequest deleteJobRequest) {
+        String endpoint = new EndpointBuilder()
+                .addPathPartAsIs("_xpack")
+                .addPathPartAsIs("ml")
+                .addPathPartAsIs("anomaly_detectors")
+                .addPathPart(deleteJobRequest.getJobId())
+                .build();
+        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
+
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.putParam("force", Boolean.toString(deleteJobRequest.isForce()));
+
+        return request;
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java

@@ -0,0 +1,169 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse;
+import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
+import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
+import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
+import org.elasticsearch.protocol.xpack.ml.PutJobResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+/**
+ * Machine Learning API client wrapper for the {@link RestHighLevelClient}
+ *
+ * <p>
+ * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-apis.html">
+ * X-Pack Machine Learning APIs </a> for additional information.
+ */
+public final class MachineLearningClient {
+
+    private final RestHighLevelClient restHighLevelClient;
+
+    MachineLearningClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Creates a new Machine Learning Job
+     * <p>
+     * For additional info
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-job.html">ML PUT job documentation</a>
+     *
+     * @param request the PutJobRequest containing the {@link org.elasticsearch.protocol.xpack.ml.job.config.Job} settings
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return PutJobResponse with enclosed {@link org.elasticsearch.protocol.xpack.ml.job.config.Job} object
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public PutJobResponse putJob(PutJobRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::putJob,
+            options,
+            PutJobResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Creates a new Machine Learning Job asynchronously and notifies listener on completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-job.html">ML PUT job documentation</a>
+     *
+     * @param request  the request containing the {@link org.elasticsearch.protocol.xpack.ml.job.config.Job} settings
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void putJobAsync(PutJobRequest request, RequestOptions options, ActionListener<PutJobResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::putJob,
+            options,
+            PutJobResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
+    /**
+     * Deletes the given Machine Learning Job
+     * <p>
+     *     For additional info
+     *     see <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-job.html">ML Delete Job documentation</a>
+     * </p>
+     * @param request the request to delete the job
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return action acknowledgement
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public DeleteJobResponse deleteJob(DeleteJobRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::deleteJob,
+            options,
+            DeleteJobResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Deletes the given Machine Learning Job asynchronously and notifies the listener on completion
+     * <p>
+     *     For additional info
+     *     see <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-job.html">ML Delete Job documentation</a>
+     * </p>
+     * @param request the request to delete the job
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void deleteJobAsync(DeleteJobRequest request, RequestOptions options, ActionListener<DeleteJobResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::deleteJob,
+            options,
+            DeleteJobResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
+    /**
+     * Opens a Machine Learning Job.
+     * When you open a new job, it starts with an empty model.
+     *
+     * When you open an existing job, the most recent model state is automatically loaded.
+     * The job is ready to resume its analysis from where it left off, once new data is received.
+     *
+     * <p>
+     *     For additional info
+     *     see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-open-job.html"></a>
+     * </p>
+     * @param request request containing job_id and additional optional options
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return response containing if the job was successfully opened or not.
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public OpenJobResponse openJob(OpenJobRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::openJob,
+            options,
+            OpenJobResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Opens a Machine Learning Job asynchronously, notifies listener on completion.
+     * When you open a new job, it starts with an empty model.
+     *
+     * When you open an existing job, the most recent model state is automatically loaded.
+     * The job is ready to resume its analysis from where it left off, once new data is received.
+     * <p>
+     *     For additional info
+     *     see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-open-job.html"></a>
+     * </p>
+     * @param request request containing job_id and additional optional options
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void openJobAsync(OpenJobRequest request, RequestOptions options, ActionListener<OpenJobResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::openJob,
+            options,
+            OpenJobResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/MigrationClient.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+/**
+ * A wrapper for the {@link RestHighLevelClient} that provides methods for
+ * accessing the Elastic License-related methods
+ * <p>
+ * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/migration-api.html">
+ * X-Pack Migration APIs on elastic.co</a> for more information.
+ */
+public final class MigrationClient {
+
+    private final RestHighLevelClient restHighLevelClient;
+
+    MigrationClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Get Migration Assistance for one or more indices
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public IndexUpgradeInfoResponse getAssistance(IndexUpgradeInfoRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::getMigrationAssistance, options,
+            IndexUpgradeInfoResponse::fromXContent, Collections.emptySet());
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -39,12 +39,12 @@ import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyReposito
 import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
@@ -78,8 +78,8 @@ import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.action.ingest.GetPipelineRequest;
-import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -96,7 +96,7 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContent;
@@ -107,10 +107,13 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
-import org.elasticsearch.protocol.xpack.XPackUsageRequest;
-import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
 import org.elasticsearch.script.mustache.SearchTemplateRequest;
@@ -422,8 +425,14 @@ final class RequestConverters {
                 BytesReference indexSource = indexRequest.source();
                 XContentType indexXContentType = indexRequest.getContentType();
 
-                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
-                    LoggingDeprecationHandler.INSTANCE, indexSource, indexXContentType)) {
+                try (XContentParser parser = XContentHelper.createParser(
+                        /*
+                         * EMPTY and THROW are fine here because we just call
+                         * copyCurrentStructure which doesn't touch the
+                         * registry or deprecation.
+                         */
+                        NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                        indexSource, indexXContentType)) {
                     try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                         builder.copyCurrentStructure(parser);
                         source = BytesReference.bytes(builder).toBytesRef();
@@ -1154,7 +1163,11 @@ final class RequestConverters {
     }
 
     static Request putLicense(PutLicenseRequest putLicenseRequest) {
-        Request request = new Request(HttpPut.METHOD_NAME, "/_xpack/license");
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("license")
+            .build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
         Params parameters = new Params(request);
         parameters.withTimeout(putLicenseRequest.timeout());
         parameters.withMasterTimeout(putLicenseRequest.masterNodeTimeout());
@@ -1165,7 +1178,37 @@ final class RequestConverters {
         return request;
     }
 
-    private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
+    static Request getLicense(GetLicenseRequest getLicenseRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("license")
+            .build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Params parameters = new Params(request);
+        parameters.withLocal(getLicenseRequest.local());
+        return request;
+    }
+
+    static Request deleteLicense(DeleteLicenseRequest deleteLicenseRequest) {
+        Request request = new Request(HttpDelete.METHOD_NAME, "/_xpack/license");
+        Params parameters = new Params(request);
+        parameters.withTimeout(deleteLicenseRequest.timeout());
+        parameters.withMasterTimeout(deleteLicenseRequest.masterNodeTimeout());
+        return request;
+    }
+
+    static Request getMigrationAssistance(IndexUpgradeInfoRequest indexUpgradeInfoRequest) {
+        EndpointBuilder endpointBuilder = new EndpointBuilder()
+            .addPathPartAsIs("_xpack/migration/assistance")
+            .addCommaSeparatedPathParts(indexUpgradeInfoRequest.indices());
+        String endpoint = endpointBuilder.build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Params parameters = new Params(request);
+        parameters.withIndicesOptions(indexUpgradeInfoRequest.indicesOptions());
+        return request;
+    }
+
+    static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
     }

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -27,7 +27,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptResponse;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
@@ -53,13 +52,14 @@ import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ContextParser;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -163,8 +163,11 @@ import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipel
 import org.elasticsearch.search.aggregations.pipeline.derivative.ParsedDerivative;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
 import org.elasticsearch.search.suggest.term.TermSuggestion;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -205,6 +208,10 @@ public class RestHighLevelClient implements Closeable {
     private final SnapshotClient snapshotClient = new SnapshotClient(this);
     private final TasksClient tasksClient = new TasksClient(this);
     private final XPackClient xPackClient = new XPackClient(this);
+    private final WatcherClient watcherClient = new WatcherClient(this);
+    private final LicenseClient licenseClient = new LicenseClient(this);
+    private final MigrationClient migrationClient = new MigrationClient(this);
+    private final MachineLearningClient machineLearningClient = new MachineLearningClient(this);
 
     /**
      * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
@@ -296,18 +303,64 @@ public class RestHighLevelClient implements Closeable {
     }
 
     /**
-     * A wrapper for the {@link RestHighLevelClient} that provides methods for
-     * accessing the Elastic Licensed X-Pack APIs that are shipped with the
-     * default distribution of Elasticsearch. All of these APIs will 404 if run
-     * against the OSS distribution of Elasticsearch.
+     * Provides methods for accessing the Elastic Licensed X-Pack Info
+     * and Usage APIs that are shipped with the default distribution of
+     * Elasticsearch. All of these APIs will 404 if run against the OSS
+     * distribution of Elasticsearch.
      * <p>
-     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-api.html">
-     * X-Pack APIs on elastic.co</a> for more information.
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/info-api.html">
+     * Info APIs on elastic.co</a> for more information.
      */
     public final XPackClient xpack() {
         return xPackClient;
     }
 
+    /**
+     * Provides methods for accessing the Elastic Licensed Watcher APIs that
+     * are shipped with the default distribution of Elasticsearch. All of
+     * these APIs will 404 if run against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api.html">
+     * Watcher APIs on elastic.co</a> for more information.
+     */
+    public WatcherClient watcher() { return watcherClient; }
+
+    /**
+     * Provides methods for accessing the Elastic Licensed Licensing APIs that
+     * are shipped with the default distribution of Elasticsearch. All of
+     * these APIs will 404 if run against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html">
+     * Licensing APIs on elastic.co</a> for more information.
+     */
+    public LicenseClient license() { return licenseClient; }
+
+    /**
+     * Provides methods for accessing the Elastic Licensed Licensing APIs that
+     * are shipped with the default distribution of Elasticsearch. All of
+     * these APIs will 404 if run against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/migration-api.html">
+     * Migration APIs on elastic.co</a> for more information.
+     */
+    public MigrationClient migration() {
+        return migrationClient;
+    }
+
+    /**
+     * Provides methods for accessing the Elastic Licensed Machine Learning APIs that
+     * are shipped with the Elastic Stack distribution of Elasticsearch. All of
+     * these APIs will 404 if run against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-apis.html">
+     * Machine Learning APIs on elastic.co</a> for more information.
+     *
+     * @return the client wrapper for making Machine Learning API calls
+     */
+    public MachineLearningClient machineLearning() {
+        return machineLearningClient;
+    }
+
     /**
      * Executes a bulk request using the Bulk API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html">Bulk API on elastic.co</a>
@@ -863,9 +916,9 @@ public class RestHighLevelClient implements Closeable {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public DeleteStoredScriptResponse deleteScript(DeleteStoredScriptRequest request, RequestOptions options) throws IOException {
+    public AcknowledgedResponse deleteScript(DeleteStoredScriptRequest request, RequestOptions options) throws IOException {
         return performRequestAndParseEntity(request, RequestConverters::deleteScript, options,
-            DeleteStoredScriptResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -877,9 +930,9 @@ public class RestHighLevelClient implements Closeable {
      * @param listener the listener to be notified upon request completion
      */
     public void deleteScriptAsync(DeleteStoredScriptRequest request, RequestOptions options,
-                                  ActionListener<DeleteStoredScriptResponse> listener) {
+                                  ActionListener<AcknowledgedResponse> listener) {
         performRequestAsyncAndParseEntity(request, RequestConverters::deleteScript, options,
-            DeleteStoredScriptResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**
@@ -1050,8 +1103,7 @@ public class RestHighLevelClient implements Closeable {
         if (xContentType == null) {
             throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType().getValue());
         }
-        try (XContentParser parser = xContentType.xContent().createParser(registry,
-            LoggingDeprecationHandler.INSTANCE, entity.getContent())) {
+        try (XContentParser parser = xContentType.xContent().createParser(registry, DEPRECATION_HANDLER, entity.getContent())) {
             return entityParser.apply(parser);
         }
     }
@@ -1069,6 +1121,19 @@ public class RestHighLevelClient implements Closeable {
         return response.getStatusLine().getStatusCode() == 200;
     }
 
+    /**
+     * Ignores deprecation warnings. This is appropriate because it is only
+     * used to parse responses from Elasticsearch. Any deprecation warnings
+     * emitted there just mean that you are talking to an old version of
+     * Elasticsearch. There isn't anything you can do about the deprecation.
+     */
+    private static final DeprecationHandler DEPRECATION_HANDLER = new DeprecationHandler() {
+        @Override
+        public void usedDeprecatedName(String usedName, String modernName) {}
+        @Override
+        public void usedDeprecatedField(String usedName, String replacedWith) {}
+    };
+
     static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         Map<String, ContextParser<Object, ? extends Aggregation>> map = new HashMap<>();
         map.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c));
@@ -1119,11 +1184,11 @@ public class RestHighLevelClient implements Closeable {
         List<NamedXContentRegistry.Entry> entries = map.entrySet().stream()
                 .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
                 .collect(Collectors.toList());
-        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(TermSuggestion.NAME),
+        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(TermSuggestionBuilder.SUGGESTION_NAME),
                 (parser, context) -> TermSuggestion.fromXContent(parser, (String)context)));
-        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(PhraseSuggestion.NAME),
+        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(PhraseSuggestionBuilder.SUGGESTION_NAME),
                 (parser, context) -> PhraseSuggestion.fromXContent(parser, (String)context)));
-        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(CompletionSuggestion.NAME),
+        entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(CompletionSuggestionBuilder.SUGGESTION_NAME),
                 (parser, context) -> CompletionSuggestion.fromXContent(parser, (String)context)));
         return entries;
     }

client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java

@@ -21,23 +21,21 @@ package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
-import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
-import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
-import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
-import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 
 import java.io.IOException;
 
@@ -95,9 +93,9 @@ public final class SnapshotClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public PutRepositoryResponse createRepository(PutRepositoryRequest putRepositoryRequest, RequestOptions options) throws IOException {
+    public AcknowledgedResponse createRepository(PutRepositoryRequest putRepositoryRequest, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(putRepositoryRequest, RequestConverters::createRepository, options,
-            PutRepositoryResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -109,9 +107,9 @@ public final class SnapshotClient {
      * @param listener the listener to be notified upon request completion
      */
     public void createRepositoryAsync(PutRepositoryRequest putRepositoryRequest, RequestOptions options,
-                                      ActionListener<PutRepositoryResponse> listener) {
+                                      ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(putRepositoryRequest, RequestConverters::createRepository, options,
-            PutRepositoryResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**
@@ -123,10 +121,10 @@ public final class SnapshotClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public DeleteRepositoryResponse deleteRepository(DeleteRepositoryRequest deleteRepositoryRequest, RequestOptions options)
+    public AcknowledgedResponse deleteRepository(DeleteRepositoryRequest deleteRepositoryRequest, RequestOptions options)
         throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(deleteRepositoryRequest, RequestConverters::deleteRepository, options,
-            DeleteRepositoryResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -138,9 +136,9 @@ public final class SnapshotClient {
      * @param listener the listener to be notified upon request completion
      */
     public void deleteRepositoryAsync(DeleteRepositoryRequest deleteRepositoryRequest, RequestOptions options,
-                                      ActionListener<DeleteRepositoryResponse> listener) {
+                                      ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(deleteRepositoryRequest, RequestConverters::deleteRepository, options,
-            DeleteRepositoryResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 
     /**
@@ -294,9 +292,9 @@ public final class SnapshotClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public DeleteSnapshotResponse delete(DeleteSnapshotRequest deleteSnapshotRequest, RequestOptions options) throws IOException {
+    public AcknowledgedResponse delete(DeleteSnapshotRequest deleteSnapshotRequest, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(deleteSnapshotRequest, RequestConverters::deleteSnapshot, options,
-            DeleteSnapshotResponse::fromXContent, emptySet());
+            AcknowledgedResponse::fromXContent, emptySet());
     }
 
     /**
@@ -309,8 +307,8 @@ public final class SnapshotClient {
      * @param listener the listener to be notified upon request completion
      */
     public void deleteAsync(DeleteSnapshotRequest deleteSnapshotRequest, RequestOptions options,
-                            ActionListener<DeleteSnapshotResponse> listener) {
+                            ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(deleteSnapshotRequest, RequestConverters::deleteSnapshot, options,
-            DeleteSnapshotResponse::fromXContent, listener, emptySet());
+            AcknowledgedResponse::fromXContent, listener, emptySet());
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java

@@ -41,17 +41,9 @@ import static java.util.Collections.emptySet;
 public final class XPackClient {
 
     private final RestHighLevelClient restHighLevelClient;
-    private final WatcherClient watcherClient;
-    private final LicenseClient licenseClient;
 
     XPackClient(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
-        this.watcherClient = new WatcherClient(restHighLevelClient);
-        this.licenseClient = new LicenseClient(restHighLevelClient);
-    }
-
-    public WatcherClient watcher() {
-        return watcherClient;
     }
 
     /**
@@ -102,15 +94,4 @@ public final class XPackClient {
         restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xpackUsage, options,
             XPackUsageResponse::fromXContent, listener, emptySet());
     }
-
-    /**
-     * A wrapper for the {@link RestHighLevelClient} that provides methods for
-     * accessing the Elastic Licensing APIs.
-     * <p>
-     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html">
-     * X-Pack APIs on elastic.co</a> for more information.
-     */
-    public LicenseClient license() {
-        return licenseClient;
-    }
 }

client/rest-high-level/src/main/resources/forbidden/rest-high-level-signatures.txt

@@ -19,3 +19,6 @@ org.apache.http.entity.ContentType#create(java.lang.String)
 org.apache.http.entity.ContentType#create(java.lang.String,java.lang.String)
 org.apache.http.entity.ContentType#create(java.lang.String,java.nio.charset.Charset)
 org.apache.http.entity.ContentType#create(java.lang.String,org.apache.http.NameValuePair[])
+
+@defaultMessage We can't rely on log4j2 being on the classpath so don't log deprecations!
+org.elasticsearch.common.xcontent.LoggingDeprecationHandler

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -27,18 +27,15 @@ import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
-import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest;
@@ -51,7 +48,6 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRespon
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -61,20 +57,19 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
-import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
@@ -411,7 +406,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
         mappingBuilder.endObject().endObject().endObject();
         putMappingRequest.source(mappingBuilder);
 
-        PutMappingResponse putMappingResponse =
+        AcknowledgedResponse putMappingResponse =
                 execute(putMappingRequest, highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync);
         assertTrue(putMappingResponse.isAcknowledged());
 
@@ -431,7 +426,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
         mappingBuilder.endObject().endObject().endObject();
         putMappingRequest.source(mappingBuilder);
 
-        PutMappingResponse putMappingResponse =
+        AcknowledgedResponse putMappingResponse =
             execute(putMappingRequest, highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync);
         assertTrue(putMappingResponse.isAcknowledged());
 
@@ -467,7 +462,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
         mappingBuilder.endObject().endObject().endObject();
         putMappingRequest.source(mappingBuilder);
 
-        PutMappingResponse putMappingResponse =
+        AcknowledgedResponse putMappingResponse =
             execute(putMappingRequest, highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync);
         assertTrue(putMappingResponse.isAcknowledged());
 
@@ -497,7 +492,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
             createIndex(indexName, Settings.EMPTY);
 
             DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName);
-            DeleteIndexResponse deleteIndexResponse =
+            AcknowledgedResponse deleteIndexResponse =
                     execute(deleteIndexRequest, highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync);
             assertTrue(deleteIndexResponse.isAcknowledged());
 
@@ -529,7 +524,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
         AliasActions addAction = new AliasActions(AliasActions.Type.ADD).index(index).aliases(alias);
         addAction.routing("routing").searchRouting("search_routing").filter("{\"term\":{\"year\":2016}}");
         aliasesAddRequest.addAliasAction(addAction);
-        IndicesAliasesResponse aliasesAddResponse = execute(aliasesAddRequest, highLevelClient().indices()::updateAliases,
+        AcknowledgedResponse aliasesAddResponse = execute(aliasesAddRequest, highLevelClient().indices()::updateAliases,
                 highLevelClient().indices()::updateAliasesAsync);
         assertTrue(aliasesAddResponse.isAcknowledged());
         assertThat(aliasExists(alias), equalTo(true));
@@ -547,7 +542,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
         aliasesAddRemoveRequest.addAliasAction(addAction);
         AliasActions removeAction = new AliasActions(AliasActions.Type.REMOVE).index(index).alias(alias);
         aliasesAddRemoveRequest.addAliasAction(removeAction);
-        IndicesAliasesResponse aliasesAddRemoveResponse = execute(aliasesAddRemoveRequest, highLevelClient().indices()::updateAliases,
+        AcknowledgedResponse aliasesAddRemoveResponse = execute(aliasesAddRemoveRequest, highLevelClient().indices()::updateAliases,
                 highLevelClient().indices()::updateAliasesAsync);
         assertTrue(aliasesAddRemoveResponse.isAcknowledged());
         assertThat(aliasExists(alias), equalTo(false));
@@ -558,7 +553,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
         IndicesAliasesRequest aliasesRemoveIndexRequest = new IndicesAliasesRequest();
         AliasActions removeIndexAction = new AliasActions(AliasActions.Type.REMOVE_INDEX).index(index);
         aliasesRemoveIndexRequest.addAliasAction(removeIndexAction);
-        IndicesAliasesResponse aliasesRemoveIndexResponse = execute(aliasesRemoveIndexRequest, highLevelClient().indices()::updateAliases,
+        AcknowledgedResponse aliasesRemoveIndexResponse = execute(aliasesRemoveIndexRequest, highLevelClient().indices()::updateAliases,
                 highLevelClient().indices()::updateAliasesAsync);
         assertTrue(aliasesRemoveIndexResponse.isAcknowledged());
         assertThat(aliasExists(alias), equalTo(false));
@@ -654,7 +649,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
         assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
 
         CloseIndexRequest closeIndexRequest = new CloseIndexRequest(index);
-        CloseIndexResponse closeIndexResponse = execute(closeIndexRequest, highLevelClient().indices()::close,
+        AcknowledgedResponse closeIndexResponse = execute(closeIndexRequest, highLevelClient().indices()::close,
                 highLevelClient().indices()::closeAsync);
         assertTrue(closeIndexResponse.isAcknowledged());
 
@@ -1144,7 +1139,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
         assertThat(dynamicSetting.getDefault(Settings.EMPTY), not(dynamicSettingValue));
         UpdateSettingsRequest dynamicSettingRequest = new UpdateSettingsRequest();
         dynamicSettingRequest.settings(Settings.builder().put(dynamicSettingKey, dynamicSettingValue).build());
-        UpdateSettingsResponse response = execute(dynamicSettingRequest, highLevelClient().indices()::putSettings,
+        AcknowledgedResponse response = execute(dynamicSettingRequest, highLevelClient().indices()::putSettings,
                 highLevelClient().indices()::putSettingsAsync);
 
         assertTrue(response.isAcknowledged());
@@ -1227,7 +1222,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
             .mapping("doc", "host_name", "type=keyword", "description", "type=text")
             .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz"));
 
-        PutIndexTemplateResponse putTemplateResponse = execute(putTemplateRequest,
+        AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest,
             highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync);
         assertThat(putTemplateResponse.isAcknowledged(), equalTo(true));
 

client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java

@@ -28,7 +28,7 @@ import org.elasticsearch.action.ingest.SimulateDocumentResult;
 import org.elasticsearch.action.ingest.SimulateDocumentVerboseResult;
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -50,7 +50,7 @@ public class IngestClientIT extends ESRestHighLevelClientTestCase {
             BytesReference.bytes(pipelineBuilder),
             pipelineBuilder.contentType());
 
-        WritePipelineResponse putPipelineResponse =
+        AcknowledgedResponse putPipelineResponse =
             execute(request, highLevelClient().ingest()::putPipeline, highLevelClient().ingest()::putPipelineAsync);
         assertTrue(putPipelineResponse.isAcknowledged());
     }
@@ -86,7 +86,7 @@ public class IngestClientIT extends ESRestHighLevelClientTestCase {
 
         DeletePipelineRequest request = new DeletePipelineRequest(id);
 
-        WritePipelineResponse response =
+        AcknowledgedResponse response =
             execute(request, highLevelClient().ingest()::deletePipeline, highLevelClient().ingest()::deletePipelineAsync);
         assertTrue(response.isAcknowledged());
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
+import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
+import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
+import org.elasticsearch.protocol.xpack.ml.job.config.AnalysisConfig;
+import org.elasticsearch.protocol.xpack.ml.job.config.Detector;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MLRequestConvertersTests extends ESTestCase {
+
+    public void testPutJob() throws IOException {
+        Job job = createValidJob("foo");
+        PutJobRequest putJobRequest = new PutJobRequest(job);
+
+        Request request = MLRequestConverters.putJob(putJobRequest);
+
+        assertThat(request.getEndpoint(), equalTo("/_xpack/ml/anomaly_detectors/foo"));
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
+            Job parsedJob = Job.PARSER.apply(parser, null).build();
+            assertThat(parsedJob, equalTo(job));
+        }
+    }
+
+    public void testOpenJob() throws Exception {
+        String jobId = "some-job-id";
+        OpenJobRequest openJobRequest = new OpenJobRequest(jobId);
+        openJobRequest.setTimeout(TimeValue.timeValueMinutes(10));
+
+        Request request = MLRequestConverters.openJob(openJobRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_xpack/ml/anomaly_detectors/" + jobId + "/_open", request.getEndpoint());
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        request.getEntity().writeTo(bos);
+        assertEquals(bos.toString("UTF-8"), "{\"job_id\":\""+ jobId +"\",\"timeout\":\"10m\"}");
+    }
+
+    public void testDeleteJob() {
+        String jobId = randomAlphaOfLength(10);
+        DeleteJobRequest deleteJobRequest = new DeleteJobRequest(jobId);
+
+        Request request = MLRequestConverters.deleteJob(deleteJobRequest);
+        assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
+        assertEquals("/_xpack/ml/anomaly_detectors/" + jobId, request.getEndpoint());
+        assertEquals(Boolean.toString(false), request.getParameters().get("force"));
+
+        deleteJobRequest.setForce(true);
+        request = MLRequestConverters.deleteJob(deleteJobRequest);
+        assertEquals(Boolean.toString(true), request.getParameters().get("force"));
+    }
+
+    private static Job createValidJob(String jobId) {
+        AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList(
+                Detector.builder().setFunction("count").build()));
+        Job.Builder jobBuilder = Job.builder(jobId);
+        jobBuilder.setAnalysisConfig(analysisConfig);
+        return jobBuilder.build();
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse;
+import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
+import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
+import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
+import org.elasticsearch.protocol.xpack.ml.PutJobResponse;
+import org.elasticsearch.protocol.xpack.ml.job.config.AnalysisConfig;
+import org.elasticsearch.protocol.xpack.ml.job.config.DataDescription;
+import org.elasticsearch.protocol.xpack.ml.job.config.Detector;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.is;
+
+public class MachineLearningIT extends ESRestHighLevelClientTestCase {
+
+    public void testPutJob() throws Exception {
+        String jobId = randomValidJobId();
+        Job job = buildJob(jobId);
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+
+        PutJobResponse putJobResponse = execute(new PutJobRequest(job), machineLearningClient::putJob, machineLearningClient::putJobAsync);
+        Job createdJob = putJobResponse.getResponse();
+
+        assertThat(createdJob.getId(), is(jobId));
+        assertThat(createdJob.getJobType(), is(Job.ANOMALY_DETECTOR_JOB_TYPE));
+    }
+
+    public void testDeleteJob() throws Exception {
+        String jobId = randomValidJobId();
+        Job job = buildJob(jobId);
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        machineLearningClient.putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
+
+        DeleteJobResponse response = execute(new DeleteJobRequest(jobId),
+            machineLearningClient::deleteJob,
+            machineLearningClient::deleteJobAsync);
+
+        assertTrue(response.isAcknowledged());
+    }
+
+    public void testOpenJob() throws Exception {
+        String jobId = randomValidJobId();
+        Job job = buildJob(jobId);
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+
+        machineLearningClient.putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
+
+        OpenJobResponse response = execute(new OpenJobRequest(jobId), machineLearningClient::openJob, machineLearningClient::openJobAsync);
+
+        assertTrue(response.isOpened());
+    }
+
+    public static String randomValidJobId() {
+        CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz0123456789".toCharArray());
+        return generator.ofCodePointsLength(random(), 10, 10);
+    }
+
+    public static Job buildJob(String jobId) {
+        Job.Builder builder = new Job.Builder(jobId);
+        builder.setDescription(randomAlphaOfLength(10));
+
+        Detector detector = new Detector.Builder()
+            .setFieldName("total")
+            .setFunction("sum")
+            .setDetectorDescription(randomAlphaOfLength(10))
+            .build();
+        AnalysisConfig.Builder configBuilder = new AnalysisConfig.Builder(Arrays.asList(detector));
+        configBuilder.setBucketSpan(new TimeValue(randomIntBetween(1, 10), TimeUnit.SECONDS));
+        builder.setAnalysisConfig(configBuilder);
+
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setTimeFormat(randomFrom(DataDescription.EPOCH_MS, DataDescription.EPOCH));
+        dataDescription.setTimeField(randomAlphaOfLength(10));
+        builder.setDataDescription(dataDescription);
+
+        return builder.build();
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
+
+import java.io.IOException;
+
+public class MigrationIT extends ESRestHighLevelClientTestCase {
+
+    public void testGetAssistance() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            IndexUpgradeInfoResponse response = client.migration().getAssistance(new IndexUpgradeInfoRequest(), RequestOptions.DEFAULT);
+            assertEquals(0, response.getActions().size());
+        }
+        {
+            client.indices().create(new CreateIndexRequest("test"), RequestOptions.DEFAULT);
+            IndexUpgradeInfoResponse response = client.migration().getAssistance(
+                new IndexUpgradeInfoRequest("test"), RequestOptions.DEFAULT);
+            assertEquals(0, response.getActions().size());
+        }
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -126,6 +126,7 @@ import org.elasticsearch.index.rankeval.RankEvalSpec;
 import org.elasticsearch.index.rankeval.RatedRequest;
 import org.elasticsearch.index.rankeval.RestRankEvalAction;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.repositories.fs.FsRepository;
@@ -2552,6 +2553,23 @@ public class RequestConvertersTests extends ESTestCase {
         assertEquals(expectedParams, request.getParameters());
     }
 
+    public void testGetMigrationAssistance() {
+        IndexUpgradeInfoRequest upgradeInfoRequest = new IndexUpgradeInfoRequest();
+        String expectedEndpoint = "/_xpack/migration/assistance";
+        if (randomBoolean()) {
+            String[] indices = randomIndicesNames(1, 5);
+            upgradeInfoRequest.indices(indices);
+            expectedEndpoint += "/" + String.join(",", indices);
+        }
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomIndicesOptions(upgradeInfoRequest::indicesOptions, upgradeInfoRequest::indicesOptions, expectedParams);
+        Request request = RequestConverters.getMigrationAssistance(upgradeInfoRequest);
+        assertEquals(HttpGet.METHOD_NAME, request.getMethod());
+        assertEquals(expectedEndpoint, request.getEndpoint());
+        assertNull(request.getEntity());
+        assertEquals(expectedParams, request.getParameters());
+    }
+
     public void testXPackPutWatch() throws Exception {
         PutWatchRequest putWatchRequest = new PutWatchRequest();
         String watchId = randomAlphaOfLength(10);

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -20,7 +20,6 @@
 package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
-
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
@@ -755,7 +754,11 @@ public class RestHighLevelClientTests extends ESTestCase {
                             method.isAnnotationPresent(Deprecated.class));
                     } else {
                         //TODO xpack api are currently ignored, we need to load xpack yaml spec too
-                        if (apiName.startsWith("xpack.") == false) {
+                        if (apiName.startsWith("xpack.") == false &&
+                            apiName.startsWith("license.") == false &&
+                            apiName.startsWith("machine_learning.") == false &&
+                            apiName.startsWith("watcher.") == false &&
+                            apiName.startsWith("migration.") == false) {
                             apiNotFound.add(apiName);
                         }
                     }

client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java

@@ -21,24 +21,22 @@ package org.elasticsearch.client;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
-import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
-import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
-import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
-import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.RestStatus;
@@ -55,7 +53,7 @@ import static org.hamcrest.Matchers.is;
 
 public class SnapshotIT extends ESRestHighLevelClientTestCase {
 
-    private PutRepositoryResponse createTestRepository(String repository, String type, String settings) throws IOException {
+    private AcknowledgedResponse createTestRepository(String repository, String type, String settings) throws IOException {
         PutRepositoryRequest request = new PutRepositoryRequest(repository);
         request.settings(settings, XContentType.JSON);
         request.type(type);
@@ -71,7 +69,7 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
     }
 
     public void testCreateRepository() throws IOException {
-        PutRepositoryResponse response = createTestRepository("test", FsRepository.TYPE, "{\"location\": \".\"}");
+        AcknowledgedResponse response = createTestRepository("test", FsRepository.TYPE, "{\"location\": \".\"}");
         assertTrue(response.isAcknowledged());
     }
 
@@ -117,14 +115,14 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
         assertThat(1, equalTo(response.repositories().size()));
 
         DeleteRepositoryRequest deleteRequest = new DeleteRepositoryRequest(repository);
-        DeleteRepositoryResponse deleteResponse = execute(deleteRequest, highLevelClient().snapshot()::deleteRepository,
+        AcknowledgedResponse deleteResponse = execute(deleteRequest, highLevelClient().snapshot()::deleteRepository,
             highLevelClient().snapshot()::deleteRepositoryAsync);
 
         assertTrue(deleteResponse.isAcknowledged());
     }
 
     public void testVerifyRepository() throws IOException {
-        PutRepositoryResponse putRepositoryResponse = createTestRepository("test", FsRepository.TYPE, "{\"location\": \".\"}");
+        AcknowledgedResponse putRepositoryResponse = createTestRepository("test", FsRepository.TYPE, "{\"location\": \".\"}");
         assertTrue(putRepositoryResponse.isAcknowledged());
 
         VerifyRepositoryRequest request = new VerifyRepositoryRequest("test");
@@ -153,7 +151,7 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
         String snapshot1 = "test_snapshot1";
         String snapshot2 = "test_snapshot2";
 
-        PutRepositoryResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}");
+        AcknowledgedResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}");
         assertTrue(putRepositoryResponse.isAcknowledged());
 
         CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository, snapshot1);
@@ -187,7 +185,7 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
         String testSnapshot = "snapshot";
         String testIndex = "test_index";
 
-        PutRepositoryResponse putRepositoryResponse = createTestRepository(testRepository, FsRepository.TYPE, "{\"location\": \".\"}");
+        AcknowledgedResponse putRepositoryResponse = createTestRepository(testRepository, FsRepository.TYPE, "{\"location\": \".\"}");
         assertTrue(putRepositoryResponse.isAcknowledged());
 
         createIndex(testIndex, Settings.EMPTY);
@@ -216,7 +214,7 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
         String testIndex = "test_index";
         String restoredIndex = testIndex + "_restored";
 
-        PutRepositoryResponse putRepositoryResponse = createTestRepository(testRepository, FsRepository.TYPE, "{\"location\": \".\"}");
+        AcknowledgedResponse putRepositoryResponse = createTestRepository(testRepository, FsRepository.TYPE, "{\"location\": \".\"}");
         assertTrue(putRepositoryResponse.isAcknowledged());
 
         createIndex(testIndex, Settings.EMPTY);
@@ -250,7 +248,7 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
         String repository = "test_repository";
         String snapshot = "test_snapshot";
 
-        PutRepositoryResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}");
+        AcknowledgedResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}");
         assertTrue(putRepositoryResponse.isAcknowledged());
 
         CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(repository, snapshot);
@@ -260,7 +258,7 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
         assertEquals(RestStatus.OK, createSnapshotResponse.status());
 
         DeleteSnapshotRequest request = new DeleteSnapshotRequest(repository, snapshot);
-        DeleteSnapshotResponse response = execute(request, highLevelClient().snapshot()::delete, highLevelClient().snapshot()::deleteAsync);
+        AcknowledgedResponse response = execute(request, highLevelClient().snapshot()::delete, highLevelClient().snapshot()::deleteAsync);
 
         assertTrue(response.isAcknowledged());
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java

@@ -23,9 +23,9 @@ import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptResponse;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -90,7 +90,7 @@ public class StoredScriptsIT extends ESRestHighLevelClientTestCase {
         deleteRequest.masterNodeTimeout("50s");
         deleteRequest.timeout("50s");
 
-        DeleteStoredScriptResponse deleteResponse = execute(deleteRequest, highLevelClient()::deleteScript,
+        AcknowledgedResponse deleteResponse = execute(deleteRequest, highLevelClient()::deleteScript,
             highLevelClient()::deleteScriptAsync);
 
         assertThat(deleteResponse.isAcknowledged(), equalTo(true));

client/rest-high-level/src/test/java/org/elasticsearch/client/SyncedFlushResponseTests.java

@@ -24,7 +24,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -63,7 +63,7 @@ public class SyncedFlushResponseTests extends ESTestCase {
                 .xContent()
                 .createParser(
                     xContentRegistry(),
-                    LoggingDeprecationHandler.INSTANCE,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                     BytesReference.bytes(serverResponsebuilder).streamInput()
                 ).map()
         );
@@ -74,7 +74,7 @@ public class SyncedFlushResponseTests extends ESTestCase {
                 .xContent()
                 .createParser(
                     xContentRegistry(),
-                    LoggingDeprecationHandler.INSTANCE,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                     BytesReference.bytes(clientResponsebuilder).streamInput()
                 )
                 .map()
@@ -94,7 +94,9 @@ public class SyncedFlushResponseTests extends ESTestCase {
             .contentType()
             .xContent()
             .createParser(
-                xContentRegistry(), LoggingDeprecationHandler.INSTANCE, BytesReference.bytes(builder).streamInput()
+                xContentRegistry(),
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                BytesReference.bytes(builder).streamInput()
             );
         SyncedFlushResponse originalResponse = plan.clientResult;
         SyncedFlushResponse parsedResponse = SyncedFlushResponse.fromXContent(parser);
@@ -175,7 +177,8 @@ public class SyncedFlushResponseTests extends ESTestCase {
                                     .contentType()
                                     .xContent()
                                     .createParser(
-                                        xContentRegistry(), LoggingDeprecationHandler.INSTANCE,
+                                        xContentRegistry(),
+                                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                                         BytesReference.bytes(builder).streamInput()
                                     )
                                     .map();

client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java

@@ -46,7 +46,7 @@ public class WatcherIT extends ESRestHighLevelClientTestCase {
             "}";
         BytesReference bytesReference = new BytesArray(json);
         PutWatchRequest putWatchRequest = new PutWatchRequest(watchId, bytesReference, XContentType.JSON);
-        return highLevelClient().xpack().watcher().putWatch(putWatchRequest, RequestOptions.DEFAULT);
+        return highLevelClient().watcher().putWatch(putWatchRequest, RequestOptions.DEFAULT);
     }
 
     public void testDeleteWatch() throws Exception {
@@ -54,7 +54,7 @@ public class WatcherIT extends ESRestHighLevelClientTestCase {
         {
             String watchId = randomAlphaOfLength(10);
             createWatch(watchId);
-            DeleteWatchResponse deleteWatchResponse = highLevelClient().xpack().watcher().deleteWatch(new DeleteWatchRequest(watchId),
+            DeleteWatchResponse deleteWatchResponse = highLevelClient().watcher().deleteWatch(new DeleteWatchRequest(watchId),
                 RequestOptions.DEFAULT);
             assertThat(deleteWatchResponse.getId(), is(watchId));
             assertThat(deleteWatchResponse.getVersion(), is(2L));
@@ -64,7 +64,7 @@ public class WatcherIT extends ESRestHighLevelClientTestCase {
         // delete watch that does not exist
         {
             String watchId = randomAlphaOfLength(10);
-            DeleteWatchResponse deleteWatchResponse = highLevelClient().xpack().watcher().deleteWatch(new DeleteWatchRequest(watchId),
+            DeleteWatchResponse deleteWatchResponse = highLevelClient().watcher().deleteWatch(new DeleteWatchRequest(watchId),
                 RequestOptions.DEFAULT);
             assertThat(deleteWatchResponse.getId(), is(watchId));
             assertThat(deleteWatchResponse.getVersion(), is(1L));

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java

@@ -25,7 +25,6 @@ import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
@@ -33,11 +32,9 @@ import org.elasticsearch.action.admin.indices.analyze.DetailAnalyzeResponse;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
-import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest;
@@ -50,7 +47,6 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRespon
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -60,20 +56,19 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
-import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 import org.elasticsearch.action.admin.indices.validate.query.QueryExplanation;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.GetAliasesResponse;
 import org.elasticsearch.client.RequestOptions;
@@ -220,7 +215,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             // end::delete-index-request-indicesOptions
 
             // tag::delete-index-execute
-            DeleteIndexResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
+            AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
             // end::delete-index-execute
 
             // tag::delete-index-response
@@ -255,10 +250,10 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             DeleteIndexRequest request = new DeleteIndexRequest("posts");
 
             // tag::delete-index-execute-listener
-            ActionListener<DeleteIndexResponse> listener =
-                    new ActionListener<DeleteIndexResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                    new ActionListener<AcknowledgedResponse>() {
                 @Override
-                public void onResponse(DeleteIndexResponse deleteIndexResponse) {
+                public void onResponse(AcknowledgedResponse deleteIndexResponse) {
                     // <1>
                 }
 
@@ -480,7 +475,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
                     "}", // <1>
                     XContentType.JSON);
                 // end::put-mapping-request-source
-                PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
+                AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
                 assertTrue(putMappingResponse.isAcknowledged());
             }
 
@@ -494,7 +489,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
                 jsonMap.put("properties", properties);
                 request.source(jsonMap); // <1>
                 //end::put-mapping-map
-                PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
+                AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
                 assertTrue(putMappingResponse.isAcknowledged());
             }
             {
@@ -515,14 +510,14 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
                 builder.endObject();
                 request.source(builder); // <1>
                 //end::put-mapping-xcontent
-                PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
+                AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
                 assertTrue(putMappingResponse.isAcknowledged());
             }
             {
                 //tag::put-mapping-shortcut
                 request.source("message", "type=text"); // <1>
                 //end::put-mapping-shortcut
-                PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
+                AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
                 assertTrue(putMappingResponse.isAcknowledged());
             }
 
@@ -536,7 +531,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             // end::put-mapping-request-masterTimeout
 
             // tag::put-mapping-execute
-            PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
+            AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
             // end::put-mapping-execute
 
             // tag::put-mapping-response
@@ -558,10 +553,10 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             PutMappingRequest request = new PutMappingRequest("twitter").type("tweet");
 
             // tag::put-mapping-execute-listener
-            ActionListener<PutMappingResponse> listener =
-                new ActionListener<PutMappingResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                new ActionListener<AcknowledgedResponse>() {
                     @Override
-                    public void onResponse(PutMappingResponse putMappingResponse) {
+                    public void onResponse(AcknowledgedResponse putMappingResponse) {
                         // <1>
                     }
 
@@ -601,7 +596,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
                     "  }\n" +
                     "}", // <1>
                 XContentType.JSON);
-            PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
+            AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
             assertTrue(putMappingResponse.isAcknowledged());
         }
 
@@ -658,7 +653,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
                     "  }\n" +
                     "}", // <1>
                 XContentType.JSON);
-            PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
+            AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
             assertTrue(putMappingResponse.isAcknowledged());
         }
 
@@ -731,7 +726,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
                     "  }\n" +
                     "}", // <1>
                 XContentType.JSON);
-            PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
+            AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
             assertTrue(putMappingResponse.isAcknowledged());
         }
 
@@ -1497,7 +1492,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             // end::close-index-request-indicesOptions
 
             // tag::close-index-execute
-            CloseIndexResponse closeIndexResponse = client.indices().close(request, RequestOptions.DEFAULT);
+            AcknowledgedResponse closeIndexResponse = client.indices().close(request, RequestOptions.DEFAULT);
             // end::close-index-execute
 
             // tag::close-index-response
@@ -1506,10 +1501,10 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             assertTrue(acknowledged);
 
             // tag::close-index-execute-listener
-            ActionListener<CloseIndexResponse> listener =
-                    new ActionListener<CloseIndexResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                    new ActionListener<AcknowledgedResponse>() {
                 @Override
-                public void onResponse(CloseIndexResponse closeIndexResponse) {
+                public void onResponse(AcknowledgedResponse closeIndexResponse) {
                     // <1>
                 }
 
@@ -1649,7 +1644,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             // end::update-aliases-request-masterTimeout
 
             // tag::update-aliases-execute
-            IndicesAliasesResponse indicesAliasesResponse =
+            AcknowledgedResponse indicesAliasesResponse =
                     client.indices().updateAliases(request, RequestOptions.DEFAULT);
             // end::update-aliases-execute
 
@@ -1665,10 +1660,10 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             request.addAliasAction(aliasAction);
 
             // tag::update-aliases-execute-listener
-            ActionListener<IndicesAliasesResponse> listener =
-                    new ActionListener<IndicesAliasesResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                    new ActionListener<AcknowledgedResponse>() {
                 @Override
-                public void onResponse(IndicesAliasesResponse indicesAliasesResponse) {
+                public void onResponse(AcknowledgedResponse indicesAliasesResponse) {
                     // <1>
                 }
 
@@ -2056,7 +2051,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
         // end::put-settings-request-indicesOptions
 
         // tag::put-settings-execute
-        UpdateSettingsResponse updateSettingsResponse =
+        AcknowledgedResponse updateSettingsResponse =
                 client.indices().putSettings(request, RequestOptions.DEFAULT);
         // end::put-settings-execute
 
@@ -2066,11 +2061,11 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
         assertTrue(acknowledged);
 
         // tag::put-settings-execute-listener
-        ActionListener<UpdateSettingsResponse> listener =
-                new ActionListener<UpdateSettingsResponse>() {
+        ActionListener<AcknowledgedResponse> listener =
+                new ActionListener<AcknowledgedResponse>() {
 
             @Override
-            public void onResponse(UpdateSettingsResponse updateSettingsResponse) {
+            public void onResponse(AcknowledgedResponse updateSettingsResponse) {
                 // <1>
             }
 
@@ -2219,7 +2214,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
         request.create(false); // make test happy
 
         // tag::put-template-execute
-        PutIndexTemplateResponse putTemplateResponse = client.indices().putTemplate(request, RequestOptions.DEFAULT);
+        AcknowledgedResponse putTemplateResponse = client.indices().putTemplate(request, RequestOptions.DEFAULT);
         // end::put-template-execute
 
         // tag::put-template-response
@@ -2228,10 +2223,10 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
         assertTrue(acknowledged);
 
         // tag::put-template-execute-listener
-        ActionListener<PutIndexTemplateResponse> listener =
-            new ActionListener<PutIndexTemplateResponse>() {
+        ActionListener<AcknowledgedResponse> listener =
+            new ActionListener<AcknowledgedResponse>() {
                 @Override
-                public void onResponse(PutIndexTemplateResponse putTemplateResponse) {
+                public void onResponse(AcknowledgedResponse putTemplateResponse) {
                     // <1>
                 }
 
@@ -2461,7 +2456,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             .indices("my_index")
             .type("_doc")
             .source("my_field", "type=text,analyzer=english");
-        PutMappingResponse pmResp = client.indices().putMapping(pmReq, RequestOptions.DEFAULT);
+        AcknowledgedResponse pmResp = client.indices().putMapping(pmReq, RequestOptions.DEFAULT);
         assertTrue(pmResp.isAcknowledged());
 
         {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java

@@ -31,7 +31,7 @@ import org.elasticsearch.action.ingest.SimulateDocumentVerboseResult;
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
 import org.elasticsearch.action.ingest.SimulateProcessorResult;
-import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
@@ -93,7 +93,7 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
             // end::put-pipeline-request-masterTimeout
 
             // tag::put-pipeline-execute
-            WritePipelineResponse response = client.ingest().putPipeline(request, RequestOptions.DEFAULT); // <1>
+            AcknowledgedResponse response = client.ingest().putPipeline(request, RequestOptions.DEFAULT); // <1>
             // end::put-pipeline-execute
 
             // tag::put-pipeline-response
@@ -117,10 +117,10 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
             );
 
             // tag::put-pipeline-execute-listener
-            ActionListener<WritePipelineResponse> listener =
-                new ActionListener<WritePipelineResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                new ActionListener<AcknowledgedResponse>() {
                     @Override
-                    public void onResponse(WritePipelineResponse response) {
+                    public void onResponse(AcknowledgedResponse response) {
                         // <1>
                     }
 
@@ -236,7 +236,7 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
             // end::delete-pipeline-request-masterTimeout
 
             // tag::delete-pipeline-execute
-            WritePipelineResponse response = client.ingest().deletePipeline(request, RequestOptions.DEFAULT); // <1>
+            AcknowledgedResponse response = client.ingest().deletePipeline(request, RequestOptions.DEFAULT); // <1>
             // end::delete-pipeline-execute
 
             // tag::delete-pipeline-response
@@ -257,10 +257,10 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
             DeletePipelineRequest request = new DeletePipelineRequest("my-pipeline-id");
 
             // tag::delete-pipeline-execute-listener
-            ActionListener<WritePipelineResponse> listener =
-                new ActionListener<WritePipelineResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                new ActionListener<AcknowledgedResponse>() {
                     @Override
-                    public void onResponse(WritePipelineResponse response) {
+                    public void onResponse(AcknowledgedResponse response) {
                         // <1>
                     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java

@@ -19,11 +19,17 @@
 
 package org.elasticsearch.client.documentation;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.Booleans;
+import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.GetLicenseResponse;
 import org.elasticsearch.protocol.xpack.license.LicensesStatus;
 import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
@@ -32,6 +38,8 @@ import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.startsWith;
@@ -42,7 +50,8 @@ import static org.hamcrest.Matchers.startsWith;
  */
 public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase {
 
-    public void testPutLicense() throws Exception {
+    public void testLicense() throws Exception {
+        assumeTrue("License is only valid when tested against snapshot/test builds", Build.CURRENT.isSnapshot());
         RestHighLevelClient client = highLevelClient();
         String license = "{\"license\": {\"uid\":\"893361dc-9749-4997-93cb-802e3d7fa4a8\",\"type\":\"gold\"," +
             "\"issue_date_in_millis\":1411948800000,\"expiry_date_in_millis\":1914278399999,\"max_nodes\":1,\"issued_to\":\"issued_to\"," +
@@ -60,7 +69,7 @@ public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase {
             request.setLicenseDefinition(license);  // <1>
             request.setAcknowledge(false);          // <2>
 
-            PutLicenseResponse response = client.xpack().license().putLicense(request, RequestOptions.DEFAULT);
+            PutLicenseResponse response = client.license().putLicense(request, RequestOptions.DEFAULT);
             //end::put-license-execute
 
             //tag::put-license-response
@@ -80,7 +89,7 @@ public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase {
             // tag::put-license-execute-listener
             ActionListener<PutLicenseResponse> listener = new ActionListener<PutLicenseResponse>() {
                 @Override
-                public void onResponse(PutLicenseResponse indexResponse) {
+                public void onResponse(PutLicenseResponse putLicenseResponse) {
                     // <1>
                 }
 
@@ -96,11 +105,114 @@ public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::put-license-execute-async
-            client.xpack().license().putLicenseAsync(
+            client.license().putLicenseAsync(
                     request, RequestOptions.DEFAULT, listener); // <1>
             // end::put-license-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
         }
+
+        // we cannot actually delete the license, otherwise the remaining tests won't work
+        if (Booleans.isTrue("true")) {
+            return;
+        }
+        {
+            //tag::delete-license-execute
+            DeleteLicenseRequest request = new DeleteLicenseRequest();
+
+            AcknowledgedResponse response = client.license().deleteLicense(request, RequestOptions.DEFAULT);
+            //end::delete-license-execute
+
+            //tag::delete-license-response
+            boolean acknowledged = response.isAcknowledged(); // <1>
+            //end::delete-license-response
+
+            assertTrue(acknowledged);
+        }
+        {
+            DeleteLicenseRequest request = new DeleteLicenseRequest();
+            // tag::delete-license-execute-listener
+            ActionListener<AcknowledgedResponse> listener = new ActionListener<AcknowledgedResponse>() {
+                @Override
+                public void onResponse(AcknowledgedResponse deleteLicenseResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::delete-license-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::delete-license-execute-async
+            client.license().deleteLicenseAsync(
+                request, RequestOptions.DEFAULT, listener); // <1>
+            // end::delete-license-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
+    public void testGetLicense() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        {
+            //tag::get-license-execute
+            GetLicenseRequest request = new GetLicenseRequest();
+
+            GetLicenseResponse response = client.license().getLicense(request, RequestOptions.DEFAULT);
+            //end::get-license-execute
+
+            //tag::get-license-response
+            String currentLicense = response.getLicenseDefinition(); // <1>
+            //end::get-license-response
+
+            assertThat(currentLicense, containsString("trial"));
+            assertThat(currentLicense, containsString("client_rest-high-level_integTestCluster"));
+        }
+        {
+            GetLicenseRequest request = new GetLicenseRequest();
+            // tag::get-license-execute-listener
+            ActionListener<GetLicenseResponse> listener = new ActionListener<GetLicenseResponse>() {
+                @Override
+                public void onResponse(GetLicenseResponse indexResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::get-license-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::get-license-execute-async
+            client.license().getLicenseAsync(
+                request, RequestOptions.DEFAULT, listener); // <1>
+            // end::get-license-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+        {
+            GetLicenseRequest request = new GetLicenseRequest();
+            RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
+            // Make sure that it still works in other formats
+            builder.addHeader("Accept", randomFrom("application/smile", "application/cbor"));
+            RequestOptions options = builder.build();
+            GetLicenseResponse response = client.license().getLicense(request, options);
+            String currentLicense = response.getLicenseDefinition();
+            assertThat(currentLicense, startsWith("{"));
+            assertThat(currentLicense, containsString("trial"));
+            assertThat(currentLicense, containsString("client_rest-high-level_integTestCluster"));
+            assertThat(currentLicense, endsWith("}"));
+        }
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationClientDocumentationIT.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.documentation;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
+import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * This class is used to generate the Java Migration API documentation.
+ * You need to wrap your code between two tags like:
+ * // tag::example
+ * // end::example
+ *
+ * Where example is your tag name.
+ *
+ * Then in the documentation, you can extract what is between tag and end tags with
+ * ["source","java",subs="attributes,callouts,macros"]
+ * --------------------------------------------------
+ * include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[example]
+ * --------------------------------------------------
+ *
+ * The column width of the code block is 84. If the code contains a line longer
+ * than 84, the line will be cut and a horizontal scroll bar will be displayed.
+ * (the code indentation of the tag is not included in the width)
+ */
+public class MigrationClientDocumentationIT extends ESRestHighLevelClientTestCase {
+
+    public void testGetAssistance() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+
+        // tag::get-assistance-request
+        IndexUpgradeInfoRequest request = new IndexUpgradeInfoRequest(); // <1>
+        // end::get-assistance-request
+
+        // tag::get-assistance-request-indices
+        request.indices("index1", "index2"); // <1>
+        // end::get-assistance-request-indices
+
+        request.indices(Strings.EMPTY_ARRAY);
+
+        // tag::get-assistance-request-indices-options
+        request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1>
+        // end::get-assistance-request-indices-options
+
+        // tag::get-assistance-execute
+        IndexUpgradeInfoResponse response = client.migration().getAssistance(request, RequestOptions.DEFAULT);
+        // end::get-assistance-execute
+
+        // tag::get-assistance-response
+        Map<String, UpgradeActionRequired> actions = response.getActions();
+        for (Map.Entry<String, UpgradeActionRequired> entry : actions.entrySet()) {
+            String index = entry.getKey(); // <1>
+            UpgradeActionRequired actionRequired = entry.getValue(); // <2>
+        }
+        // end::get-assistance-response
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -0,0 +1,224 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.documentation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.LatchedActionListener;
+import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.MachineLearningIT;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse;
+import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
+import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
+import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
+import org.elasticsearch.protocol.xpack.ml.PutJobResponse;
+import org.elasticsearch.protocol.xpack.ml.job.config.AnalysisConfig;
+import org.elasticsearch.protocol.xpack.ml.job.config.DataDescription;
+import org.elasticsearch.protocol.xpack.ml.job.config.Detector;
+import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.greaterThan;
+
+public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
+
+    public void testCreateJob() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        //tag::x-pack-ml-put-job-detector
+        Detector.Builder detectorBuilder = new Detector.Builder()
+            .setFunction("sum")                                    // <1>
+            .setFieldName("total")                                 // <2>
+            .setDetectorDescription("Sum of total");               // <3>
+        //end::x-pack-ml-put-job-detector
+
+        //tag::x-pack-ml-put-job-analysis-config
+        List<Detector> detectors = Collections.singletonList(detectorBuilder.build());       // <1>
+        AnalysisConfig.Builder analysisConfigBuilder = new AnalysisConfig.Builder(detectors) // <2>
+            .setBucketSpan(TimeValue.timeValueMinutes(10));                                  // <3>
+        //end::x-pack-ml-put-job-analysis-config
+
+        //tag::x-pack-ml-put-job-data-description
+        DataDescription.Builder dataDescriptionBuilder = new DataDescription.Builder()
+            .setTimeField("timestamp");  // <1>
+        //end::x-pack-ml-put-job-data-description
+
+        {
+            String id = "job_1";
+
+            //tag::x-pack-ml-put-job-config
+            Job.Builder jobBuilder = new Job.Builder(id)      // <1>
+                .setAnalysisConfig(analysisConfigBuilder)     // <2>
+                .setDataDescription(dataDescriptionBuilder)   // <3>
+                .setDescription("Total sum of requests");     // <4>
+            //end::x-pack-ml-put-job-config
+
+            //tag::x-pack-ml-put-job-request
+            PutJobRequest request = new PutJobRequest(jobBuilder.build()); // <1>
+            //end::x-pack-ml-put-job-request
+
+            //tag::x-pack-ml-put-job-execute
+            PutJobResponse response = client.machineLearning().putJob(request, RequestOptions.DEFAULT);
+            //end::x-pack-ml-put-job-execute
+
+            //tag::x-pack-ml-put-job-response
+            Date createTime = response.getResponse().getCreateTime(); // <1>
+            //end::x-pack-ml-put-job-response
+            assertThat(createTime.getTime(), greaterThan(0L));
+        }
+        {
+            String id = "job_2";
+            Job.Builder jobBuilder = new Job.Builder(id)
+                .setAnalysisConfig(analysisConfigBuilder)
+                .setDataDescription(dataDescriptionBuilder)
+                .setDescription("Total sum of requests");
+
+            PutJobRequest request = new PutJobRequest(jobBuilder.build());
+            // tag::x-pack-ml-put-job-execute-listener
+            ActionListener<PutJobResponse> listener = new ActionListener<PutJobResponse>() {
+                @Override
+                public void onResponse(PutJobResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::x-pack-ml-put-job-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::x-pack-ml-put-job-execute-async
+            client.machineLearning().putJobAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::x-pack-ml-put-job-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
+    public void testDeleteJob() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        String jobId = "my-first-machine-learning-job";
+
+        Job job = MachineLearningIT.buildJob(jobId);
+        client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
+
+        Job secondJob = MachineLearningIT.buildJob("my-second-machine-learning-job");
+        client.machineLearning().putJob(new PutJobRequest(secondJob), RequestOptions.DEFAULT);
+
+        {
+            //tag::x-pack-delete-ml-job-request
+            DeleteJobRequest deleteJobRequest = new DeleteJobRequest("my-first-machine-learning-job");
+            deleteJobRequest.setForce(false); //<1>
+            DeleteJobResponse deleteJobResponse = client.machineLearning().deleteJob(deleteJobRequest, RequestOptions.DEFAULT);
+            //end::x-pack-delete-ml-job-request
+
+            //tag::x-pack-delete-ml-job-response
+            boolean isAcknowledged = deleteJobResponse.isAcknowledged(); //<1>
+            //end::x-pack-delete-ml-job-response
+        }
+        {
+            //tag::x-pack-delete-ml-job-request-listener
+            ActionListener<DeleteJobResponse> listener = new ActionListener<DeleteJobResponse>() {
+                @Override
+                public void onResponse(DeleteJobResponse deleteJobResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            //end::x-pack-delete-ml-job-request-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            //tag::x-pack-delete-ml-job-request-async
+            DeleteJobRequest deleteJobRequest = new DeleteJobRequest("my-second-machine-learning-job");
+            client.machineLearning().deleteJobAsync(deleteJobRequest, RequestOptions.DEFAULT, listener); // <1>
+            //end::x-pack-delete-ml-job-request-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
+    public void testOpenJob() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        Job job = MachineLearningIT.buildJob("opening-my-first-machine-learning-job");
+        client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
+
+        Job secondJob = MachineLearningIT.buildJob("opening-my-second-machine-learning-job");
+        client.machineLearning().putJob(new PutJobRequest(secondJob), RequestOptions.DEFAULT);
+
+        {
+            //tag::x-pack-ml-open-job-request
+            OpenJobRequest openJobRequest = new OpenJobRequest("opening-my-first-machine-learning-job"); //<1>
+            openJobRequest.setTimeout(TimeValue.timeValueMinutes(10)); //<2>
+            //end::x-pack-ml-open-job-request
+
+            //tag::x-pack-ml-open-job-execute
+            OpenJobResponse openJobResponse = client.machineLearning().openJob(openJobRequest, RequestOptions.DEFAULT);
+            boolean isOpened = openJobResponse.isOpened(); //<1>
+            //end::x-pack-ml-open-job-execute
+
+        }
+        {
+            //tag::x-pack-ml-open-job-listener
+            ActionListener<OpenJobResponse> listener = new ActionListener<OpenJobResponse>() {
+                @Override
+                public void onResponse(OpenJobResponse openJobResponse) {
+                    //<1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            //end::x-pack-ml-open-job-listener
+            OpenJobRequest openJobRequest = new OpenJobRequest("opening-my-second-machine-learning-job");
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::x-pack-ml-open-job-execute-async
+            client.machineLearning().openJobAsync(openJobRequest, RequestOptions.DEFAULT, listener); //<1>
+            // end::x-pack-ml-open-job-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java

@@ -22,27 +22,25 @@ package org.elasticsearch.client.documentation;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
-import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
-import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
@@ -163,7 +161,7 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
         // end::create-repository-request-verify
 
         // tag::create-repository-execute
-        PutRepositoryResponse response = client.snapshot().createRepository(request, RequestOptions.DEFAULT);
+        AcknowledgedResponse response = client.snapshot().createRepository(request, RequestOptions.DEFAULT);
         // end::create-repository-execute
 
         // tag::create-repository-response
@@ -178,10 +176,10 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
             PutRepositoryRequest request = new PutRepositoryRequest(repositoryName);
 
             // tag::create-repository-execute-listener
-            ActionListener<PutRepositoryResponse> listener =
-                new ActionListener<PutRepositoryResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                new ActionListener<AcknowledgedResponse>() {
                     @Override
-                    public void onResponse(PutRepositoryResponse putRepositoryResponse) {
+                    public void onResponse(AcknowledgedResponse putRepositoryResponse) {
                         // <1>
                     }
 
@@ -388,7 +386,7 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
         // end::delete-repository-request-timeout
 
         // tag::delete-repository-execute
-        DeleteRepositoryResponse response = client.snapshot().deleteRepository(request, RequestOptions.DEFAULT);
+        AcknowledgedResponse response = client.snapshot().deleteRepository(request, RequestOptions.DEFAULT);
         // end::delete-repository-execute
 
         // tag::delete-repository-response
@@ -403,10 +401,10 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
             DeleteRepositoryRequest request = new DeleteRepositoryRequest();
 
             // tag::delete-repository-execute-listener
-            ActionListener<DeleteRepositoryResponse> listener =
-                new ActionListener<DeleteRepositoryResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                new ActionListener<AcknowledgedResponse>() {
                     @Override
-                    public void onResponse(DeleteRepositoryResponse deleteRepositoryResponse) {
+                    public void onResponse(AcknowledgedResponse deleteRepositoryResponse) {
                         // <1>
                     }
 
@@ -752,7 +750,7 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
         // end::delete-snapshot-request-masterTimeout
 
         // tag::delete-snapshot-execute
-        DeleteSnapshotResponse response = client.snapshot().delete(request, RequestOptions.DEFAULT);
+        AcknowledgedResponse response = client.snapshot().delete(request, RequestOptions.DEFAULT);
         // end::delete-snapshot-execute
 
         // tag::delete-snapshot-response
@@ -767,10 +765,10 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
             DeleteSnapshotRequest request = new DeleteSnapshotRequest();
 
             // tag::delete-snapshot-execute-listener
-            ActionListener<DeleteSnapshotResponse> listener =
-                new ActionListener<DeleteSnapshotResponse>() {
+            ActionListener<AcknowledgedResponse> listener =
+                new ActionListener<AcknowledgedResponse>() {
                     @Override
-                    public void onResponse(DeleteSnapshotResponse deleteSnapshotResponse) {
+                    public void onResponse(AcknowledgedResponse deleteSnapshotResponse) {
                         // <1>
                     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java

@@ -23,9 +23,9 @@ import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptResponse;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
@@ -154,7 +154,7 @@ public class StoredScriptsDocumentationIT extends ESRestHighLevelClientTestCase
         // end::delete-stored-script-request-timeout
 
         // tag::delete-stored-script-execute
-        DeleteStoredScriptResponse deleteResponse = client.deleteScript(deleteRequest, RequestOptions.DEFAULT);
+        AcknowledgedResponse deleteResponse = client.deleteScript(deleteRequest, RequestOptions.DEFAULT);
         // end::delete-stored-script-execute
 
         // tag::delete-stored-script-response
@@ -164,10 +164,10 @@ public class StoredScriptsDocumentationIT extends ESRestHighLevelClientTestCase
         putStoredScript("calculate-score", scriptSource);
 
         // tag::delete-stored-script-execute-listener
-        ActionListener<DeleteStoredScriptResponse> listener =
-            new ActionListener<DeleteStoredScriptResponse>() {
+        ActionListener<AcknowledgedResponse> listener =
+            new ActionListener<AcknowledgedResponse>() {
                 @Override
-                public void onResponse(DeleteStoredScriptResponse response) {
+                public void onResponse(AcknowledgedResponse response) {
                     // <1>
                 }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java

@@ -49,7 +49,7 @@ public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase {
                 "}");
             PutWatchRequest request = new PutWatchRequest("my_watch_id", watch, XContentType.JSON);
             request.setActive(false); // <1>
-            PutWatchResponse response = client.xpack().watcher().putWatch(request, RequestOptions.DEFAULT);
+            PutWatchResponse response = client.watcher().putWatch(request, RequestOptions.DEFAULT);
             //end::x-pack-put-watch-execute
 
             //tag::x-pack-put-watch-response
@@ -85,7 +85,7 @@ public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::x-pack-put-watch-execute-async
-            client.xpack().watcher().putWatchAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.watcher().putWatchAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::x-pack-put-watch-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
@@ -94,7 +94,7 @@ public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase {
         {
             //tag::x-pack-delete-watch-execute
             DeleteWatchRequest request = new DeleteWatchRequest("my_watch_id");
-            DeleteWatchResponse response = client.xpack().watcher().deleteWatch(request, RequestOptions.DEFAULT);
+            DeleteWatchResponse response = client.watcher().deleteWatch(request, RequestOptions.DEFAULT);
             //end::x-pack-delete-watch-execute
 
             //tag::x-pack-delete-watch-response
@@ -125,7 +125,7 @@ public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::x-pack-delete-watch-execute-async
-            client.xpack().watcher().deleteWatchAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.watcher().deleteWatchAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::x-pack-delete-watch-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));

client/sniffer/licenses/jackson-core-2.8.10.jar.sha1

@@ -1 +0,0 @@
-eb21a035c66ad307e66ec8fce37f5d50fd62d039

client/sniffer/licenses/jackson-core-2.8.11.jar.sha1

@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b

distribution/bwc/build.gradle

@@ -157,7 +157,7 @@ subprojects {
       environment('JAVA_HOME', getJavaHome(it, 8))
     } else if ("6.2".equals(bwcBranch)) {
       environment('JAVA_HOME', getJavaHome(it, 9))
-    } else if (["6.3", "6.x"].contains(bwcBranch)) {
+    } else if (["6.3", "6.4", "6.x"].contains(bwcBranch)) {
       environment('JAVA_HOME', getJavaHome(it, 10))
     } else {
       environment('JAVA_HOME', project.compilerJavaHome)

distribution/packages/src/common/systemd/elasticsearch.service

@@ -6,6 +6,7 @@ After=network-online.target
 
 [Service]
 RuntimeDirectory=elasticsearch
+PrivateTmp=true
 Environment=ES_HOME=/usr/share/elasticsearch
 Environment=ES_PATH_CONF=${path.conf}
 Environment=PID_DIR=/var/run/elasticsearch

docs/build.gradle

@@ -37,6 +37,13 @@ integTestCluster {
   extraConfigFile 'hunspell/en_US/en_US.dic', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic'
   // Whitelist reindexing from the local node so we can test it.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
+
+  // TODO: remove this for 7.0, this exists to allow the doc examples in 6.x to continue using the defaults
+  systemProperty 'es.scripting.use_java_time', 'false'
+  systemProperty 'es.scripting.update.ctx_in_params', 'false'
+
+  // TODO: remove this deprecation compatibility setting for 7.0
+  systemProperty 'es.aggregations.enable_scripted_metric_agg_param', 'false'
 }
 
 // remove when https://github.com/elastic/elasticsearch/issues/31305 is fixed
@@ -396,25 +403,25 @@ buildRestTests.setups['stored_scripted_metric_script'] = '''
   - do:
       put_script:
         id: "my_init_script"
-        body: { "script": { "lang": "painless", "source": "params._agg.transactions = []" } }
+        body: { "script": { "lang": "painless", "source": "state.transactions = []" } }
   - match: { acknowledged: true }
 
   - do:
       put_script:
         id: "my_map_script"
-        body: { "script": { "lang": "painless", "source": "params._agg.transactions.add(doc.type.value == 'sale' ? doc.amount.value : -1 * doc.amount.value)" } }
+        body: { "script": { "lang": "painless", "source": "state.transactions.add(doc.type.value == 'sale' ? doc.amount.value : -1 * doc.amount.value)" } }
   - match: { acknowledged: true }
 
   - do:
       put_script:
         id: "my_combine_script"
-        body: { "script": { "lang": "painless", "source": "double profit = 0;for (t in params._agg.transactions) { profit += t; } return profit" } }
+        body: { "script": { "lang": "painless", "source": "double profit = 0;for (t in state.transactions) { profit += t; } return profit" } }
   - match: { acknowledged: true }
 
   - do:
       put_script:
         id: "my_reduce_script"
-        body: { "script": { "lang": "painless", "source": "double profit = 0;for (a in params._aggs) { profit += a; } return profit" } }
+        body: { "script": { "lang": "painless", "source": "double profit = 0;for (a in states) { profit += a; } return profit" } }
   - match: { acknowledged: true }
 '''
 

docs/java-api/index.asciidoc

@@ -81,7 +81,7 @@ You need to also include Log4j 2 dependencies:
 <dependency>
     <groupId>org.apache.logging.log4j</groupId>
     <artifactId>log4j-core</artifactId>
-    <version>2.9.1</version>
+    <version>2.11.1</version>
 </dependency>
 --------------------------------------------------
 
@@ -109,7 +109,7 @@ If you want to use another logger than Log4j 2, you can use http://www.slf4j.org
 <dependency>
     <groupId>org.apache.logging.log4j</groupId>
     <artifactId>log4j-to-slf4j</artifactId>
-    <version>2.9.1</version>
+    <version>2.11.1</version>
 </dependency>
 <dependency>
     <groupId>org.slf4j</groupId>

docs/java-api/query-dsl/percolate-query.asciidoc

@@ -49,7 +49,7 @@ XContentBuilder docBuilder = XContentFactory.jsonBuilder().startObject();
 docBuilder.field("content", "This is amazing!");
 docBuilder.endObject(); //End of the JSON root object
 
-PercolateQueryBuilder percolateQuery = new PercolateQueryBuilder("query", "docs", docBuilder.bytes());
+PercolateQueryBuilder percolateQuery = new PercolateQueryBuilder("query", "docs", BytesReference.bytes(docBuilder));
 
 // Percolate, by executing the percolator query in the query dsl:
 SearchResponse response = client().prepareSearch("myIndexName")

docs/java-rest/high-level/licensing/delete-license.asciidoc

@@ -0,0 +1,51 @@
+[[java-rest-high-delete-license]]
+=== Delete License
+
+[[java-rest-high-delete-license-execution]]
+==== Execution
+
+The license can be deleted using the `deleteLicense()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[delete-license-execute]
+--------------------------------------------------
+
+[[java-rest-high-delete-license-response]]
+==== Response
+
+The returned `DeleteLicenseResponse` contains the `acknowledged` flag, which
+returns true if the request was processed by all nodes.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[delete-license-response]
+--------------------------------------------------
+<1> Check the acknowledge flag. It should be true if license deletion is acknowledged.
+
+[[java-rest-high-delete-license-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[delete-license-execute-async]
+--------------------------------------------------
+<1> The `DeleteLicenseRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `DeleteLicenseResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[delete-license-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument

docs/java-rest/high-level/licensing/get-license.asciidoc

@@ -0,0 +1,50 @@
+[[java-rest-high-get-license]]
+=== Get License
+
+[[java-rest-high-get-license-execution]]
+==== Execution
+
+The license can be added or updated using the `getLicense()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[get-license-execute]
+--------------------------------------------------
+
+[[java-rest-high-get-license-response]]
+==== Response
+
+The returned `GetLicenseResponse` contains the license in the JSON format.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[get-license-response]
+--------------------------------------------------
+<1> The text of the license.
+
+[[java-rest-high-get-license-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[get-license-execute-async]
+--------------------------------------------------
+<1> The `GetLicenseRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `GetLicenseResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[get-license-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument

docs/java-rest/high-level/migration/get-assistance.asciidoc

@@ -0,0 +1,49 @@
+[[java-rest-high-migration-get-assistance]]
+=== Migration Get Assistance
+
+[[java-rest-high-migraton-get-assistance-request]]
+==== Index Upgrade Info Request
+
+An `IndexUpgradeInfoRequest` does not require any argument:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[get-assistance-request]
+--------------------------------------------------
+<1> Create a new request instance
+
+==== Optional arguments
+The following arguments can optionally be provided:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[get-assistance-request-indices]
+--------------------------------------------------
+<1> Set the indices to the request
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[get-assistance-request-indices-options]
+--------------------------------------------------
+<1> Set the `IndicesOptions` to control how unavailable indices are resolved and
+how wildcard expressions are expanded
+
+[[java-rest-high-migration-get-assistance-execution]]
+==== Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[get-assistance-execute]
+--------------------------------------------------
+
+[[java-rest-high-migration-get-assistance-response]]
+==== Response
+
+The returned `IndexUpgradeInfoResponse` contains the actions required for each index.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[get-assistance-response]
+--------------------------------------------------
+<1> Retrieve the index
+<2> Retrieve the action required for the migration of the current index

docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc

@@ -12,7 +12,7 @@ retrieved using the `usage()` method:
 include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-usage-execute]
 --------------------------------------------------
 
-[[java-rest-high-x-pack-info-response]]
+[[java-rest-high-x-pack-usage-response]]
 ==== Response
 
 The returned `XPackUsageResponse` contains a `Map` keyed by feature name.

docs/java-rest/high-level/ml/delete-job.asciidoc

@@ -0,0 +1,49 @@
+[[java-rest-high-x-pack-ml-delete-job]]
+=== Delete Job API
+
+[[java-rest-high-x-pack-machine-learning-delete-job-request]]
+==== Delete Job Request
+
+A `DeleteJobRequest` object requires a non-null `jobId` and can optionally set `force`.
+Can be executed as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request]
+---------------------------------------------------
+<1> Use to forcefully delete an opened job;
+this method is quicker than closing and deleting the job.
+Defaults to `false`
+
+[[java-rest-high-x-pack-machine-learning-delete-job-response]]
+==== Delete Job Response
+
+The returned `DeleteJobResponse` object indicates the acknowledgement of the request:
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-response]
+---------------------------------------------------
+<1> `isAcknowledged` was the deletion request acknowledged or not
+
+[[java-rest-high-x-pack-machine-learning-delete-job-async]]
+==== Delete Job Asynchronously
+
+This request can also be made asynchronously.
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request-async]
+---------------------------------------------------
+<1> The `DeleteJobRequest` to execute and the `ActionListener` to alert on completion or error.
+
+The deletion request returns immediately. Once the request is completed, the `ActionListener` is
+called back using the `onResponse` or `onFailure`. The latter indicates some failure occurred when
+making the request.
+
+A typical listener for a `DeleteJobRequest` could be defined as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request-listener]
+---------------------------------------------------
+<1> The action to be taken when it is completed
+<2> What to do when a failure occurs

docs/java-rest/high-level/ml/open-job.asciidoc

@@ -0,0 +1,55 @@
+[[java-rest-high-x-pack-ml-open-job]]
+=== Open Job API
+
+The Open Job API provides the ability to open {ml} jobs in the cluster.
+It accepts a `OpenJobRequest` object and responds
+with a `OpenJobResponse` object.
+
+[[java-rest-high-x-pack-ml-open-job-request]]
+==== Open Job Request
+
+An `OpenJobRequest` object gets created with an existing non-null `jobId`.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-open-job-request]
+--------------------------------------------------
+<1> Constructing a new request referencing an existing `jobId`
+<2> Optionally setting the `timeout` value for how long the
+execution should wait for the job to be opened.
+
+[[java-rest-high-x-pack-ml-open-job-execution]]
+==== Execution
+
+The request can be executed through the `MachineLearningClient` contained
+in the `RestHighLevelClient` object, accessed via the `machineLearningClient()` method.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-open-job-execute]
+--------------------------------------------------
+<1> `isOpened()` from the `OpenJobResponse` indicates if the job was successfully
+opened or not.
+
+[[java-rest-high-x-pack-ml-open-job-execution-async]]
+==== Asynchronous Execution
+
+The request can also be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-open-job-execute-async]
+--------------------------------------------------
+<1> The `OpenJobRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The method does not block and returns immediately. The passed `ActionListener` is used
+to notify the caller of completion. A typical `ActionListner` for `OpenJobResponse` may
+look like
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-open-job-listener]
+--------------------------------------------------
+<1> `onResponse` is called back when the action is completed successfully
+<2> `onFailure` is called back when some unexpected error occurs

docs/java-rest/high-level/ml/put-job.asciidoc

@@ -0,0 +1,161 @@
+[[java-rest-high-x-pack-ml-put-job]]
+=== Put Job API
+
+The Put Job API can be used to create a new {ml} job
+in the cluster. The API accepts a `PutJobRequest` object
+as a request and returns a `PutJobResponse`.
+
+[[java-rest-high-x-pack-ml-put-job-request]]
+==== Put Job Request
+
+A `PutJobRequest` requires the following argument:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-job-request]
+--------------------------------------------------
+<1> The configuration of the {ml} job to create as a `Job`
+
+[[java-rest-high-x-pack-ml-put-job-config]]
+==== Job Configuration
+
+The `Job` object contains all the details about the {ml} job
+configuration.
+
+A `Job` requires the following arguments:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-job-config]
+--------------------------------------------------
+<1> The job ID
+<2> An analysis configuration
+<3> A data description
+<4> Optionally, a human-readable description
+
+[[java-rest-high-x-pack-ml-put-job-analysis-config]]
+==== Analysis Configuration
+
+The analysis configuration of the {ml} job is defined in the `AnalysisConfig`.
+`AnalysisConfig` reflects all the configuration
+settings that can be defined using the REST API.
+
+Using the REST API, we could define this analysis configuration:
+
+[source,js]
+--------------------------------------------------
+"analysis_config" : {
+  "bucket_span" : "10m",
+  "detectors" : [
+    {
+      "detector_description" : "Sum of total",
+      "function" : "sum",
+      "field_name" : "total"
+    }
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+Using the `AnalysisConfig` object and the high level REST client, the list
+of detectors must be built first.
+
+An example of building a `Detector` instance is as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-job-detector]
+--------------------------------------------------
+<1> The function to use
+<2> The field to apply the function to
+<3> Optionally, a human-readable description
+
+Then the same configuration would be:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-job-analysis-config]
+--------------------------------------------------
+<1> Create a list of detectors
+<2> Pass the list of detectors to the analysis config builder constructor
+<3> The bucket span
+
+[[java-rest-high-x-pack-ml-put-job-data-description]]
+==== Data Description
+
+After defining the analysis config, the next thing to define is the
+data description, using a `DataDescription` instance. `DataDescription`
+reflects all the configuration settings that can be defined using the
+REST API.
+
+Using the REST API, we could define this metrics configuration:
+
+[source,js]
+--------------------------------------------------
+"data_description" : {
+  "time_field" : "timestamp"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+Using the `DataDescription` object and the high level REST client, the same
+configuration would be:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-job-data-description]
+--------------------------------------------------
+<1> The time field
+
+[[java-rest-high-x-pack-ml-put-job-execution]]
+==== Execution
+
+The Put Job API can be executed through a `MachineLearningClient`
+instance. Such an instance can be retrieved from a `RestHighLevelClient`
+using the `machineLearning()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-job-execute]
+--------------------------------------------------
+
+[[java-rest-high-x-pack-ml-put-job-response]]
+==== Response
+
+The returned `PutJobResponse` returns the full representation of
+the new {ml} job if it has been successfully created. This will
+contain the creation time and other fields initialized using
+default values:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-job-response]
+--------------------------------------------------
+<1> The creation time is a field that was not passed in the `Job` object in the request
+
+[[java-rest-high-x-pack-ml-put-job-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-job-execute-async]
+--------------------------------------------------
+<1> The `PutMlJobRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `PutJobResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-put-job-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument

docs/java-rest/high-level/search/explain.asciidoc

@@ -80,7 +80,7 @@ A typical listener for `ExplainResponse` is constructed as follows:
 include-tagged::{doc-tests}/SearchDocumentationIT.java[explain-execute-listener]
 --------------------------------------------------
 <1> Called when the execution is successfully completed.
-<2> Called when the whole `FieldCapabilitiesRequest` fails.
+<2> Called when the whole `ExplainRequest` fails.
 
 [[java-rest-high-explain-response]]
 ==== ExplainResponse

docs/java-rest/high-level/supported-apis.asciidoc

@@ -54,14 +54,12 @@ The Java High Level REST Client supports the following Miscellaneous APIs:
 * <<java-rest-high-main>>
 * <<java-rest-high-ping>>
 * <<java-rest-high-x-pack-info>>
-* <<java-rest-high-x-pack-watcher-put-watch>>
-* <<java-rest-high-x-pack-watcher-delete-watch>>
+* <<java-rest-high-x-pack-usage>>
 
 include::miscellaneous/main.asciidoc[]
 include::miscellaneous/ping.asciidoc[]
-include::x-pack/x-pack-info.asciidoc[]
-include::x-pack/watcher/put-watch.asciidoc[]
-include::x-pack/watcher/delete-watch.asciidoc[]
+include::miscellaneous/x-pack-info.asciidoc[]
+include::miscellaneous/x-pack-usage.asciidoc[]
 
 == Indices APIs
 
@@ -190,11 +188,44 @@ The Java High Level REST Client supports the following Scripts APIs:
 include::script/get_script.asciidoc[]
 include::script/delete_script.asciidoc[]
 
-
 == Licensing APIs
 
 The Java High Level REST Client supports the following Licensing APIs:
 
 * <<java-rest-high-put-license>>
+* <<java-rest-high-get-license>>
+* <<java-rest-high-delete-license>>
 
 include::licensing/put-license.asciidoc[]
+include::licensing/get-license.asciidoc[]
+include::licensing/delete-license.asciidoc[]
+
+== Machine Learning APIs
+
+The Java High Level REST Client supports the following Machine Learning APIs:
+
+* <<java-rest-high-x-pack-ml-put-job>>
+* <<java-rest-high-x-pack-ml-delete-job>>
+* <<java-rest-high-x-pack-ml-open-job>>
+
+include::ml/put-job.asciidoc[]
+include::ml/delete-job.asciidoc[]
+include::ml/open-job.asciidoc[]
+
+== Migration APIs
+
+The Java High Level REST Client supports the following Migration APIs:
+
+* <<java-rest-high-migration-get-assistance>>
+
+include::migration/get-assistance.asciidoc[]
+
+== Watcher APIs
+
+The Java High Level REST Client supports the following Watcher APIs:
+
+* <<java-rest-high-x-pack-watcher-put-watch>>
+* <<java-rest-high-x-pack-watcher-delete-watch>>
+
+include::watcher/put-watch.asciidoc[]
+include::watcher/delete-watch.asciidoc[]

docs/java-rest/high-level/watcher/delete-watch.asciidoc

@@ -1,5 +1,5 @@
 [[java-rest-high-x-pack-watcher-delete-watch]]
-=== X-Pack Delete Watch API
+=== Delete Watch API
 
 [[java-rest-high-x-pack-watcher-delete-watch-execution]]
 ==== Execution

docs/java-rest/high-level/watcher/put-watch.asciidoc

@@ -1,5 +1,5 @@
 [[java-rest-high-x-pack-watcher-put-watch]]
-=== X-Pack Put Watch API
+=== Put Watch API
 
 [[java-rest-high-x-pack-watcher-put-watch-execution]]
 ==== Execution

docs/painless/painless-contexts.asciidoc

@@ -14,6 +14,8 @@ specialized code may define new ways to use a Painless script.
 |====
 | Name                              | Painless Documentation
                                     | Elasticsearch Documentation
+| Ingest processor                  | <<painless-ingest-processor-context, Painless Documentation>>
+                                    | {ref}/script-processor.html[Elasticsearch Documentation]
 | Update                            | <<painless-update-context, Painless Documentation>>
                                     | {ref}/docs-update.html[Elasticsearch Documentation]
 | Update by query                   | <<painless-update-by-query-context, Painless Documentation>>
@@ -44,12 +46,12 @@ specialized code may define new ways to use a Painless script.
                                     | {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
 | Bucket aggregation                | <<painless-bucket-agg-context, Painless Documentation>>
                                     | {ref}/search-aggregations-pipeline-bucket-script-aggregation.html[Elasticsearch Documentation]
-| Ingest processor                  | <<painless-ingest-processor-context, Painless Documentation>>
-                                    | {ref}/script-processor.html[Elasticsearch Documentation]
 | Watcher condition                 | <<painless-watcher-condition-context, Painless Documentation>>
                                     | {xpack-ref}/condition-script.html[Elasticsearch Documentation]
 | Watcher transform                 | <<painless-watcher-transform-context, Painless Documentation>>
                                     | {xpack-ref}/transform-script.html[Elasticsearch Documentation]
 |====
 
+include::painless-contexts/painless-context-examples.asciidoc[]
+
 include::painless-contexts/index.asciidoc[]

docs/painless/painless-contexts/index.asciidoc

@@ -1,3 +1,5 @@
+include::painless-ingest-processor-context.asciidoc[]
+
 include::painless-update-context.asciidoc[]
 
 include::painless-update-by-query-context.asciidoc[]
@@ -28,8 +30,6 @@ include::painless-metric-agg-reduce-context.asciidoc[]
 
 include::painless-bucket-agg-context.asciidoc[]
 
-include::painless-ingest-processor-context.asciidoc[]
-
 include::painless-watcher-condition-context.asciidoc[]
 
 include::painless-watcher-transform-context.asciidoc[]

docs/painless/painless-contexts/painless-context-examples.asciidoc

@@ -0,0 +1,80 @@
+[[painless-context-examples]]
+=== Context examples
+
+To run the examples, index the sample seat data into Elasticsearch. The examples
+must be run sequentially to work correctly.
+
+. Download the
+https://download.elastic.co/demos/painless/contexts/seats.json[seat data]. This
+data set contains booking information for a collection of plays. Each document
+represents a single seat for a play at a particular theater on a specific date
+and time.
++
+Each document contains the following fields:
++
+`theatre` ({ref}/keyword.html[`keyword`])::
+        The name of the theater the play is in.
+`play` ({ref}/text.html[`text`])::
+        The name of the play.
+`actors` ({ref}/text.html[`text`])::
+        A list of actors in the play.
+`row` ({ref}/number.html[`integer`])::
+        The row of the seat.
+`number` ({ref}/number.html[`integer`])::
+        The number of the seat within a row.
+`cost` ({ref}/number.html[`double`])::
+        The cost of the ticket for the seat.
+`sold` ({ref}/boolean.html[`boolean`])::
+        Whether or not the seat is sold.
+`datetime` ({ref}/date.html[`date`])::
+        The date and time of the play as a date object.
+`date` ({ref}/keyword.html[`keyword`])::
+        The date of the play as a keyword.
+`time` ({ref}/keyword.html[`keyword`])::
+        The time of the play as a keyword.
+
+. {defguide}/running-elasticsearch.html[Start] Elasticsearch. Note these
+examples assume Elasticsearch and Kibana are running locally. To use the Console
+editor with a remote Kibana instance, click the settings icon and enter the
+Console URL. To submit a cURL request to a remote Elasticsearch instance, edit
+the request URL.
+
+. Create {ref}/mapping.html[mappings] for the sample data:
++
+[source,js]
+----
+PUT /seats
+{
+  "mappings": {
+    "seat": {
+      "properties": {
+        "theatre":  { "type": "keyword" },
+        "play":     { "type": "text"    },
+        "actors":   { "type": "text"    },
+        "row":      { "type": "integer" },
+        "number":   { "type": "integer" },
+        "cost":     { "type": "double"  },
+        "sold":     { "type": "boolean" },
+        "datetime": { "type": "date"    },
+        "date":     { "type": "keyword" },
+        "time":     { "type": "keyword" }
+      }
+    }
+  }
+}
+----
++
+// CONSOLE
+
+. Run the <<painless-ingest-processor-context, ingest processor context>>
+example. This sets up a script ingest processor used on each document as the
+seat data is indexed.
+
+. Index the seat data:
++
+[source,js]
+----
+curl -XPOST localhost:9200/seats/seat/_bulk?pipeline=seats -H "Content-Type: application/x-ndjson" --data-binary "@/<local-file-path>/seats.json"
+----
+// NOTCONSOLE
+

docs/painless/painless-contexts/painless-ingest-processor-context.asciidoc

@@ -27,7 +27,7 @@ to modify documents upon insertion.
 {ref}/mapping-type-field.html[`ctx['_type']`]::
         Modify this to change the type for the current document.
 
-`ctx` (`Map`, read-only)::
+`ctx` (`Map`)::
         Modify the values in the `Map/List` structure to add, modify, or delete
         the fields of a document.
 
@@ -38,4 +38,158 @@ void::
 
 *API*
 
-The standard <<painless-api-reference, Painless API>> is available.
+The standard <<painless-api-reference, Painless API>> is available.
+
+*Example*
+
+To run this example, first follow the steps in
+<<painless-context-examples, context examples>>.
+
+The seat data contains:
+
+* A date in the format `YYYY-MM-DD` where the second digit of both month and day
+  is optional.
+* A time in the format HH:MM* where the second digit of both hours and minutes
+  is optional. The star (*) represents either the `String` `AM` or `PM`.
+
+The following ingest script processes the date and time `Strings` and stores the
+result in a `datetime` field.
+
+[source,Painless]
+----
+String[] split(String s, char d) {                                   <1>
+    int count = 0;
+
+    for (char c : s.toCharArray()) {                                 <2>
+        if (c == d) {
+            ++count;
+        }
+    }
+
+    if (count == 0) {
+        return new String[] {s};                                     <3>
+    }
+
+    String[] r = new String[count + 1];                              <4>
+    int i0 = 0, i1 = 0;
+    count = 0;
+
+    for (char c : s.toCharArray()) {                                 <5>
+        if (c == d) {
+            r[count++] = s.substring(i0, i1);
+            i0 = i1 + 1;
+        }
+
+        ++i1;
+    }
+
+    r[count] = s.substring(i0, i1);                                  <6>
+
+    return r;
+}
+
+String[] dateSplit = split(ctx.date, (char)"-");                     <7>
+String year = dateSplit[0].trim();
+String month = dateSplit[1].trim();
+
+if (month.length() == 1) {                                           <8>
+    month = "0" + month;
+}
+
+String day = dateSplit[2].trim();
+
+if (day.length() == 1) {                                             <9>
+    day = "0" + day;
+}
+
+boolean pm = ctx.time.substring(ctx.time.length() - 2).equals("PM"); <10>
+String[] timeSplit = split(
+        ctx.time.substring(0, ctx.time.length() - 2), (char)":");    <11>
+int hours = Integer.parseInt(timeSplit[0].trim());
+int minutes = Integer.parseInt(timeSplit[1].trim());
+
+if (pm) {                                                            <12>
+    hours += 12;
+}
+
+String dts = year + "-" + month + "-" + day + "T" +
+        (hours < 10 ? "0" + hours : "" + hours) + ":" +
+        (minutes < 10 ? "0" + minutes : "" + minutes) +
+        ":00+08:00";                                                 <13>
+
+ZonedDateTime dt = ZonedDateTime.parse(
+         dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME);               <14>
+ctx.datetime = dt.getLong(ChronoField.INSTANT_SECONDS)*1000L;        <15>
+----
+<1> Creates a `split` <<painless-functions, function>> to split a
+    <<string-type, `String`>> type value using a <<primitive-types, `char`>>
+    type value as the delimiter. This is useful for handling the necessity of
+    pulling out the individual pieces of the date and time `Strings` from the
+    original seat data.
+<2> The first pass through each `char` in the `String` collects how many new
+    `Strings` the original is split into.
+<3> Returns the original `String` if there are no instances of the delimiting
+    `char`.
+<4> Creates an <<array-type, array type>> value to collect the split `Strings`
+    into based on the number of `char` delimiters found in the first pass.
+<5> The second pass through each `char` in the `String` collects each split
+    substring into an array type value of `Strings`.
+<6> Collects the last substring into the array type value of `Strings`.
+<7> Uses the `split` function to separate the date `String` from the seat data
+    into year, month, and day `Strings`.
+    Note::
+    * The use of a `String` type value to `char` type value
+      <<string-character-casting, cast>> as part of the second argument since
+      character literals do not exist.
+    * The use of the `ctx` ingest processor context variable to retrieve the
+      data from the `date` field.
+<8> Appends the <<string-literals, string literal>> `"0"` value to a single
+    digit month since the format of the seat data allows for this case.
+<9> Appends the <<string-literals, string literal>> `"0"` value to a single
+    digit day since the format of the seat data allows for this case.
+<10> Sets the <<primitive-types, `boolean type`>>
+     <<painless-variables, variable>> to `true` if the time `String` is a time
+     in the afternoon or evening.
+     Note::
+     * The use of the `ctx` ingest processor context variable to retrieve the
+       data from the `time` field.
+<11> Uses the `split` function to separate the time `String` from the seat data
+     into hours and minutes `Strings`.
+     Note::
+     * The use of the `substring` method to remove the `AM` or `PM` portion of
+       the time `String`.
+     * The use of a `String` type value to `char` type value
+       <<string-character-casting, cast>> as part of the second argument since
+       character literals do not exist.
+     * The use of the `ctx` ingest processor context variable to retrieve the
+       data from the `date` field.
+<12> If the time `String` is an afternoon or evening value adds the
+     <<integer-literals, integer literal>> `12` to the existing hours to move to
+     a 24-hour based time.
+<13> Builds a new time `String` that is parsable using existing API methods.
+<14> Creates a `ZonedDateTime` <<reference-types, reference type>> value by using
+     the API method `parse` to parse the new time `String`.
+<15> Sets the datetime field `datetime` to the number of milliseconds retrieved
+     from the API method `getLong`.
+     Note::
+     * The use of the `ctx` ingest processor context variable to set the field
+       `datetime`. Manipulate each document's fields with the `ctx` variable as
+       each document is indexed.
+
+Submit the following request:
+
+[source,js]
+----
+PUT /_ingest/pipeline/seats
+{
+    "description": "update datetime for seats",
+    "processors": [
+      {
+        "script": {
+          "source": "String[] split(String s, char d) { int count = 0; for (char c : s.toCharArray()) { if (c == d) { ++count; } } if (count == 0) { return new String[] {s}; } String[] r = new String[count + 1]; int i0 = 0, i1 = 0; count = 0; for (char c : s.toCharArray()) { if (c == d) { r[count++] = s.substring(i0, i1); i0 = i1 + 1; } ++i1; } r[count] = s.substring(i0, i1); return r; } String[] dateSplit = split(ctx.date, (char)\"-\"); String year = dateSplit[0].trim(); String month = dateSplit[1].trim(); if (month.length() == 1) { month = \"0\" + month; } String day = dateSplit[2].trim(); if (day.length() == 1) { day = \"0\" + day; } boolean pm = ctx.time.substring(ctx.time.length() - 2).equals(\"PM\"); String[] timeSplit = split(ctx.time.substring(0, ctx.time.length() - 2), (char)\":\"); int hours = Integer.parseInt(timeSplit[0].trim()); int minutes = Integer.parseInt(timeSplit[1].trim()); if (pm) { hours += 12; } String dts = year + \"-\" + month + \"-\" + day + \"T\" + (hours < 10 ? \"0\" + hours : \"\" + hours) + \":\" + (minutes < 10 ? \"0\" + minutes : \"\" + minutes) + \":00+08:00\"; ZonedDateTime dt = ZonedDateTime.parse(dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME); ctx.datetime = dt.getLong(ChronoField.INSTANT_SECONDS)*1000L;"
+        }
+      }
+    ]
+}
+----
+// CONSOLE

docs/painless/painless-contexts/painless-metric-agg-combine-context.asciidoc

@@ -12,7 +12,7 @@ optional as part of a full metric aggregation.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`params['_agg']` (`Map`)::
+`state` (`Map`)::
         `Map` with values available from the prior map script.
 
 *Return*

docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc

@@ -12,13 +12,13 @@ full metric aggregation.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`params['_agg']` (`Map`)::
+`state` (`Map`)::
         Empty `Map` used to add values for use in a
         <<painless-metric-agg-map-context, map script>>.
 
 *Side Effects*
 
-`params['_agg']` (`Map`)::
+`state` (`Map`)::
         Add values to this `Map` to for use in a map.  Additional values must
         be of the type `Map`, `List`, `String` or primitive.
 

docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc

@@ -13,10 +13,9 @@ part of a full metric aggregation.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`params['_agg']` (`Map`)::
+`state` (`Map`)::
         `Map` used to add values for processing in a
-        <<painless-metric-agg-map-context, combine script>> or returned
-        directly.
+        <<painless-metric-agg-map-context, combine script>> or to be returned from the aggregation.
 
 `doc` (`Map`, read-only)::
         Contains the fields of the current document where each field is a
@@ -27,15 +26,16 @@ part of a full metric aggregation.
 
 *Side Effects*
 
-`params['_agg']` (`Map`)::
+`state` (`Map`)::
         Use this `Map` to add values for processing in a combine script.
         Additional values must be of the type `Map`, `List`, `String` or
-        primitive. If an initialization script is provided as part the
+        primitive. The same `state` `Map` is shared between all aggregated documents
+        on a given shard. If an initialization script is provided as part of the
         aggregation then values added from the initialization script are
-        available as well.  If no combine script is specified, values must be
-        directly stored in `_agg`. If no combine script and no
+        available.  If no combine script is specified, values must be
+        directly stored in `state` in a usable form. If no combine script and no
         <<painless-metric-agg-reduce-context, reduce script>> are specified, the
-        values are used as the result.
+        `state` values are used as the result.
 
 *Return*
 

docs/painless/painless-contexts/painless-metric-agg-reduce-context.asciidoc

@@ -14,7 +14,7 @@ specified) and is optional as part of a full metric aggregation.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`params['_aggs']` (`Map`)::
+`states` (`Map`)::
         `Map` with values available from the prior combine script (or a map
         script if no combine script is specified).
 

docs/painless/painless-getting-started.asciidoc

@@ -198,7 +198,7 @@ POST hockey/player/1/_update
 ==== Dates
 
 Date fields are exposed as
-`ReadableDateTime`
+`ReadableDateTime` or
 so they support methods like
 `getYear`,
 and `getDayOfWeek`.
@@ -220,6 +220,11 @@ GET hockey/_search
 }
 ----------------------------------------------------------------
 // CONSOLE
+// TEST[warning:The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true to use the java time api for date field doc values]
+
+NOTE: Date fields are changing in 7.0 to be exposed as `ZonedDateTime`
+from Java 8's time API. To switch to this functionality early, 
+add `-Des.scripting.use_java_time=true` to `jvm.options`.
 
 [float]
 [[modules-scripting-painless-regex]]

docs/painless/painless-keywords.asciidoc

@@ -5,7 +5,7 @@ Keywords are reserved tokens for built-in language features.
 
 *Errors*
 
-If a keyword is used as an <<painless-identifiers, identifier>>.
+* If a keyword is used as an <<painless-identifiers, identifier>>.
 
 *Keywords*
 

docs/plugins/integrations.asciidoc

@@ -17,14 +17,11 @@ Integrations are not plugins, but are external tools or modules that make it eas
 * https://drupal.org/project/elasticsearch_connector[Drupal]:
   Drupal Elasticsearch integration.
 
-* https://wordpress.org/plugins/wpsolr-search-engine/[WPSOLR]:
-  Elasticsearch (and Apache Solr) WordPress Plugin
- 
-* http://searchbox-io.github.com/wp-elasticsearch/[Wp-Elasticsearch]:
+* https://wordpress.org/plugins/elasticpress/[ElasticPress]:
   Elasticsearch WordPress Plugin
 
-* https://github.com/wallmanderco/elasticsearch-indexer[Elasticsearch Indexer]:
-  Elasticsearch WordPress Plugin
+* https://wordpress.org/plugins/wpsolr-search-engine/[WPSOLR]:
+  Elasticsearch (and Apache Solr) WordPress Plugin
 
 * https://doc.tiki.org/Elasticsearch[Tiki Wiki CMS Groupware]:
   Tiki has native support for Elasticsearch. This provides faster & better

docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc

@@ -425,6 +425,7 @@ POST /sales/_search?size=0
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:sales]
+// TEST[warning:The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true to use the java time api for date field doc values]
 
 Response:
 

docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc

@@ -87,7 +87,7 @@ The above example produces the following term:
 
 [source,text]
 ---------------------------
-[ My, credit, card, is 123_456_789 ]
+[ My, credit, card, is, 123_456_789 ]
 ---------------------------
 
 WARNING: Using a replacement string that changes the length of the original

docs/reference/cluster/stats.asciidoc

@@ -22,6 +22,7 @@ Will return, for example:
       "successful" : 1,
       "failed" : 0
    },
+   "cluster_uuid": "YjAvIhsCQ9CbjWZb2qJw3Q",
    "cluster_name": "elasticsearch",
    "timestamp": 1459427693515,
    "status": "green",

docs/reference/docs/update.asciidoc

@@ -47,7 +47,7 @@ POST test/_doc/1/_update
 // TEST[continued]
 
 We can add a tag to the list of tags (note, if the tag exists, it
-will still add it, since its a list):
+will still add it, since it's a list):
 
 [source,js]
 --------------------------------------------------
@@ -65,6 +65,28 @@ POST test/_doc/1/_update
 // CONSOLE
 // TEST[continued]
 
+We can remove a tag from the list of tags. Note that the Painless function to
+`remove` a tag takes as its parameter the array index of the element you wish
+to remove, so you need a bit more logic to locate it while avoiding a runtime
+error. Note that if the tag was present more than once in the list, this will
+remove only one occurrence of it:
+
+[source,js]
+--------------------------------------------------
+POST test/_doc/1/_update
+{
+    "script" : {
+        "source": "if (ctx._source.tags.contains(params.tag)) { ctx._source.tags.remove(ctx._source.tags.indexOf(params.tag)) }",
+        "lang": "painless",
+        "params" : {
+            "tag" : "blue"
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
 In addition to `_source`, the following variables are available through
 the `ctx` map: `_index`, `_type`, `_id`, `_version`, `_routing`
 and `_now` (the current timestamp).
@@ -172,7 +194,7 @@ the request was ignored.
    "_index": "test",
    "_type": "_doc",
    "_id": "1",
-   "_version": 6,
+   "_version": 7,
    "result": "noop"
 }
 --------------------------------------------------

docs/reference/getting-started.asciidoc

@@ -23,7 +23,7 @@ There are a few concepts that are core to Elasticsearch. Understanding these con
 [float]
 === Near Realtime (NRT)
 
-Elasticsearch is a near real time search platform. What this means is there is a slight latency (normally one second) from the time you index a document until the time it becomes searchable.
+Elasticsearch is a near-realtime search platform. What this means is there is a slight latency (normally one second) from the time you index a document until the time it becomes searchable.
 
 [float]
 === Cluster
@@ -59,7 +59,7 @@ In a single cluster, you can define as many indexes as you want.
 
 deprecated[6.0.0,See <<removal-of-types>>]
 
-A type used to be a logical category/partition of your index to allow you to store different types of documents in the same index, eg one type for users, another type for blog posts.  It is no longer possible to create multiple types in an index, and the whole concept of types will be removed in a later version.  See <<removal-of-types>> for more.
+A type used to be a logical category/partition of your index to allow you to store different types of documents in the same index, e.g. one type for users, another type for blog posts.  It is no longer possible to create multiple types in an index, and the whole concept of types will be removed in a later version.  See <<removal-of-types>> for more.
 
 [float]
 === Document
@@ -93,7 +93,8 @@ Replication is important for two primary reasons:
 
 
 To summarize, each index can be split into multiple shards. An index can also be replicated zero (meaning no replicas) or more times. Once replicated, each index will have primary shards (the original shards that were replicated from) and replica shards (the copies of the primary shards).
-The number of shards and replicas can be defined per index at the time the index is created. After the index is created, you may change the number of replicas dynamically anytime but you cannot change the number of shards after-the-fact.
+
+The number of shards and replicas can be defined per index at the time the index is created. After the index is created, you may also change the number of replicas dynamically anytime. You can change the number of shards for an existing index using the {ref}/indices-shrink-index.html[`_shrink`] and {ref}/indices-split-index.html[`_split`] APIs, however this is not a trivial task and pre-planning for the correct number of shards is the optimal approach.
 
 By default, each index in Elasticsearch is allocated one primary shard and one replica which means that if you have at least two nodes in your cluster, your index will have one primary shard and another replica shard (one complete replica) for a total of two shards per index.
 
@@ -1065,7 +1066,7 @@ In the previous section, we skipped over a little detail called the document sco
 
 But queries do not always need to produce scores, in particular when they are only used for "filtering" the document set. Elasticsearch detects these situations and automatically optimizes query execution in order not to compute useless scores.
 
-The {ref}/query-dsl-bool-query.html[`bool` query] that we introduced in the previous section also supports `filter` clauses which allow to use a query to restrict the documents that will be matched by other clauses, without changing how scores are computed. As an example, let's introduce the {ref}/query-dsl-range-query.html[`range` query], which allows us to filter documents by a range of values. This is generally used for numeric or date filtering.
+The {ref}/query-dsl-bool-query.html[`bool` query] that we introduced in the previous section also supports `filter` clauses which allow us to use a query to restrict the documents that will be matched by other clauses, without changing how scores are computed. As an example, let's introduce the {ref}/query-dsl-range-query.html[`range` query], which allows us to filter documents by a range of values. This is generally used for numeric or date filtering.
 
 This example uses a bool query to return all accounts with balances between 20000 and 30000, inclusive. In other words, we want to find accounts with a balance that is greater than or equal to 20000 and less than or equal to 30000.
 

docs/reference/ingest/ingest-node.asciidoc

@@ -1049,6 +1049,125 @@ understands this to mean `2016-04-01` as is explained in the <<date-math-index-n
 | `index_name_format`    | no        | yyyy-MM-dd                   | The format to be used when printing the parsed date into the index name. An valid Joda pattern is expected here.
 |======
 
+[[dot-expand-processor]]
+=== Dot Expander Processor
+
+Expands a field with dots into an object field. This processor allows fields
+with dots in the name to be accessible by other processors in the pipeline.
+Otherwise these <<accessing-data-in-pipelines,fields>> can't be accessed by any processor.
+
+[[dot-expender-options]]
+.Dot Expand Options
+[options="header"]
+|======
+| Name     | Required  | Default  | Description
+| `field`  | yes       | -        | The field to expand into an object field
+| `path`   | no        | -        | The field that contains the field to expand. Only required if the field to expand is part another object field, because the `field` option can only understand leaf fields.
+|======
+
+[source,js]
+--------------------------------------------------
+{
+  "dot_expander": {
+    "field": "foo.bar"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+For example the dot expand processor would turn this document:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo.bar" : "value"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+into:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo" : {
+    "bar" : "value"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+If there is already a `bar` field nested under `foo` then
+this processor merges the `foo.bar` field into it. If the field is
+a scalar value then it will turn that field into an array field.
+
+For example, the following document:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo.bar" : "value2",
+  "foo" : {
+    "bar" : "value1"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+is transformed by the `dot_expander` processor into:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo" : {
+    "bar" : ["value1", "value2"]
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+If any field outside of the leaf field conflicts with a pre-existing field of the same name,
+then that field needs to be renamed first.
+
+Consider the following document:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo": "value1",
+  "foo.bar": "value2"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+Then the `foo` needs to be renamed first before the `dot_expander`
+processor is applied. So in order for the `foo.bar` field to properly
+be expanded into the `bar` field under the `foo` field the following
+pipeline should be used:
+
+[source,js]
+--------------------------------------------------
+{
+  "processors" : [
+    {
+      "rename" : {
+        "field" : "foo",
+        "target_field" : "foo.bar""
+      }
+    },
+    {
+      "dot_expander": {
+        "field": "foo.bar"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+The reason for this is that Ingest doesn't know how to automatically cast
+a scalar field to an object field.
+
 [[fail-processor]]
 === Fail Processor
 Raises an exception. This is useful for when
@@ -2058,125 +2177,6 @@ Converts a string to its uppercase equivalent.
 --------------------------------------------------
 // NOTCONSOLE
 
-[[dot-expand-processor]]
-=== Dot Expander Processor
-
-Expands a field with dots into an object field. This processor allows fields
-with dots in the name to be accessible by other processors in the pipeline.
-Otherwise these <<accessing-data-in-pipelines,fields>> can't be accessed by any processor.
-
-[[dot-expender-options]]
-.Dot Expand Options
-[options="header"]
-|======
-| Name     | Required  | Default  | Description
-| `field`  | yes       | -        | The field to expand into an object field
-| `path`   | no        | -        | The field that contains the field to expand. Only required if the field to expand is part another object field, because the `field` option can only understand leaf fields.
-|======
-
-[source,js]
---------------------------------------------------
-{
-  "dot_expander": {
-    "field": "foo.bar"
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-For example the dot expand processor would turn this document:
-
-[source,js]
---------------------------------------------------
-{
-  "foo.bar" : "value"
-}
---------------------------------------------------
-// NOTCONSOLE
-
-into:
-
-[source,js]
---------------------------------------------------
-{
-  "foo" : {
-    "bar" : "value"
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-If there is already a `bar` field nested under `foo` then
-this processor merges the `foo.bar` field into it. If the field is
-a scalar value then it will turn that field into an array field.
-
-For example, the following document:
-
-[source,js]
---------------------------------------------------
-{
-  "foo.bar" : "value2",
-  "foo" : {
-    "bar" : "value1"
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-is transformed by the `dot_expander` processor into:
-
-[source,js]
---------------------------------------------------
-{
-  "foo" : {
-    "bar" : ["value1", "value2"]
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-If any field outside of the leaf field conflicts with a pre-existing field of the same name,
-then that field needs to be renamed first.
-
-Consider the following document:
-
-[source,js]
---------------------------------------------------
-{
-  "foo": "value1",
-  "foo.bar": "value2"
-}
---------------------------------------------------
-// NOTCONSOLE
-
-Then the `foo` needs to be renamed first before the `dot_expander`
-processor is applied. So in order for the `foo.bar` field to properly
-be expanded into the `bar` field under the `foo` field the following
-pipeline should be used:
-
-[source,js]
---------------------------------------------------
-{
-  "processors" : [
-    {
-      "rename" : {
-        "field" : "foo",
-        "target_field" : "foo.bar""
-      }
-    },
-    {
-      "dot_expander": {
-        "field": "foo.bar"
-      }
-    }
-  ]
-}
---------------------------------------------------
-// NOTCONSOLE
-
-The reason for this is that Ingest doesn't know how to automatically cast
-a scalar field to an object field.
-
 [[urldecode-processor]]
 === URL Decode Processor
 URL-decodes a string

docs/reference/migration/migrate_7_0/search.asciidoc

@@ -92,6 +92,9 @@ deprecated in 6.x, has been removed. Context enabled suggestion queries
 without contexts have to visit every suggestion, which degrades the search performance
 considerably.
 
+For geo context the value of the `path` parameter is now validated against the mapping,
+and the context is only accepted if `path` points to a field with `geo_point` type.
+
 ==== Semantics changed for `max_concurrent_shard_requests`
 
 `max_concurrent_shard_requests` used to limit the total number of concurrent shard

docs/reference/redirects.asciidoc

@@ -503,3 +503,31 @@ guide to the {painless}/index.html[Painless Scripting Language].
 
 See the {painless}/painless-api-reference.html[Painless API Reference] in
 the guide to the {painless}/index.html[Painless Scripting Language].
+
+[role="exclude", id="security-api-roles"]
+=== Role management APIs
+
+You can use the following APIs to add, remove, and retrieve roles in the native realm:
+
+* <<security-api-put-role,Create role>>, <<security-api-delete-role,Delete role>>
+* <<security-api-clear-role-cache,Clear roles cache>>
+* <<security-api-get-role,Get roles>>
+
+[role="exclude",id="security-api-tokens"]
+=== Token management APIs
+
+You can use the following APIs to create and invalidate bearer tokens for access
+without requiring basic authentication:
+
+* <<security-api-get-token,Get token>>, <<security-api-invalidate-token,Invalidate token>>
+
+[role="exclude",id="security-api-users"]
+=== User Management APIs
+
+You can use the following APIs to create, read, update, and delete users from the
+native realm:
+
+* <<security-api-put-user,Create users>>, <<security-api-delete-user,Delete users>>
+* <<security-api-enable-user,Enable users>>, <<security-api-disable-user,Disable users>>
+* <<security-api-change-password,Change passwords>>
+* <<security-api-get-user,Get users>>

docs/reference/release-notes/7.0.0-alpha1.asciidoc

@@ -21,4 +21,10 @@ Aggregations::
 * The Percentiles and PercentileRanks aggregations now return `null` in the REST response,
   instead of `NaN`.  This makes it consistent with the rest of the aggregations.  Note:
   this only applies to the REST response, the java objects continue to return `NaN` (also
-  consistent with other aggregations)
+  consistent with other aggregations)
+
+Suggesters::
+* Plugins that register suggesters can now define their own types of suggestions and must
+  explicitly indicate the type of suggestion that they produce. Existing plugins will
+  require changes to their plugin registration. See the `custom-suggester` example
+  plugin {pull}30284[#30284]

docs/reference/search/request-body.asciidoc

@@ -90,7 +90,8 @@ And here is a sample response:
 
     Set to `false` to return an overall failure if the request would produce partial 
     results. Defaults to true, which will allow partial results in the case of timeouts
-    or partial failures.
+    or partial failures. This default can be controlled using the cluster-level setting
+    `search.default_allow_partial_results`.
 
 `terminate_after`::
 

docs/reference/search/uri-request.asciidoc

@@ -125,5 +125,6 @@ more details on the different types of search that can be performed.
 
 |`allow_partial_search_results` |Set to `false` to return an overall failure if the request would produce
 partial results. Defaults to true, which will allow partial results in the case of timeouts
-or partial failures..
+or partial failures. This default can be controlled using the cluster-level setting
+`search.default_allow_partial_results`.
 |=======================================================================

docs/reference/setup/important-settings.asciidoc

@@ -14,6 +14,7 @@ The following settings *must* be considered before going to production:
 * <<heap-size,Heap size>>
 * <<heap-dump-path,Heap dump path>>
 * <<gc-logging,GC logging>>
+* <<es-tmpdir,Temp directory>>
 
 include::important-settings/path-settings.asciidoc[]
 
@@ -31,4 +32,6 @@ include::important-settings/heap-dump-path.asciidoc[]
 
 include::important-settings/gc-logging.asciidoc[]
 
+include::important-settings/es-tmpdir.asciidoc[]
+
 include::important-settings/error-file.asciidoc[]

docs/reference/setup/important-settings/es-tmpdir.asciidoc

@@ -0,0 +1,23 @@
+[[es-tmpdir]]
+=== Temp directory
+
+By default, Elasticsearch uses a private temporary directory that the startup
+script creates immediately below the system temporary directory.
+
+On some Linux distributions a system utility will clean files and directories
+from `/tmp` if they have not been recently accessed. This can lead to the
+private temporary directory being removed while Elasticsearch is running if
+features that require the temporary directory are not used for a long time.
+This causes problems if a feature that requires the temporary directory is
+subsequently used.
+
+If you install Elasticsearch using the `.deb` or `.rpm` packages and run it
+under `systemd` then the private temporary directory that Elasticsearch uses
+is excluded from periodic cleanup.
+
+However, if you intend to run the `.tar.gz` distribution on Linux for an
+extended period then you should consider creating a dedicated temporary
+directory for Elasticsearch that is not under a path that will have old files
+and directories cleaned from it. This directory should have permissions set
+so that only the user that Elasticsearch runs as can access it. Then set the
+`$ES_TMPDIR` environment variable to point to it before starting Elasticsearch.

docs/reference/setup/important-settings/heap-dump-path.asciidoc

@@ -8,8 +8,8 @@ distributions, and the `data` directory under the root of the
 Elasticsearch installation for the <<zip-targz,tar and zip>> archive
 distributions). If this path is not suitable for receiving heap dumps,
 you should modify the entry `-XX:HeapDumpPath=...` in
-<<jvm-options,`jvm.options`>>. If you specify a fixed filename instead
-of a directory, the JVM will repeatedly use the same file; this is one
-mechanism for preventing heap dumps from accumulating in the heap dump
-path. Alternatively, you can configure a scheduled task via your OS to
-remove heap dumps that are older than a configured age.
+<<jvm-options,`jvm.options`>>. If you specify a directory, the JVM
+will generate a filename for the heap dump based on the PID of the running
+instance. If you specify a fixed filename instead of a directory, the file must
+not exist when the JVM needs to perform a heap dump on an out of memory
+exception, otherwise the heap dump will fail.

docs/reference/setup/important-settings/network-host.asciidoc

@@ -9,7 +9,7 @@ location on a single node.  This can be useful for testing Elasticsearch's
 ability to form clusters, but it is not a configuration recommended for
 production.
 
-In order to communicate and to form a cluster with nodes on other servers, your
+In order to form a cluster with nodes on other servers, your
 node will need to bind to a non-loopback address. While there are many
 <<modules-network,network settings>>, usually all you need to configure is
 `network.host`:

docs/reference/setup/important-settings/node-name.asciidoc

@@ -2,7 +2,7 @@
 === `node.name`
 
 By default, Elasticsearch will use the first seven characters of the randomly
-generated UUID as the node id.Note that the node id is persisted and does
+generated UUID as the node id. Note that the node id is persisted and does
 not change when a node restarts and therefore the default node name will also
 not change.
 
@@ -19,4 +19,4 @@ The `node.name` can also be set to the server's HOSTNAME as follows:
 [source,yaml]
 --------------------------------------------------
 node.name: ${HOSTNAME}
---------------------------------------------------
+--------------------------------------------------

docs/reference/setup/install/docker.asciidoc

@@ -4,9 +4,9 @@
 {es} is also available as Docker images.
 The images use https://hub.docker.com/_/centos/[centos:7] as the base image.
 
-A list of all published Docker images and tags can be found in
-https://www.docker.elastic.co[www.docker.elastic.co]. The source code can be found
-on https://github.com/elastic/elasticsearch-docker/tree/{branch}[GitHub].
+A list of all published Docker images and tags is available at
+https://www.docker.elastic.co[www.docker.elastic.co]. The source code is in
+https://github.com/elastic/elasticsearch-docker/tree/{branch}[GitHub].
 
 These images are free to use under the Elastic license. They contain open source 
 and free commercial features and access to paid commercial features.  
@@ -29,15 +29,13 @@ endif::[]
 
 ifeval::["{release-state}"!="unreleased"]
 
-For example, the Docker image can be retrieved with the following command:
-
 ["source","sh",subs="attributes"]
 --------------------------------------------
 docker pull {docker-repo}:{version}
 --------------------------------------------
 
 Alternatively, you can download other Docker images that contain only features 
-that are available under the Apache 2.0 license from 
+available under the Apache 2.0 license. To download the images, go to 
 https://www.docker.elastic.co[www.docker.elastic.co]. 
 
 endif::[]

docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -92,8 +92,7 @@ public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
             final List<HttpHost> hosts,
             final Version esVersion,
             final Version masterVersion) {
-        return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion,
-                restClientBuilder -> configureClient(restClientBuilder, restClientSettings()));
+        return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion, this::getClientBuilderWithSniffedHosts);
     }
 
     /**

libs/core/src/main/java/org/elasticsearch/common/CharArrays.java

@@ -0,0 +1,150 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * Helper class similar to Arrays to handle conversions for Char arrays
+ */
+public final class CharArrays {
+
+    private CharArrays() {}
+
+    /**
+     * Decodes the provided byte[] to a UTF-8 char[]. This is done while avoiding
+     * conversions to String. The provided byte[] is not modified by this method, so
+     * the caller needs to take care of clearing the value if it is sensitive.
+     */
+    public static char[] utf8BytesToChars(byte[] utf8Bytes) {
+        final ByteBuffer byteBuffer = ByteBuffer.wrap(utf8Bytes);
+        final CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer);
+        final char[] chars;
+        if (charBuffer.hasArray()) {
+            // there is no guarantee that the char buffers backing array is the right size
+            // so we need to make a copy
+            chars = Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit());
+            Arrays.fill(charBuffer.array(), (char) 0); // clear sensitive data
+        } else {
+            final int length = charBuffer.limit() - charBuffer.position();
+            chars = new char[length];
+            charBuffer.get(chars);
+            // if the buffer is not read only we can reset and fill with 0's
+            if (charBuffer.isReadOnly() == false) {
+                charBuffer.clear(); // reset
+                for (int i = 0; i < charBuffer.limit(); i++) {
+                    charBuffer.put((char) 0);
+                }
+            }
+        }
+        return chars;
+    }
+
+    /**
+     * Encodes the provided char[] to a UTF-8 byte[]. This is done while avoiding
+     * conversions to String. The provided char[] is not modified by this method, so
+     * the caller needs to take care of clearing the value if it is sensitive.
+     */
+    public static byte[] toUtf8Bytes(char[] chars) {
+        final CharBuffer charBuffer = CharBuffer.wrap(chars);
+        final ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(charBuffer);
+        final byte[] bytes;
+        if (byteBuffer.hasArray()) {
+            // there is no guarantee that the byte buffers backing array is the right size
+            // so we need to make a copy
+            bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
+            Arrays.fill(byteBuffer.array(), (byte) 0); // clear sensitive data
+        } else {
+            final int length = byteBuffer.limit() - byteBuffer.position();
+            bytes = new byte[length];
+            byteBuffer.get(bytes);
+            // if the buffer is not read only we can reset and fill with 0's
+            if (byteBuffer.isReadOnly() == false) {
+                byteBuffer.clear(); // reset
+                for (int i = 0; i < byteBuffer.limit(); i++) {
+                    byteBuffer.put((byte) 0);
+                }
+            }
+        }
+        return bytes;
+    }
+
+    /**
+     * Tests if a char[] contains a sequence of characters that match the prefix. This is like
+     * {@link String#startsWith(String)} but does not require conversion of the char[] to a string.
+     */
+    public static boolean charsBeginsWith(String prefix, char[] chars) {
+        if (chars == null || prefix == null) {
+            return false;
+        }
+
+        if (prefix.length() > chars.length) {
+            return false;
+        }
+
+        for (int i = 0; i < prefix.length(); i++) {
+            if (chars[i] != prefix.charAt(i)) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Constant time equality check of char arrays to avoid potential timing attacks.
+     */
+    public static boolean constantTimeEquals(char[] a, char[] b) {
+        Objects.requireNonNull(a, "char arrays must not be null for constantTimeEquals");
+        Objects.requireNonNull(b, "char arrays must not be null for constantTimeEquals");
+        if (a.length != b.length) {
+            return false;
+        }
+
+        int equals = 0;
+        for (int i = 0; i < a.length; i++) {
+            equals |= a[i] ^ b[i];
+        }
+
+        return equals == 0;
+    }
+
+    /**
+     * Constant time equality check of strings to avoid potential timing attacks.
+     */
+    public static boolean constantTimeEquals(String a, String b) {
+        Objects.requireNonNull(a, "strings must not be null for constantTimeEquals");
+        Objects.requireNonNull(b, "strings must not be null for constantTimeEquals");
+        if (a.length() != b.length()) {
+            return false;
+        }
+
+        int equals = 0;
+        for (int i = 0; i < a.length(); i++) {
+            equals |= a.charAt(i) ^ b.charAt(i);
+        }
+
+        return equals == 0;
+    }
+}

libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.nio.charset.StandardCharsets;
+
+public class CharArraysTests extends ESTestCase {
+
+    public void testCharsToBytes() {
+        final String originalValue = randomUnicodeOfCodepointLengthBetween(0, 32);
+        final byte[] expectedBytes = originalValue.getBytes(StandardCharsets.UTF_8);
+        final char[] valueChars = originalValue.toCharArray();
+
+        final byte[] convertedBytes = CharArrays.toUtf8Bytes(valueChars);
+        assertArrayEquals(expectedBytes, convertedBytes);
+    }
+
+    public void testBytesToUtf8Chars() {
+        final String originalValue = randomUnicodeOfCodepointLengthBetween(0, 32);
+        final byte[] bytes = originalValue.getBytes(StandardCharsets.UTF_8);
+        final char[] expectedChars = originalValue.toCharArray();
+
+        final char[] convertedChars = CharArrays.utf8BytesToChars(bytes);
+        assertArrayEquals(expectedChars, convertedChars);
+    }
+
+    public void testCharsBeginsWith() {
+        assertFalse(CharArrays.charsBeginsWith(randomAlphaOfLength(4), null));
+        assertFalse(CharArrays.charsBeginsWith(null, null));
+        assertFalse(CharArrays.charsBeginsWith(null, randomAlphaOfLength(4).toCharArray()));
+        assertFalse(CharArrays.charsBeginsWith(randomAlphaOfLength(2), randomAlphaOfLengthBetween(3, 8).toCharArray()));
+
+        final String prefix = randomAlphaOfLengthBetween(2, 4);
+        assertTrue(CharArrays.charsBeginsWith(prefix, prefix.toCharArray()));
+        final char[] prefixedValue = prefix.concat(randomAlphaOfLengthBetween(1, 12)).toCharArray();
+        assertTrue(CharArrays.charsBeginsWith(prefix, prefixedValue));
+
+        final String modifiedPrefix = randomBoolean() ? prefix.substring(1) : prefix.substring(0, prefix.length() - 1);
+        char[] nonMatchingValue;
+        do {
+            nonMatchingValue = modifiedPrefix.concat(randomAlphaOfLengthBetween(0, 12)).toCharArray();
+        } while (new String(nonMatchingValue).startsWith(prefix));
+        assertFalse(CharArrays.charsBeginsWith(prefix, nonMatchingValue));
+        assertTrue(CharArrays.charsBeginsWith(modifiedPrefix, nonMatchingValue));
+    }
+
+    public void testConstantTimeEquals() {
+        final String value = randomAlphaOfLengthBetween(0, 32);
+        assertTrue(CharArrays.constantTimeEquals(value, value));
+        assertTrue(CharArrays.constantTimeEquals(value.toCharArray(), value.toCharArray()));
+
+        final String other = randomAlphaOfLengthBetween(1, 32);
+        assertFalse(CharArrays.constantTimeEquals(value, other));
+        assertFalse(CharArrays.constantTimeEquals(value.toCharArray(), other.toCharArray()));
+    }
+}