From 1a45b27d8b692853036ea3acafd50e873d8d0e85 Mon Sep 17 00:00:00 2001
From: Jay Modi <jaymode@users.noreply.github.com>
Date: Wed, 15 Aug 2018 15:26:00 -0600
Subject: [PATCH 01/16] Move CharArrays to core lib (#32851)

This change cleans up some methods in the CharArrays class from x-pack, which
includes the unification of char[] to utf8 and utf8 to char[] conversions that
intentionally do not use strings. There was previously an implementation in
x-pack and in the reloading of secure settings. The method from the reloading
of secure settings was adopted as it handled more scenarios related to the
backing byte and char buffers that were used to perform the conversions. The
cleaned up class is moved into libs/core to allow it to be used by requests
that will be migrated to the high level rest client.

Relates #32332
---
 .../org/elasticsearch/common/CharArrays.java  | 150 ++++++++++++++++++
 .../elasticsearch/common/CharArraysTests.java |  72 +++++++++
 .../NodesReloadSecureSettingsRequest.java     |  66 +-------
 .../action/token/CreateTokenRequest.java      |   2 +-
 .../action/user/ChangePasswordRequest.java    |   2 +-
 .../security/action/user/PutUserRequest.java  |   2 +-
 .../core/security/authc/support/BCrypt.java   |   9 +-
 .../security/authc/support/CharArrays.java    | 101 ------------
 .../core/security/authc/support/Hasher.java   |   1 +
 .../authc/support/UsernamePasswordToken.java  |  14 +-
 .../xpack/core/ssl/PemUtils.java              |   2 +-
 .../core/watcher/crypto/CryptoService.java    |   2 +-
 .../ldap/ActiveDirectorySessionFactory.java   |   2 +-
 .../authc/ldap/LdapSessionFactory.java        |   2 +-
 .../ldap/LdapUserSearchSessionFactory.java    |   2 +-
 .../authc/ldap/PoolingSessionFactory.java     |   2 +-
 .../esnative/ESNativeMigrateToolTests.java    |   2 +-
 .../example/realm/CustomRealm.java            |   2 +-
 18 files changed, 257 insertions(+), 178 deletions(-)
 create mode 100644 libs/core/src/main/java/org/elasticsearch/common/CharArrays.java
 create mode 100644 libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CharArrays.java

diff --git a/libs/core/src/main/java/org/elasticsearch/common/CharArrays.java b/libs/core/src/main/java/org/elasticsearch/common/CharArrays.java
new file mode 100644
index 00000000000..907874ca573
--- /dev/null
+++ b/libs/core/src/main/java/org/elasticsearch/common/CharArrays.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * Helper class similar to Arrays to handle conversions for Char arrays
+ */
+public final class CharArrays {
+
+    private CharArrays() {}
+
+    /**
+     * Decodes the provided byte[] to a UTF-8 char[]. This is done while avoiding
+     * conversions to String. The provided byte[] is not modified by this method, so
+     * the caller needs to take care of clearing the value if it is sensitive.
+     */
+    public static char[] utf8BytesToChars(byte[] utf8Bytes) {
+        final ByteBuffer byteBuffer = ByteBuffer.wrap(utf8Bytes);
+        final CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer);
+        final char[] chars;
+        if (charBuffer.hasArray()) {
+            // there is no guarantee that the char buffers backing array is the right size
+            // so we need to make a copy
+            chars = Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit());
+            Arrays.fill(charBuffer.array(), (char) 0); // clear sensitive data
+        } else {
+            final int length = charBuffer.limit() - charBuffer.position();
+            chars = new char[length];
+            charBuffer.get(chars);
+            // if the buffer is not read only we can reset and fill with 0's
+            if (charBuffer.isReadOnly() == false) {
+                charBuffer.clear(); // reset
+                for (int i = 0; i < charBuffer.limit(); i++) {
+                    charBuffer.put((char) 0);
+                }
+            }
+        }
+        return chars;
+    }
+
+    /**
+     * Encodes the provided char[] to a UTF-8 byte[]. This is done while avoiding
+     * conversions to String. The provided char[] is not modified by this method, so
+     * the caller needs to take care of clearing the value if it is sensitive.
+     */
+    public static byte[] toUtf8Bytes(char[] chars) {
+        final CharBuffer charBuffer = CharBuffer.wrap(chars);
+        final ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(charBuffer);
+        final byte[] bytes;
+        if (byteBuffer.hasArray()) {
+            // there is no guarantee that the byte buffers backing array is the right size
+            // so we need to make a copy
+            bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
+            Arrays.fill(byteBuffer.array(), (byte) 0); // clear sensitive data
+        } else {
+            final int length = byteBuffer.limit() - byteBuffer.position();
+            bytes = new byte[length];
+            byteBuffer.get(bytes);
+            // if the buffer is not read only we can reset and fill with 0's
+            if (byteBuffer.isReadOnly() == false) {
+                byteBuffer.clear(); // reset
+                for (int i = 0; i < byteBuffer.limit(); i++) {
+                    byteBuffer.put((byte) 0);
+                }
+            }
+        }
+        return bytes;
+    }
+
+    /**
+     * Tests if a char[] contains a sequence of characters that match the prefix. This is like
+     * {@link String#startsWith(String)} but does not require conversion of the char[] to a string.
+     */
+    public static boolean charsBeginsWith(String prefix, char[] chars) {
+        if (chars == null || prefix == null) {
+            return false;
+        }
+
+        if (prefix.length() > chars.length) {
+            return false;
+        }
+
+        for (int i = 0; i < prefix.length(); i++) {
+            if (chars[i] != prefix.charAt(i)) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Constant time equality check of char arrays to avoid potential timing attacks.
+     */
+    public static boolean constantTimeEquals(char[] a, char[] b) {
+        Objects.requireNonNull(a, "char arrays must not be null for constantTimeEquals");
+        Objects.requireNonNull(b, "char arrays must not be null for constantTimeEquals");
+        if (a.length != b.length) {
+            return false;
+        }
+
+        int equals = 0;
+        for (int i = 0; i < a.length; i++) {
+            equals |= a[i] ^ b[i];
+        }
+
+        return equals == 0;
+    }
+
+    /**
+     * Constant time equality check of strings to avoid potential timing attacks.
+     */
+    public static boolean constantTimeEquals(String a, String b) {
+        Objects.requireNonNull(a, "strings must not be null for constantTimeEquals");
+        Objects.requireNonNull(b, "strings must not be null for constantTimeEquals");
+        if (a.length() != b.length()) {
+            return false;
+        }
+
+        int equals = 0;
+        for (int i = 0; i < a.length(); i++) {
+            equals |= a.charAt(i) ^ b.charAt(i);
+        }
+
+        return equals == 0;
+    }
+}
diff --git a/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java b/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java
new file mode 100644
index 00000000000..64b1ecd1f8a
--- /dev/null
+++ b/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.nio.charset.StandardCharsets;
+
+public class CharArraysTests extends ESTestCase {
+
+    public void testCharsToBytes() {
+        final String originalValue = randomUnicodeOfCodepointLengthBetween(0, 32);
+        final byte[] expectedBytes = originalValue.getBytes(StandardCharsets.UTF_8);
+        final char[] valueChars = originalValue.toCharArray();
+
+        final byte[] convertedBytes = CharArrays.toUtf8Bytes(valueChars);
+        assertArrayEquals(expectedBytes, convertedBytes);
+    }
+
+    public void testBytesToUtf8Chars() {
+        final String originalValue = randomUnicodeOfCodepointLengthBetween(0, 32);
+        final byte[] bytes = originalValue.getBytes(StandardCharsets.UTF_8);
+        final char[] expectedChars = originalValue.toCharArray();
+
+        final char[] convertedChars = CharArrays.utf8BytesToChars(bytes);
+        assertArrayEquals(expectedChars, convertedChars);
+    }
+
+    public void testCharsBeginsWith() {
+        assertFalse(CharArrays.charsBeginsWith(randomAlphaOfLength(4), null));
+        assertFalse(CharArrays.charsBeginsWith(null, null));
+        assertFalse(CharArrays.charsBeginsWith(null, randomAlphaOfLength(4).toCharArray()));
+        assertFalse(CharArrays.charsBeginsWith(randomAlphaOfLength(2), randomAlphaOfLengthBetween(3, 8).toCharArray()));
+
+        final String prefix = randomAlphaOfLengthBetween(2, 4);
+        assertTrue(CharArrays.charsBeginsWith(prefix, prefix.toCharArray()));
+        final char[] prefixedValue = prefix.concat(randomAlphaOfLengthBetween(1, 12)).toCharArray();
+        assertTrue(CharArrays.charsBeginsWith(prefix, prefixedValue));
+
+        final String modifiedPrefix = randomBoolean() ? prefix.substring(1) : prefix.substring(0, prefix.length() - 1);
+        final char[] nonMatchingValue = modifiedPrefix.concat(randomAlphaOfLengthBetween(0, 12)).toCharArray();
+        assertFalse(CharArrays.charsBeginsWith(prefix, nonMatchingValue));
+        assertTrue(CharArrays.charsBeginsWith(modifiedPrefix, nonMatchingValue));
+    }
+
+    public void testConstantTimeEquals() {
+        final String value = randomAlphaOfLengthBetween(0, 32);
+        assertTrue(CharArrays.constantTimeEquals(value, value));
+        assertTrue(CharArrays.constantTimeEquals(value.toCharArray(), value.toCharArray()));
+
+        final String other = randomAlphaOfLengthBetween(1, 32);
+        assertFalse(CharArrays.constantTimeEquals(value, other));
+        assertFalse(CharArrays.constantTimeEquals(value.toCharArray(), other.toCharArray()));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java
index 50df7b1bb26..5320470d366 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java
@@ -22,14 +22,12 @@ package org.elasticsearch.action.admin.cluster.node.reload;
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.SecureString;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
@@ -83,7 +81,7 @@ public class NodesReloadSecureSettingsRequest extends BaseNodesRequest<NodesRelo
         super.readFrom(in);
         final byte[] passwordBytes = in.readByteArray();
         try {
-            this.secureSettingsPassword = new SecureString(utf8BytesToChars(passwordBytes));
+            this.secureSettingsPassword = new SecureString(CharArrays.utf8BytesToChars(passwordBytes));
         } finally {
             Arrays.fill(passwordBytes, (byte) 0);
         }
@@ -92,69 +90,11 @@ public class NodesReloadSecureSettingsRequest extends BaseNodesRequest<NodesRelo
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        final byte[] passwordBytes = charsToUtf8Bytes(this.secureSettingsPassword.getChars());
+        final byte[] passwordBytes = CharArrays.toUtf8Bytes(this.secureSettingsPassword.getChars());
         try {
             out.writeByteArray(passwordBytes);
         } finally {
             Arrays.fill(passwordBytes, (byte) 0);
         }
     }
-
-    /**
-     * Encodes the provided char[] to a UTF-8 byte[]. This is done while avoiding
-     * conversions to String. The provided char[] is not modified by this method, so
-     * the caller needs to take care of clearing the value if it is sensitive.
-     */
-    private static byte[] charsToUtf8Bytes(char[] chars) {
-        final CharBuffer charBuffer = CharBuffer.wrap(chars);
-        final ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(charBuffer);
-        final byte[] bytes;
-        if (byteBuffer.hasArray()) {
-            // there is no guarantee that the byte buffers backing array is the right size
-            // so we need to make a copy
-            bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
-            Arrays.fill(byteBuffer.array(), (byte) 0); // clear sensitive data
-        } else {
-            final int length = byteBuffer.limit() - byteBuffer.position();
-            bytes = new byte[length];
-            byteBuffer.get(bytes);
-            // if the buffer is not read only we can reset and fill with 0's
-            if (byteBuffer.isReadOnly() == false) {
-                byteBuffer.clear(); // reset
-                for (int i = 0; i < byteBuffer.limit(); i++) {
-                    byteBuffer.put((byte) 0);
-                }
-            }
-        }
-        return bytes;
-    }
-
-    /**
-     * Decodes the provided byte[] to a UTF-8 char[]. This is done while avoiding
-     * conversions to String. The provided byte[] is not modified by this method, so
-     * the caller needs to take care of clearing the value if it is sensitive.
-     */
-    public static char[] utf8BytesToChars(byte[] utf8Bytes) {
-        final ByteBuffer byteBuffer = ByteBuffer.wrap(utf8Bytes);
-        final CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer);
-        final char[] chars;
-        if (charBuffer.hasArray()) {
-            // there is no guarantee that the char buffers backing array is the right size
-            // so we need to make a copy
-            chars = Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit());
-            Arrays.fill(charBuffer.array(), (char) 0); // clear sensitive data
-        } else {
-            final int length = charBuffer.limit() - charBuffer.position();
-            chars = new char[length];
-            charBuffer.get(chars);
-            // if the buffer is not read only we can reset and fill with 0's
-            if (charBuffer.isReadOnly() == false) {
-                charBuffer.clear(); // reset
-                for (int i = 0; i < charBuffer.limit(); i++) {
-                    charBuffer.put((char) 0);
-                }
-            }
-        }
-        return chars;
-    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java
index 5956e1a6613..fdb46711c0c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java
@@ -15,7 +15,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
+import org.elasticsearch.common.CharArrays;
 
 import java.io.IOException;
 import java.util.Arrays;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequest.java
index f84b133d984..b78b81c0600 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequest.java
@@ -12,7 +12,7 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
+import org.elasticsearch.common.CharArrays;
 
 import java.io.IOException;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequest.java
index f37072b9cf0..e704259396a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequest.java
@@ -8,12 +8,12 @@ package org.elasticsearch.xpack.core.security.action.user;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
 
 import java.io.IOException;
 import java.util.Map;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/BCrypt.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/BCrypt.java
index ceb93dc4c85..a93476bbdc8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/BCrypt.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/BCrypt.java
@@ -14,6 +14,7 @@ package org.elasticsearch.xpack.core.security.authc.support;
 // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.settings.SecureString;
 
 import java.security.SecureRandom;
@@ -54,7 +55,7 @@ import java.security.SecureRandom;
  * String stronger_salt = BCrypt.gensalt(12)<br>
  * </code>
  * <p>
- * The amount of work increases exponentially (2**log_rounds), so 
+ * The amount of work increases exponentially (2**log_rounds), so
  * each increment is twice as much work. The default log_rounds is
  * 10, and the valid range is 4 to 30.
  *
@@ -689,7 +690,11 @@ public class BCrypt {
 
         // the next lines are the SecureString replacement for the above commented-out section
         if (minor >= 'a') {
-            try (SecureString secureString = new SecureString(CharArrays.concat(password.getChars(), "\000".toCharArray()))) {
+            final char[] suffix = "\000".toCharArray();
+            final char[] result = new char[password.length() + suffix.length];
+            System.arraycopy(password.getChars(), 0, result, 0, password.length());
+            System.arraycopy(suffix, 0, result, password.length(), suffix.length);
+            try (SecureString secureString = new SecureString(result)) {
                 passwordb = CharArrays.toUtf8Bytes(secureString.getChars());
             }
         } else {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CharArrays.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CharArrays.java
deleted file mode 100644
index 26df90c31a2..00000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CharArrays.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.security.authc.support;
-
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-
-/**
- * Helper class similar to Arrays to handle conversions for Char arrays
- */
-public class CharArrays {
-
-    public static char[] utf8BytesToChars(byte[] utf8Bytes) {
-        ByteBuffer byteBuffer = ByteBuffer.wrap(utf8Bytes);
-        CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer);
-        char[] chars = Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit());
-        byteBuffer.clear();
-        charBuffer.clear();
-        return chars;
-    }
-
-    /**
-     * Like String.indexOf for for an array of chars
-     */
-    static int indexOf(char[] array, char ch) {
-        for (int i = 0; (i < array.length); i++) {
-            if (array[i] == ch) {
-                return i;
-            }
-        }
-        return -1;
-    }
-
-    /**
-     * Converts the provided char[] to a UTF-8 byte[]. The provided char[] is not modified by this
-     * method, so the caller needs to take care of clearing the value if it is sensitive.
-     */
-    public static byte[] toUtf8Bytes(char[] chars) {
-        CharBuffer charBuffer = CharBuffer.wrap(chars);
-        ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(charBuffer);
-        byte[] bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
-        Arrays.fill(byteBuffer.array(), (byte) 0); // clear sensitive data
-        return bytes;
-    }
-
-    public static boolean charsBeginsWith(String prefix, char[] chars) {
-        if (chars == null || prefix == null) {
-            return false;
-        }
-
-        if (prefix.length() > chars.length) {
-            return false;
-        }
-
-        for (int i = 0; i < prefix.length(); i++) {
-            if (chars[i] != prefix.charAt(i)) {
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    public static boolean constantTimeEquals(char[] a, char[] b) {
-        if (a.length != b.length) {
-            return false;
-        }
-
-        int equals = 0;
-        for (int i = 0; i < a.length; i++) {
-            equals |= a[i] ^ b[i];
-        }
-
-        return equals == 0;
-    }
-
-    public static boolean constantTimeEquals(String a, String b) {
-        if (a.length() != b.length()) {
-            return false;
-        }
-
-        int equals = 0;
-        for (int i = 0; i < a.length(); i++) {
-            equals |= a.charAt(i) ^ b.charAt(i);
-        }
-
-        return equals == 0;
-    }
-
-    public static char[] concat(char[] a, char[] b) {
-        final char[] result = new char[a.length + b.length];
-        System.arraycopy(a, 0, result, 0, a.length);
-        System.arraycopy(b, 0, result, a.length, b.length);
-        return result;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java
index d12547bd906..492622b2c51 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.security.authc.support;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.settings.SecureString;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java
index d8e58c29d23..13493036008 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.authc.support;
 
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -107,7 +108,7 @@ public class UsernamePasswordToken implements AuthenticationToken {
             throw authenticationError("invalid basic authentication header encoding", e);
         }
 
-        int i = CharArrays.indexOf(userpasswd, ':');
+        int i = indexOfColon(userpasswd);
         if (i < 0) {
             throw authenticationError("invalid basic authentication header value");
         }
@@ -121,4 +122,15 @@ public class UsernamePasswordToken implements AuthenticationToken {
         context.putHeader(BASIC_AUTH_HEADER, basicAuthHeaderValue(token.username, token.password));
     }
 
+    /**
+     * Like String.indexOf for for an array of chars
+     */
+    private static int indexOfColon(char[] array) {
+        for (int i = 0; (i < array.length); i++) {
+            if (array[i] == ':') {
+                return i;
+            }
+        }
+        return -1;
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java
index d959c017e0a..a3814a76a3e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.core.ssl;
 
 import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
+import org.elasticsearch.common.CharArrays;
 
 import java.io.BufferedReader;
 import java.io.IOException;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java
index b1f3a32769e..a25e79ffdf6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java
@@ -13,7 +13,7 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.core.watcher.WatcherField;
 import org.elasticsearch.xpack.core.security.SecurityField;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
+import org.elasticsearch.common.CharArrays;
 
 import javax.crypto.BadPaddingException;
 import javax.crypto.Cipher;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java
index d175e1b2293..8107d748818 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java
@@ -32,7 +32,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.ActiveDirectorySessionFactorySettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapMetaDataResolver;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java
index 36d14aa67c0..70b2f0015cf 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java
@@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapSessionFactorySettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.SearchGroupsResolverSettings;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapMetaDataResolver;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java
index 2ec87888d8c..a3541ec2759 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java
@@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapUserSearchSessionFactorySettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.SearchGroupsResolverSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession.GroupsResolver;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java
index 367bd525036..986fa1900e7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java
@@ -25,7 +25,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapMetaDataResolver;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java
index 212ee7ea499..6d75cf09371 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java
@@ -13,7 +13,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.BeforeClass;
diff --git a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java
index af3fb160e13..c6502c05d25 100644
--- a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java
+++ b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java
@@ -12,7 +12,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
-import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.protocol.xpack.security.User;
 

From 93ada98eb54d99b9d98ed51778bd6422d18fd392 Mon Sep 17 00:00:00 2001
From: debadair <debadair@elastic.co>
Date: Wed, 15 Aug 2018 15:09:41 -0700
Subject: [PATCH 02/16] [DOCS] Fixing cross doc link to Stack Overview security
 topic.

---
 x-pack/docs/en/security/configuring-es.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/docs/en/security/configuring-es.asciidoc b/x-pack/docs/en/security/configuring-es.asciidoc
index 5e8f1adbc7a..53f36afc734 100644
--- a/x-pack/docs/en/security/configuring-es.asciidoc
+++ b/x-pack/docs/en/security/configuring-es.asciidoc
@@ -9,7 +9,7 @@
 password-protect your data as well as implement more advanced security measures
 such as encrypting communications, role-based access control, IP filtering, and
 auditing. For more information, see
-{xpack-ref}/xpack-security.html[Securing the Elastic Stack].
+{xpack-ref}/elasticsearch-security.html[Securing the Elastic Stack].
 
 To use {security} in {es}:
 

From 70d80a3d093e327a05eeb7b8c49fee7c94952a8f Mon Sep 17 00:00:00 2001
From: markharwood <markharwood@gmail.com>
Date: Thu, 16 Aug 2018 10:21:37 +0100
Subject: [PATCH 03/16] Docs enhancement: added reference to cluster-level
 setting `search.default_allow_partial_results` (#32810)

Closes #32809
---
 docs/reference/search/request-body.asciidoc | 3 ++-
 docs/reference/search/uri-request.asciidoc  | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc
index 2a51d705d83..e7c9b593af3 100644
--- a/docs/reference/search/request-body.asciidoc
+++ b/docs/reference/search/request-body.asciidoc
@@ -90,7 +90,8 @@ And here is a sample response:
 
     Set to `false` to return an overall failure if the request would produce partial 
     results. Defaults to true, which will allow partial results in the case of timeouts
-    or partial failures.
+    or partial failures. This default can be controlled using the cluster-level setting
+    `search.default_allow_partial_results`.
 
 `terminate_after`::
 
diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc
index a90f32bb3cd..279bc0c0384 100644
--- a/docs/reference/search/uri-request.asciidoc
+++ b/docs/reference/search/uri-request.asciidoc
@@ -125,5 +125,6 @@ more details on the different types of search that can be performed.
 
 |`allow_partial_search_results` |Set to `false` to return an overall failure if the request would produce
 partial results. Defaults to true, which will allow partial results in the case of timeouts
-or partial failures..
+or partial failures. This default can be controlled using the cluster-level setting
+`search.default_allow_partial_results`.
 |=======================================================================

From e6bfba1d799cbc41fce5ea88e2252f78fd9cac65 Mon Sep 17 00:00:00 2001
From: datosh <datosh@gmx.de>
Date: Thu, 16 Aug 2018 11:34:41 +0200
Subject: [PATCH 04/16] [DOCS] Clarify sentence in network-host.asciidoc
 (#32429)

---
 docs/reference/setup/important-settings/network-host.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/setup/important-settings/network-host.asciidoc b/docs/reference/setup/important-settings/network-host.asciidoc
index 7e29e73123d..1788bfebc66 100644
--- a/docs/reference/setup/important-settings/network-host.asciidoc
+++ b/docs/reference/setup/important-settings/network-host.asciidoc
@@ -9,7 +9,7 @@ location on a single node.  This can be useful for testing Elasticsearch's
 ability to form clusters, but it is not a configuration recommended for
 production.
 
-In order to communicate and to form a cluster with nodes on other servers, your
+In order to form a cluster with nodes on other servers, your
 node will need to bind to a non-loopback address. While there are many
 <<modules-network,network settings>>, usually all you need to configure is
 `network.host`:

From 996ed73d735b1a2943cb5c7af6ba315a225b64c0 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jimczi@apache.org>
Date: Thu, 16 Aug 2018 11:44:20 +0200
Subject: [PATCH 05/16] Test: Fix unpredictive merges in
 DocumentSubsetReaderTests

The merge policy that was used could lead to unpredictive merges due to the
randomization of `setDeletesPctAllowed`.

Closes #32457
---
 .../authz/accesscontrol/DocumentSubsetReaderTests.java         | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
index 38857e2170d..dca2f37f3f2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
@@ -80,9 +80,8 @@ public class DocumentSubsetReaderTests extends ESTestCase {
         bitsetFilterCache.close();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32457")
     public void testSearch() throws Exception {
-        IndexWriter iw = new IndexWriter(directory, newIndexWriterConfig());
+        IndexWriter iw = new IndexWriter(directory, newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random())));
 
         Document document = new Document();
         document.add(new StringField("field", "value1", Field.Store.NO));

From 7f6802cb51eda172c64fb223bf5728719f586b11 Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Thu, 16 Aug 2018 10:48:56 +0100
Subject: [PATCH 06/16] [ML] Choose seconds to fix intermittent
 DatafeeedConfigTest failure

---
 .../xpack/core/ml/datafeed/DatafeedConfigTests.java             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
index ffc13655d22..3030449abd1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
@@ -100,7 +100,7 @@ public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedCon
             if (aggHistogramInterval == null) {
                 builder.setFrequency(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)));
             } else {
-                builder.setFrequency(TimeValue.timeValueMillis(randomIntBetween(1, 5) * aggHistogramInterval));
+                builder.setFrequency(TimeValue.timeValueSeconds(randomIntBetween(1, 5) * aggHistogramInterval));
             }
         }
         if (randomBoolean()) {

From 039babddf532c918ae3f0ac5360e85e7ba5b3052 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 16 Aug 2018 11:53:01 +0200
Subject: [PATCH 07/16] CharArraysTests: Fix test bug.

---
 .../test/java/org/elasticsearch/common/CharArraysTests.java  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java b/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java
index 64b1ecd1f8a..9283283ab08 100644
--- a/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java
+++ b/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java
@@ -55,7 +55,10 @@ public class CharArraysTests extends ESTestCase {
         assertTrue(CharArrays.charsBeginsWith(prefix, prefixedValue));
 
         final String modifiedPrefix = randomBoolean() ? prefix.substring(1) : prefix.substring(0, prefix.length() - 1);
-        final char[] nonMatchingValue = modifiedPrefix.concat(randomAlphaOfLengthBetween(0, 12)).toCharArray();
+        char[] nonMatchingValue;
+        do {
+            nonMatchingValue = modifiedPrefix.concat(randomAlphaOfLengthBetween(0, 12)).toCharArray();
+        } while (new String(nonMatchingValue).startsWith(prefix));
         assertFalse(CharArrays.charsBeginsWith(prefix, nonMatchingValue));
         assertTrue(CharArrays.charsBeginsWith(modifiedPrefix, nonMatchingValue));
     }

From d80457ee2a0a5ce8973b48a06b5d60ee2c5e64c1 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Thu, 16 Aug 2018 11:07:20 +0100
Subject: [PATCH 08/16] Mutes test in DuelScrollIT

Due to https://github.com/elastic/elasticsearch/issues/32682
---
 .../test/java/org/elasticsearch/search/scroll/DuelScrollIT.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
index 31fcfa7155c..1ddd11e5d0f 100644
--- a/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
+++ b/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.scroll;
 
 import com.carrotsearch.hppc.IntHashSet;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
@@ -256,6 +257,7 @@ public class DuelScrollIT extends ESIntegTestCase {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32682")
     public void testDuelIndexOrderQueryThenFetch() throws Exception {
         final SearchType searchType = RandomPicks.randomFrom(random(), Arrays.asList(SearchType.QUERY_THEN_FETCH,
             SearchType.DFS_QUERY_THEN_FETCH));

From e35be019014aafba9c34484e5711a04ce482ffd1 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 16 Aug 2018 12:27:21 +0200
Subject: [PATCH 09/16] AwaitFix AckIT.

Relates #32767
---
 server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java b/server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
index 2cd8a2c27c7..df97854cc35 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cluster.ack;
 
+import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
@@ -50,6 +51,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
 @ClusterScope(minNumDataNodes = 2)
+@AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/32767")
 public class AckIT extends ESIntegTestCase {
 
     @Override

From f8c7414ee861b441f81716e13c3190f4d2d97c54 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 16 Aug 2018 07:24:05 -0400
Subject: [PATCH 10/16] Remove passphrase support from reload settings API
 (#32889)

We do not support passphrases on the secure settings storage (the
keystore). Yet, we added support for this in the API layer. This commit
removes this support so that we are not limited in our future options,
or have to make a breaking change.
---
 .../NodesReloadSecureSettingsRequest.java     | 68 +------------
 ...desReloadSecureSettingsRequestBuilder.java | 49 ----------
 ...nsportNodesReloadSecureSettingsAction.java |  6 +-
 .../RestReloadSecureSettingsAction.java       | 11 +--
 .../action/admin/ReloadSecureSettingsIT.java  | 98 ++-----------------
 5 files changed, 16 insertions(+), 216 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java
index 5320470d366..fb3e6ac71ad 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java
@@ -19,82 +19,22 @@
 
 package org.elasticsearch.action.admin.cluster.node.reload;
 
-
-import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
-import org.elasticsearch.common.CharArrays;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.SecureString;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 /**
- * Request for a reload secure settings action
+ * Request for a reload secure settings action.
  */
 public class NodesReloadSecureSettingsRequest extends BaseNodesRequest<NodesReloadSecureSettingsRequest> {
 
-    /**
-     * The password which is broadcasted to all nodes, but is never stored on
-     * persistent storage. The password is used to reread and decrypt the contents
-     * of the node's keystore (backing the implementation of
-     * {@code SecureSettings}).
-     */
-    private SecureString secureSettingsPassword;
-
     public NodesReloadSecureSettingsRequest() {
     }
 
     /**
-     * Reload secure settings only on certain nodes, based on the nodes ids
-     * specified. If none are passed, secure settings will be reloaded on all the
-     * nodes.
+     * Reload secure settings only on certain nodes, based on the nodes IDs specified. If none are passed, secure settings will be reloaded
+     * on all the nodes.
      */
-    public NodesReloadSecureSettingsRequest(String... nodesIds) {
+    public NodesReloadSecureSettingsRequest(final String... nodesIds) {
         super(nodesIds);
     }
 
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
-        if (secureSettingsPassword == null) {
-            validationException = addValidationError("secure settings password cannot be null (use empty string instead)",
-                    validationException);
-        }
-        return validationException;
-    }
-
-    public SecureString secureSettingsPassword() {
-        return secureSettingsPassword;
-    }
-
-    public NodesReloadSecureSettingsRequest secureStorePassword(SecureString secureStorePassword) {
-        this.secureSettingsPassword = secureStorePassword;
-        return this;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        final byte[] passwordBytes = in.readByteArray();
-        try {
-            this.secureSettingsPassword = new SecureString(CharArrays.utf8BytesToChars(passwordBytes));
-        } finally {
-            Arrays.fill(passwordBytes, (byte) 0);
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        final byte[] passwordBytes = CharArrays.toUtf8Bytes(this.secureSettingsPassword.getChars());
-        try {
-            out.writeByteArray(passwordBytes);
-        } finally {
-            Arrays.fill(passwordBytes, (byte) 0);
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java
index b5f2f73e56f..c8250455e6b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java
@@ -19,19 +19,8 @@
 
 package org.elasticsearch.action.admin.cluster.node.reload;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Objects;
 
 /**
  * Builder for the reload secure settings nodes request
@@ -39,46 +28,8 @@ import java.util.Objects;
 public class NodesReloadSecureSettingsRequestBuilder extends NodesOperationRequestBuilder<NodesReloadSecureSettingsRequest,
         NodesReloadSecureSettingsResponse, NodesReloadSecureSettingsRequestBuilder> {
 
-    public static final String SECURE_SETTINGS_PASSWORD_FIELD_NAME = "secure_settings_password";
-
     public NodesReloadSecureSettingsRequestBuilder(ElasticsearchClient client, NodesReloadSecureSettingsAction action) {
         super(client, action, new NodesReloadSecureSettingsRequest());
     }
 
-    public NodesReloadSecureSettingsRequestBuilder setSecureStorePassword(SecureString secureStorePassword) {
-        request.secureStorePassword(secureStorePassword);
-        return this;
-    }
-
-    public NodesReloadSecureSettingsRequestBuilder source(BytesReference source, XContentType xContentType) throws IOException {
-        Objects.requireNonNull(xContentType);
-        // EMPTY is ok here because we never call namedObject
-        try (InputStream stream = source.streamInput();
-                XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY,
-                        LoggingDeprecationHandler.INSTANCE, stream)) {
-            XContentParser.Token token;
-            token = parser.nextToken();
-            if (token != XContentParser.Token.START_OBJECT) {
-                throw new ElasticsearchParseException("expected an object, but found token [{}]", token);
-            }
-            token = parser.nextToken();
-            if (token != XContentParser.Token.FIELD_NAME || false == SECURE_SETTINGS_PASSWORD_FIELD_NAME.equals(parser.currentName())) {
-                throw new ElasticsearchParseException("expected a field named [{}], but found [{}]", SECURE_SETTINGS_PASSWORD_FIELD_NAME,
-                        token);
-            }
-            token = parser.nextToken();
-            if (token != XContentParser.Token.VALUE_STRING) {
-                throw new ElasticsearchParseException("expected field [{}] to be of type string, but found [{}] instead",
-                        SECURE_SETTINGS_PASSWORD_FIELD_NAME, token);
-            }
-            final String password = parser.text();
-            setSecureStorePassword(new SecureString(password.toCharArray()));
-            token = parser.nextToken();
-            if (token != XContentParser.Token.END_OBJECT) {
-                throw new ElasticsearchParseException("expected end of object, but found token [{}]", token);
-            }
-        }
-        return this;
-    }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
index 0f44170fa60..b8a36bac68d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
@@ -31,7 +31,6 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
-import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.PluginsService;
@@ -82,16 +81,13 @@ public class TransportNodesReloadSecureSettingsAction extends TransportNodesActi
 
     @Override
     protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation(NodeRequest nodeReloadRequest) {
-        final NodesReloadSecureSettingsRequest request = nodeReloadRequest.request;
-        final SecureString secureSettingsPassword = request.secureSettingsPassword();
         try (KeyStoreWrapper keystore = KeyStoreWrapper.load(environment.configFile())) {
             // reread keystore from config file
             if (keystore == null) {
                 return new NodesReloadSecureSettingsResponse.NodeResponse(clusterService.localNode(),
                         new IllegalStateException("Keystore is missing"));
             }
-            // decrypt the keystore using the password from the request
-            keystore.decrypt(secureSettingsPassword.getChars());
+            keystore.decrypt(new char[0]);
             // add the keystore to the original node settings object
             final Settings settingsWithKeystore = Settings.builder()
                     .put(environment.settings(), false)
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java
index 0697871ea5d..2251615d678 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java
@@ -59,7 +59,6 @@ public final class RestReloadSecureSettingsAction extends BaseRestHandler {
                 .cluster()
                 .prepareReloadSecureSettings()
                 .setTimeout(request.param("timeout"))
-                .source(request.requiredContent(), request.getXContentType())
                 .setNodesIds(nodesIds);
         final NodesReloadSecureSettingsRequest nodesRequest = nodesRequestBuilder.request();
         return channel -> nodesRequestBuilder
@@ -68,12 +67,12 @@ public final class RestReloadSecureSettingsAction extends BaseRestHandler {
                     public RestResponse buildResponse(NodesReloadSecureSettingsResponse response, XContentBuilder builder)
                             throws Exception {
                         builder.startObject();
-                        RestActions.buildNodesHeader(builder, channel.request(), response);
-                        builder.field("cluster_name", response.getClusterName().value());
-                        response.toXContent(builder, channel.request());
+                        {
+                            RestActions.buildNodesHeader(builder, channel.request(), response);
+                            builder.field("cluster_name", response.getClusterName().value());
+                            response.toXContent(builder, channel.request());
+                        }
                         builder.endObject();
-                        // clear password for the original request
-                        nodesRequest.secureSettingsPassword().close();
                         return new BytesRestResponse(RestStatus.OK, builder);
                     }
                 });
diff --git a/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
index 79527582405..3f9e258ffec 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
@@ -20,11 +20,9 @@
 package org.elasticsearch.action.admin;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsResponse;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.SecureSettings;
-import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
@@ -44,11 +42,11 @@ import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.containsString;
 
 public class ReloadSecureSettingsIT extends ESIntegTestCase {
 
@@ -62,7 +60,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
         Files.deleteIfExists(KeyStoreWrapper.keystorePath(environment.configFile()));
         final int initialReloadCount = mockReloadablePlugin.getReloadCount();
         final CountDownLatch latch = new CountDownLatch(1);
-        client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute(
+        client().admin().cluster().prepareReloadSecureSettings().execute(
                 new ActionListener<NodesReloadSecureSettingsResponse>() {
                     @Override
                     public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
@@ -96,44 +94,6 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
         assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount));
     }
 
-    public void testNullKeystorePassword() throws Exception {
-        final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class);
-        final MockReloadablePlugin mockReloadablePlugin = pluginsService.filterPlugins(MockReloadablePlugin.class)
-                .stream().findFirst().get();
-        final AtomicReference<AssertionError> reloadSettingsError = new AtomicReference<>();
-        final int initialReloadCount = mockReloadablePlugin.getReloadCount();
-        final CountDownLatch latch = new CountDownLatch(1);
-        client().admin().cluster().prepareReloadSecureSettings().execute(
-                new ActionListener<NodesReloadSecureSettingsResponse>() {
-                    @Override
-                    public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
-                        try {
-                            reloadSettingsError.set(new AssertionError("Null keystore password should fail"));
-                        } finally {
-                            latch.countDown();
-                        }
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        try {
-                            assertThat(e, instanceOf(ActionRequestValidationException.class));
-                            assertThat(e.getMessage(), containsString("secure settings password cannot be null"));
-                        } catch (final AssertionError ae) {
-                            reloadSettingsError.set(ae);
-                        } finally {
-                            latch.countDown();
-                        }
-                    }
-                });
-        latch.await();
-        if (reloadSettingsError.get() != null) {
-            throw reloadSettingsError.get();
-        }
-        // in the null password case no reload should be triggered
-        assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount));
-    }
-
     public void testInvalidKeystoreFile() throws Exception {
         final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class);
         final MockReloadablePlugin mockReloadablePlugin = pluginsService.filterPlugins(MockReloadablePlugin.class)
@@ -149,7 +109,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
             Files.copy(keystore, KeyStoreWrapper.keystorePath(environment.configFile()), StandardCopyOption.REPLACE_EXISTING);
         }
         final CountDownLatch latch = new CountDownLatch(1);
-        client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute(
+        client().admin().cluster().prepareReloadSecureSettings().execute(
                 new ActionListener<NodesReloadSecureSettingsResponse>() {
                     @Override
                     public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
@@ -181,52 +141,6 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
         assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount));
     }
 
-    public void testWrongKeystorePassword() throws Exception {
-        final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class);
-        final MockReloadablePlugin mockReloadablePlugin = pluginsService.filterPlugins(MockReloadablePlugin.class)
-                .stream().findFirst().get();
-        final Environment environment = internalCluster().getInstance(Environment.class);
-        final AtomicReference<AssertionError> reloadSettingsError = new AtomicReference<>();
-        final int initialReloadCount = mockReloadablePlugin.getReloadCount();
-        // "some" keystore should be present in this case
-        writeEmptyKeystore(environment, new char[0]);
-        final CountDownLatch latch = new CountDownLatch(1);
-        client().admin()
-                .cluster()
-                .prepareReloadSecureSettings()
-                .setSecureStorePassword(new SecureString(new char[] { 'W', 'r', 'o', 'n', 'g' }))
-                .execute(new ActionListener<NodesReloadSecureSettingsResponse>() {
-                    @Override
-                    public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
-                        try {
-                            assertThat(nodesReloadResponse, notNullValue());
-                            final Map<String, NodesReloadSecureSettingsResponse.NodeResponse> nodesMap = nodesReloadResponse.getNodesMap();
-                            assertThat(nodesMap.size(), equalTo(cluster().size()));
-                            for (final NodesReloadSecureSettingsResponse.NodeResponse nodeResponse : nodesReloadResponse.getNodes()) {
-                                assertThat(nodeResponse.reloadException(), notNullValue());
-                                    assertThat(nodeResponse.reloadException(), instanceOf(SecurityException.class));
-                            }
-                        } catch (final AssertionError e) {
-                            reloadSettingsError.set(e);
-                        } finally {
-                            latch.countDown();
-                        }
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        reloadSettingsError.set(new AssertionError("Nodes request failed", e));
-                        latch.countDown();
-                    }
-                });
-        latch.await();
-        if (reloadSettingsError.get() != null) {
-            throw reloadSettingsError.get();
-        }
-        // in the wrong password case no reload should be triggered
-        assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount));
-    }
-
     public void testMisbehavingPlugin() throws Exception {
         final Environment environment = internalCluster().getInstance(Environment.class);
         final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class);
@@ -247,7 +161,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
                 .get(Settings.builder().put(environment.settings()).setSecureSettings(secureSettings).build())
                 .toString();
         final CountDownLatch latch = new CountDownLatch(1);
-        client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute(
+        client().admin().cluster().prepareReloadSecureSettings().execute(
                 new ActionListener<NodesReloadSecureSettingsResponse>() {
                     @Override
                     public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
@@ -314,7 +228,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
     private void successfulReloadCall() throws InterruptedException {
         final AtomicReference<AssertionError> reloadSettingsError = new AtomicReference<>();
         final CountDownLatch latch = new CountDownLatch(1);
-        client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute(
+        client().admin().cluster().prepareReloadSecureSettings().execute(
                 new ActionListener<NodesReloadSecureSettingsResponse>() {
                     @Override
                     public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {

From b87f3062b77cab7888e9037b4996f2c26db12816 Mon Sep 17 00:00:00 2001
From: Hazem Khaled <hazem.khaled@gmail.com>
Date: Thu, 16 Aug 2018 14:54:04 +0300
Subject: [PATCH 11/16] [DOCS] Update WordPress plugins links (#32194)

---
 docs/plugins/integrations.asciidoc | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc
index 90f2c685fda..8bffe5193ed 100644
--- a/docs/plugins/integrations.asciidoc
+++ b/docs/plugins/integrations.asciidoc
@@ -17,14 +17,11 @@ Integrations are not plugins, but are external tools or modules that make it eas
 * https://drupal.org/project/elasticsearch_connector[Drupal]:
   Drupal Elasticsearch integration.
 
-* https://wordpress.org/plugins/wpsolr-search-engine/[WPSOLR]:
-  Elasticsearch (and Apache Solr) WordPress Plugin
- 
-* http://searchbox-io.github.com/wp-elasticsearch/[Wp-Elasticsearch]:
+* https://wordpress.org/plugins/elasticpress/[ElasticPress]:
   Elasticsearch WordPress Plugin
 
-* https://github.com/wallmanderco/elasticsearch-indexer[Elasticsearch Indexer]:
-  Elasticsearch WordPress Plugin
+* https://wordpress.org/plugins/wpsolr-search-engine/[WPSOLR]:
+  Elasticsearch (and Apache Solr) WordPress Plugin
 
 * https://doc.tiki.org/Elasticsearch[Tiki Wiki CMS Groupware]:
   Tiki has native support for Elasticsearch. This provides faster & better

From aedc2c1c498688970afcebe08726480e611f117b Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Thu, 16 Aug 2018 07:18:43 -0500
Subject: [PATCH 12/16] HLRC: adding machine learning delete job (#32820)

* HLRC: adding machine learning delete job

* Fixing whitespace

* Moving docs and tests around

* Unifying ml asciidoc file naming convention
---
 .../client/MachineLearningClient.java         | 40 ++++++++++
 .../client/RequestConverters.java             | 16 ++++
 .../client/MachineLearningIT.java             | 15 ++++
 .../client/RequestConvertersTests.java        | 15 ++++
 .../MlClientDocumentationIT.java              | 56 +++++++++++++-
 .../high-level/ml/delete-job.asciidoc         | 49 ++++++++++++
 .../high-level/supported-apis.asciidoc        |  2 +
 .../protocol/xpack/ml/DeleteJobRequest.java   | 75 +++++++++++++++++++
 .../protocol/xpack/ml/DeleteJobResponse.java  | 60 +++++++++++++++
 .../xpack/ml/DeleteJobRequestTests.java       | 45 +++++++++++
 .../xpack/ml/DeleteJobResponseTests.java      | 42 +++++++++++
 11 files changed, 412 insertions(+), 3 deletions(-)
 create mode 100644 docs/java-rest/high-level/ml/delete-job.asciidoc
 create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequest.java
 create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponse.java
 create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequestTests.java
 create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponseTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
index a3e5ba72b77..2e7914e64ab 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
@@ -19,6 +19,8 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse;
 import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
 import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
 import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
@@ -80,6 +82,44 @@ public final class MachineLearningClient {
             Collections.emptySet());
     }
 
+    /**
+     * Deletes the given Machine Learning Job
+     * <p>
+     *     For additional info
+     *     see <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-job.html">ML Delete Job documentation</a>
+     * </p>
+     * @param request the request to delete the job
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return action acknowledgement
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public DeleteJobResponse deleteJob(DeleteJobRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            RequestConverters::deleteMachineLearningJob,
+            options,
+            DeleteJobResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Deletes the given Machine Learning Job asynchronously and notifies the listener on completion
+     * <p>
+     *     For additional info
+     *     see <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-job.html">ML Delete Job documentation</a>
+     * </p>
+     * @param request the request to delete the job
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void deleteJobAsync(DeleteJobRequest request, RequestOptions options, ActionListener<DeleteJobResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            RequestConverters::deleteMachineLearningJob,
+            options,
+            DeleteJobResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
     /**
      * Opens a Machine Learning Job.
      * When you open a new job, it starts with an empty model.
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index 973c0ce126d..c40b4893e01 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -112,6 +112,7 @@ import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
 import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
 import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
 import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
@@ -1211,6 +1212,21 @@ final class RequestConverters {
         return request;
     }
 
+    static Request deleteMachineLearningJob(DeleteJobRequest deleteJobRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("ml")
+            .addPathPartAsIs("anomaly_detectors")
+            .addPathPart(deleteJobRequest.getJobId())
+            .build();
+        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
+
+        Params params = new Params(request);
+        params.putParam("force", Boolean.toString(deleteJobRequest.isForce()));
+
+        return request;
+    }
+
     static Request machineLearningOpenJob(OpenJobRequest openJobRequest) throws IOException {
         String endpoint = new EndpointBuilder()
             .addPathPartAsIs("_xpack")
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
index 94e73a14c18..0037460150f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -20,6 +20,8 @@ package org.elasticsearch.client;
 
 import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse;
 import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
 import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
 import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
@@ -48,6 +50,19 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
         assertThat(createdJob.getJobType(), is(Job.ANOMALY_DETECTOR_JOB_TYPE));
     }
 
+    public void testDeleteJob() throws Exception {
+        String jobId = randomValidJobId();
+        Job job = buildJob(jobId);
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        machineLearningClient.putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
+
+        DeleteJobResponse response = execute(new DeleteJobRequest(jobId),
+            machineLearningClient::deleteJob,
+            machineLearningClient::deleteJobAsync);
+
+        assertTrue(response.isAcknowledged());
+    }
+
     public void testOpenJob() throws Exception {
         String jobId = randomValidJobId();
         Job job = buildJob(jobId);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index 1c9707e0e27..786cb94f892 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -127,6 +127,7 @@ import org.elasticsearch.index.rankeval.RatedRequest;
 import org.elasticsearch.index.rankeval.RestRankEvalAction;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
 import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
@@ -2611,6 +2612,20 @@ public class RequestConvertersTests extends ESTestCase {
         assertThat(request.getEntity(), nullValue());
     }
 
+    public void testDeleteMachineLearningJob() {
+        String jobId = randomAlphaOfLength(10);
+        DeleteJobRequest deleteJobRequest = new DeleteJobRequest(jobId);
+
+        Request request = RequestConverters.deleteMachineLearningJob(deleteJobRequest);
+        assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
+        assertEquals("/_xpack/ml/anomaly_detectors/" + jobId, request.getEndpoint());
+        assertEquals(Boolean.toString(false), request.getParameters().get("force"));
+
+        deleteJobRequest.setForce(true);
+        request = RequestConverters.deleteMachineLearningJob(deleteJobRequest);
+        assertEquals(Boolean.toString(true), request.getParameters().get("force"));
+    }
+
     public void testPostMachineLearningOpenJob() throws Exception {
         String jobId = "some-job-id";
         OpenJobRequest openJobRequest = new OpenJobRequest(jobId);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
index 50cd244c0fa..a77d8b43e57 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
@@ -25,6 +25,8 @@ import org.elasticsearch.client.MachineLearningIT;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
+import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse;
 import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
 import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
 import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
@@ -122,6 +124,56 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
         }
     }
 
+    public void testDeleteJob() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        String jobId = "my-first-machine-learning-job";
+
+        Job job = MachineLearningIT.buildJob(jobId);
+        client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
+
+        Job secondJob = MachineLearningIT.buildJob("my-second-machine-learning-job");
+        client.machineLearning().putJob(new PutJobRequest(secondJob), RequestOptions.DEFAULT);
+
+        {
+            //tag::x-pack-delete-ml-job-request
+            DeleteJobRequest deleteJobRequest = new DeleteJobRequest("my-first-machine-learning-job");
+            deleteJobRequest.setForce(false); //<1>
+            DeleteJobResponse deleteJobResponse = client.machineLearning().deleteJob(deleteJobRequest, RequestOptions.DEFAULT);
+            //end::x-pack-delete-ml-job-request
+
+            //tag::x-pack-delete-ml-job-response
+            boolean isAcknowledged = deleteJobResponse.isAcknowledged(); //<1>
+            //end::x-pack-delete-ml-job-response
+        }
+        {
+            //tag::x-pack-delete-ml-job-request-listener
+            ActionListener<DeleteJobResponse> listener = new ActionListener<DeleteJobResponse>() {
+                @Override
+                public void onResponse(DeleteJobResponse deleteJobResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            //end::x-pack-delete-ml-job-request-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            //tag::x-pack-delete-ml-job-request-async
+            DeleteJobRequest deleteJobRequest = new DeleteJobRequest("my-second-machine-learning-job");
+            client.machineLearning().deleteJobAsync(deleteJobRequest, RequestOptions.DEFAULT, listener); // <1>
+            //end::x-pack-delete-ml-job-request-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
     public void testOpenJob() throws Exception {
         RestHighLevelClient client = highLevelClient();
 
@@ -143,7 +195,6 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
             //end::x-pack-ml-open-job-execute
 
         }
-
         {
             //tag::x-pack-ml-open-job-listener
             ActionListener<OpenJobResponse> listener = new ActionListener<OpenJobResponse>() {
@@ -154,7 +205,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
 
                 @Override
                 public void onFailure(Exception e) {
-                    //<2>
+                    // <2>
                 }
             };
             //end::x-pack-ml-open-job-listener
@@ -169,6 +220,5 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
         }
-
     }
 }
diff --git a/docs/java-rest/high-level/ml/delete-job.asciidoc b/docs/java-rest/high-level/ml/delete-job.asciidoc
new file mode 100644
index 00000000000..44a6a479409
--- /dev/null
+++ b/docs/java-rest/high-level/ml/delete-job.asciidoc
@@ -0,0 +1,49 @@
+[[java-rest-high-x-pack-ml-delete-job]]
+=== Delete Job API
+
+[[java-rest-high-x-pack-machine-learning-delete-job-request]]
+==== Delete Job Request
+
+A `DeleteJobRequest` object requires a non-null `jobId` and can optionally set `force`.
+Can be executed as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request]
+---------------------------------------------------
+<1> Use to forcefully delete an opened job;
+this method is quicker than closing and deleting the job.
+Defaults to `false`
+
+[[java-rest-high-x-pack-machine-learning-delete-job-response]]
+==== Delete Job Response
+
+The returned `DeleteJobResponse` object indicates the acknowledgement of the request:
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-response]
+---------------------------------------------------
+<1> `isAcknowledged` was the deletion request acknowledged or not
+
+[[java-rest-high-x-pack-machine-learning-delete-job-async]]
+==== Delete Job Asynchronously
+
+This request can also be made asynchronously.
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request-async]
+---------------------------------------------------
+<1> The `DeleteJobRequest` to execute and the `ActionListener` to alert on completion or error.
+
+The deletion request returns immediately. Once the request is completed, the `ActionListener` is
+called back using the `onResponse` or `onFailure`. The latter indicates some failure occurred when
+making the request.
+
+A typical listener for a `DeleteJobRequest` could be defined as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request-listener]
+---------------------------------------------------
+<1> The action to be taken when it is completed
+<2> What to do when a failure occurs
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
index a2db3436317..6bcb736243a 100644
--- a/docs/java-rest/high-level/supported-apis.asciidoc
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -205,9 +205,11 @@ include::licensing/delete-license.asciidoc[]
 The Java High Level REST Client supports the following Machine Learning APIs:
 
 * <<java-rest-high-x-pack-ml-put-job>>
+* <<java-rest-high-x-pack-ml-delete-job>>
 * <<java-rest-high-x-pack-ml-open-job>>
 
 include::ml/put-job.asciidoc[]
+include::ml/delete-job.asciidoc[]
 include::ml/open-job.asciidoc[]
 
 == Migration APIs
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequest.java
new file mode 100644
index 00000000000..1b7450de092
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+
+import java.util.Objects;
+
+public class DeleteJobRequest extends ActionRequest {
+
+    private String jobId;
+    private boolean force;
+
+    public DeleteJobRequest(String jobId) {
+        this.jobId = Objects.requireNonNull(jobId, "[job_id] must not be null");
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public void setJobId(String jobId) {
+        this.jobId = Objects.requireNonNull(jobId, "[job_id] must not be null");
+    }
+
+    public boolean isForce() {
+        return force;
+    }
+
+    public void setForce(boolean force) {
+        this.force = force;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+       return null;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, force);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+
+        DeleteJobRequest other = (DeleteJobRequest) obj;
+        return Objects.equals(jobId, other.jobId) && Objects.equals(force, other.force);
+    }
+
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponse.java
new file mode 100644
index 00000000000..0b4faa38f54
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponse.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml;
+
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DeleteJobResponse extends AcknowledgedResponse {
+
+   public DeleteJobResponse(boolean acknowledged) {
+       super(acknowledged);
+   }
+
+   public DeleteJobResponse() {
+   }
+
+    public static DeleteJobResponse fromXContent(XContentParser parser) throws IOException {
+        AcknowledgedResponse response = AcknowledgedResponse.fromXContent(parser);
+        return new DeleteJobResponse(response.isAcknowledged());
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        DeleteJobResponse that = (DeleteJobResponse) other;
+        return isAcknowledged() == that.isAcknowledged();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(isAcknowledged());
+    }
+
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequestTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequestTests.java
new file mode 100644
index 00000000000..fb8a38fa0c6
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequestTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml;
+
+import org.elasticsearch.protocol.xpack.ml.job.config.JobTests;
+import org.elasticsearch.test.ESTestCase;
+
+public class DeleteJobRequestTests extends ESTestCase {
+
+    private DeleteJobRequest createTestInstance() {
+        return new DeleteJobRequest(JobTests.randomValidJobId());
+    }
+
+    public void test_WithNullJobId() {
+        NullPointerException ex = expectThrows(NullPointerException.class, () -> new DeleteJobRequest(null));
+        assertEquals("[job_id] must not be null", ex.getMessage());
+
+        ex = expectThrows(NullPointerException.class, () -> createTestInstance().setJobId(null));
+        assertEquals("[job_id] must not be null", ex.getMessage());
+    }
+
+    public void test_WithForce() {
+        DeleteJobRequest deleteJobRequest = createTestInstance();
+        assertFalse(deleteJobRequest.isForce());
+
+        deleteJobRequest.setForce(true);
+        assertTrue(deleteJobRequest.isForce());
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponseTests.java
new file mode 100644
index 00000000000..a73179a0898
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponseTests.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class DeleteJobResponseTests extends AbstractXContentTestCase<DeleteJobResponse> {
+
+    @Override
+    protected DeleteJobResponse createTestInstance() {
+        return new DeleteJobResponse();
+    }
+
+    @Override
+    protected DeleteJobResponse doParseInstance(XContentParser parser) throws IOException {
+        return DeleteJobResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}

From 3dd1677cdc8fa6b4c36a79f7b206430661469ce8 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jimczi@apache.org>
Date: Thu, 16 Aug 2018 15:33:17 +0200
Subject: [PATCH 13/16] [Test] Fix
 DuelScrollIT#testDuelIndexOrderQueryThenFetch

This commit disables the automatic `refresh_interval` in order to ensure
that index readers cannot differ between the normal and scroll search.
This issue is related to the 7.5 Lucene upgrade which contains a change that
makes single segment merge more likely to occur (max deletes percentage).

Closes #32682
---
 .../java/org/elasticsearch/search/scroll/DuelScrollIT.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
index 1ddd11e5d0f..4005f1218a9 100644
--- a/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
+++ b/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
@@ -199,6 +199,8 @@ public class DuelScrollIT extends ESIntegTestCase {
         }
         // no replicas, as they might be ordered differently
         settings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
+        // we need to control refreshes as they might take different merges into account
+        settings.put("index.refresh_interval", -1);
 
         assertAcked(prepareCreate("test").setSettings(settings.build()).get());
         final int numDocs = randomIntBetween(10, 200);
@@ -257,7 +259,6 @@ public class DuelScrollIT extends ESIntegTestCase {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32682")
     public void testDuelIndexOrderQueryThenFetch() throws Exception {
         final SearchType searchType = RandomPicks.randomFrom(random(), Arrays.asList(SearchType.QUERY_THEN_FETCH,
             SearchType.DFS_QUERY_THEN_FETCH));

From eaaf37a1f99309c8a7c176c9b2cc7f56ec5c69d1 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 16 Aug 2018 15:25:51 +0200
Subject: [PATCH 14/16] AwaitFix
 FullClusterRestartIT#testRollupIDSchemeAfterRestart.

---
 .../org/elasticsearch/xpack/restart/FullClusterRestartIT.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
index 24303b8342b..6ead87aba61 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -325,6 +325,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
         }
     }
 
+    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/32773")
     public void testRollupIDSchemeAfterRestart() throws Exception {
         assumeTrue("Rollup can be tested with 6.3.0 and onwards", oldClusterVersion.onOrAfter(Version.V_6_3_0));
         assumeTrue("Rollup ID scheme changed in 6.4", oldClusterVersion.before(Version.V_6_4_0));

From d9fd74bcdcabd5020e40ef7e5dd67c6de09ae0f0 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Thu, 16 Aug 2018 08:03:21 -0700
Subject: [PATCH 15/16] Painless: Special Case def (#32871)

This removes def from the classes map in PainlessLookup and instead always special
cases it. This prevents potential calls against the def type that shouldn't be made and
forces all cases of def throughout Painless code to be special cased.
---
 .../java/org/elasticsearch/painless/ScriptClassInfo.java     | 3 ++-
 .../org/elasticsearch/painless/lookup/PainlessLookup.java    | 3 ++-
 .../elasticsearch/painless/lookup/PainlessLookupBuilder.java | 5 +----
 .../elasticsearch/painless/lookup/PainlessLookupUtility.java | 4 ++--
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
index 345db46f887..7de8353194d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.def;
 
 import java.lang.invoke.MethodType;
 import java.lang.reflect.Field;
@@ -190,7 +191,7 @@ public class ScriptClassInfo {
             componentType = componentType.getComponentType();
         }
 
-        if (painlessLookup.lookupPainlessClass(componentType) == null) {
+        if (componentType != def.class && painlessLookup.lookupPainlessClass(componentType) == null) {
             throw new IllegalArgumentException(unknownErrorMessageSource.apply(componentType));
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
index 16b8ac14f14..55855a3cb1e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
@@ -26,6 +26,7 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
 
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.DEF_CLASS_NAME;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessConstructorKey;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessFieldKey;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey;
@@ -47,7 +48,7 @@ public final class PainlessLookup {
     public boolean isValidCanonicalClassName(String canonicalClassName) {
         Objects.requireNonNull(canonicalClassName);
 
-        return canonicalClassNamesToClasses.containsKey(canonicalClassName);
+        return DEF_CLASS_NAME.equals(canonicalClassName) || canonicalClassNamesToClasses.containsKey(canonicalClassName);
     }
 
     public Class<?> canonicalTypeNameToType(String canonicalTypeName) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index e644453a4c1..c8353b54c9f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -211,9 +211,6 @@ public final class PainlessLookupBuilder {
     public PainlessLookupBuilder() {
         canonicalClassNamesToClasses = new HashMap<>();
         classesToPainlessClassBuilders = new HashMap<>();
-
-        canonicalClassNamesToClasses.put(DEF_CLASS_NAME, def.class);
-        classesToPainlessClassBuilders.put(def.class, new PainlessClassBuilder());
     }
 
     private Class<?> canonicalTypeNameToType(String canonicalTypeName) {
@@ -225,7 +222,7 @@ public final class PainlessLookupBuilder {
             type = type.getComponentType();
         }
 
-        return classesToPainlessClassBuilders.containsKey(type);
+        return type == def.class || classesToPainlessClassBuilders.containsKey(type);
     }
 
     public void addPainlessClass(ClassLoader classLoader, String javaClassName, boolean importClassName) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
index f2eb4345169..71cacab9eba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
@@ -82,7 +82,7 @@ public final class PainlessLookupUtility {
         Objects.requireNonNull(canonicalTypeName);
         Objects.requireNonNull(canonicalClassNamesToClasses);
 
-        Class<?> type = canonicalClassNamesToClasses.get(canonicalTypeName);
+        Class<?> type = DEF_CLASS_NAME.equals(canonicalTypeName) ? def.class : canonicalClassNamesToClasses.get(canonicalTypeName);
 
         if (type != null) {
             return type;
@@ -105,7 +105,7 @@ public final class PainlessLookupUtility {
             }
 
             canonicalTypeName = canonicalTypeName.substring(0, canonicalTypeName.indexOf('['));
-            type = canonicalClassNamesToClasses.get(canonicalTypeName);
+            type = DEF_CLASS_NAME.equals(canonicalTypeName) ? def.class : canonicalClassNamesToClasses.get(canonicalTypeName);
 
             if (type != null) {
                 char arrayBraces[] = new char[arrayDimensions];

From b5a85362e30632f256dd4f2b8842c55ac64482f6 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 16 Aug 2018 11:32:35 -0400
Subject: [PATCH 16/16] Fix docs for fixed filename for heap dump path (#32882)

The docs here incorrectly state that it is okay for a heap dump file to
exist when heap dump path is configured to a fixed filename. This is
incorrect, the JVM will fail to write the heap dump if a heap dump file
already exists at the specified location (see the DumpWriter constructor
DumpWriter::DumpWriter(const char* path) in the JVM source).
---
 .../setup/important-settings/heap-dump-path.asciidoc   | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/reference/setup/important-settings/heap-dump-path.asciidoc b/docs/reference/setup/important-settings/heap-dump-path.asciidoc
index b0d301b21d0..fb8c7ff35f0 100644
--- a/docs/reference/setup/important-settings/heap-dump-path.asciidoc
+++ b/docs/reference/setup/important-settings/heap-dump-path.asciidoc
@@ -8,8 +8,8 @@ distributions, and the `data` directory under the root of the
 Elasticsearch installation for the <<zip-targz,tar and zip>> archive
 distributions). If this path is not suitable for receiving heap dumps,
 you should modify the entry `-XX:HeapDumpPath=...` in
-<<jvm-options,`jvm.options`>>. If you specify a fixed filename instead
-of a directory, the JVM will repeatedly use the same file; this is one
-mechanism for preventing heap dumps from accumulating in the heap dump
-path. Alternatively, you can configure a scheduled task via your OS to
-remove heap dumps that are older than a configured age.
+<<jvm-options,`jvm.options`>>. If you specify a directory, the JVM
+will generate a filename for the heap dump based on the PID of the running
+instance. If you specify a fixed filename instead of a directory, the file must
+not exist when the JVM needs to perform a heap dump on an out of memory
+exception, otherwise the heap dump will fail.