honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into index-lifecycle

This commit is contained in:
Tal Levy 2018-08-16 08:41:57 -07:00
parent 238eb93893 b5a85362e3
commit c9de707f58
48 changed files with 714 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.client;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse;
import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
@ -80,6 +82,44 @@ public final class MachineLearningClient {
Collections.emptySet());
}
/**
* Deletes the given Machine Learning Job
* <p>
* For additional info
* see <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-job.html">ML Delete Job documentation</a>
* </p>
* @param request the request to delete the job
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return action acknowledgement
* @throws IOException when there is a serialization issue sending the request or receiving the response
*/
public DeleteJobResponse deleteJob(DeleteJobRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(request,
RequestConverters::deleteMachineLearningJob,
options,
DeleteJobResponse::fromXContent,
Collections.emptySet());
}
/**
* Deletes the given Machine Learning Job asynchronously and notifies the listener on completion
* <p>
* For additional info
* see <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-job.html">ML Delete Job documentation</a>
* </p>
* @param request the request to delete the job
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener Listener to be notified upon request completion
*/
public void deleteJobAsync(DeleteJobRequest request, RequestOptions options, ActionListener<DeleteJobResponse> listener) {
restHighLevelClient.performRequestAsyncAndParseEntity(request,
RequestConverters::deleteMachineLearningJob,
options,
DeleteJobResponse::fromXContent,
listener,
Collections.emptySet());
}
/**
* Opens a Machine Learning Job.
* When you open a new job, it starts with an empty model.

16
client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
View File

@ -116,6 +116,7 @@ import org.elasticsearch.protocol.xpack.indexlifecycle.StopILMRequest;
import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
@ -1267,6 +1268,21 @@ final class RequestConverters {
return request;
}
static Request deleteMachineLearningJob(DeleteJobRequest deleteJobRequest) {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_xpack")
.addPathPartAsIs("ml")
.addPathPartAsIs("anomaly_detectors")
.addPathPart(deleteJobRequest.getJobId())
.build();
Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
Params params = new Params(request);
params.putParam("force", Boolean.toString(deleteJobRequest.isForce()));
return request;
}
static Request machineLearningOpenJob(OpenJobRequest openJobRequest) throws IOException {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_xpack")

15
client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
View File

@ -20,6 +20,8 @@ package org.elasticsearch.client;
import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse;
import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
@ -48,6 +50,19 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
assertThat(createdJob.getJobType(), is(Job.ANOMALY_DETECTOR_JOB_TYPE));
}
public void testDeleteJob() throws Exception {
String jobId = randomValidJobId();
Job job = buildJob(jobId);
MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
machineLearningClient.putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
DeleteJobResponse response = execute(new DeleteJobRequest(jobId),
machineLearningClient::deleteJob,
machineLearningClient::deleteJobAsync);
assertTrue(response.isAcknowledged());
}
public void testOpenJob() throws Exception {
String jobId = randomValidJobId();
Job job = buildJob(jobId);

15
client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
View File

@ -131,6 +131,7 @@ import org.elasticsearch.protocol.xpack.indexlifecycle.SetIndexLifecyclePolicyRe
import org.elasticsearch.protocol.xpack.indexlifecycle.StartILMRequest;
import org.elasticsearch.protocol.xpack.indexlifecycle.StopILMRequest;
import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
@ -2673,6 +2674,20 @@ public class RequestConvertersTests extends ESTestCase {
assertThat(request.getEntity(), nullValue());
}
public void testDeleteMachineLearningJob() {
String jobId = randomAlphaOfLength(10);
DeleteJobRequest deleteJobRequest = new DeleteJobRequest(jobId);
Request request = RequestConverters.deleteMachineLearningJob(deleteJobRequest);
assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
assertEquals("/_xpack/ml/anomaly_detectors/" + jobId, request.getEndpoint());
assertEquals(Boolean.toString(false), request.getParameters().get("force"));
deleteJobRequest.setForce(true);
request = RequestConverters.deleteMachineLearningJob(deleteJobRequest);
assertEquals(Boolean.toString(true), request.getParameters().get("force"));
}
public void testPostMachineLearningOpenJob() throws Exception {
String jobId = "some-job-id";
OpenJobRequest openJobRequest = new OpenJobRequest(jobId);

56
client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
View File

@ -25,6 +25,8 @@ import org.elasticsearch.client.MachineLearningIT;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse;
import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
@ -122,6 +124,56 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
}
}
public void testDeleteJob() throws Exception {
RestHighLevelClient client = highLevelClient();
String jobId = "my-first-machine-learning-job";
Job job = MachineLearningIT.buildJob(jobId);
client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
Job secondJob = MachineLearningIT.buildJob("my-second-machine-learning-job");
client.machineLearning().putJob(new PutJobRequest(secondJob), RequestOptions.DEFAULT);
{
//tag::x-pack-delete-ml-job-request
DeleteJobRequest deleteJobRequest = new DeleteJobRequest("my-first-machine-learning-job");
deleteJobRequest.setForce(false); //<1>
DeleteJobResponse deleteJobResponse = client.machineLearning().deleteJob(deleteJobRequest, RequestOptions.DEFAULT);
//end::x-pack-delete-ml-job-request
//tag::x-pack-delete-ml-job-response
boolean isAcknowledged = deleteJobResponse.isAcknowledged(); //<1>
//end::x-pack-delete-ml-job-response
}
{
//tag::x-pack-delete-ml-job-request-listener
ActionListener<DeleteJobResponse> listener = new ActionListener<DeleteJobResponse>() {
@Override
public void onResponse(DeleteJobResponse deleteJobResponse) {
// <1>
}
@Override
public void onFailure(Exception e) {
// <2>
}
};
//end::x-pack-delete-ml-job-request-listener
// Replace the empty listener by a blocking listener in test
final CountDownLatch latch = new CountDownLatch(1);
listener = new LatchedActionListener<>(listener, latch);
//tag::x-pack-delete-ml-job-request-async
DeleteJobRequest deleteJobRequest = new DeleteJobRequest("my-second-machine-learning-job");
client.machineLearning().deleteJobAsync(deleteJobRequest, RequestOptions.DEFAULT, listener); // <1>
//end::x-pack-delete-ml-job-request-async
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}
public void testOpenJob() throws Exception {
RestHighLevelClient client = highLevelClient();
@ -143,7 +195,6 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
//end::x-pack-ml-open-job-execute
}
{
//tag::x-pack-ml-open-job-listener
ActionListener<OpenJobResponse> listener = new ActionListener<OpenJobResponse>() {
@ -154,7 +205,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
@Override
public void onFailure(Exception e) {
//<2>
// <2>
}
};
//end::x-pack-ml-open-job-listener
@ -169,6 +220,5 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}
}

49
docs/java-rest/high-level/ml/delete-job.asciidoc Normal file
View File

@ -0,0 +1,49 @@
[[java-rest-high-x-pack-ml-delete-job]]
=== Delete Job API
[[java-rest-high-x-pack-machine-learning-delete-job-request]]
==== Delete Job Request
A `DeleteJobRequest` object requires a non-null `jobId` and can optionally set `force`.
Can be executed as follows:
["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request]
---------------------------------------------------
<1> Use to forcefully delete an opened job;
this method is quicker than closing and deleting the job.
Defaults to `false`
[[java-rest-high-x-pack-machine-learning-delete-job-response]]
==== Delete Job Response
The returned `DeleteJobResponse` object indicates the acknowledgement of the request:
["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-response]
---------------------------------------------------
<1> `isAcknowledged` was the deletion request acknowledged or not
[[java-rest-high-x-pack-machine-learning-delete-job-async]]
==== Delete Job Asynchronously
This request can also be made asynchronously.
["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request-async]
---------------------------------------------------
<1> The `DeleteJobRequest` to execute and the `ActionListener` to alert on completion or error.
The deletion request returns immediately. Once the request is completed, the `ActionListener` is
called back using the `onResponse` or `onFailure`. The latter indicates some failure occurred when
making the request.
A typical listener for a `DeleteJobRequest` could be defined as follows:
["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-delete-ml-job-request-listener]
---------------------------------------------------
<1> The action to be taken when it is completed
<2> What to do when a failure occurs

2
docs/java-rest/high-level/supported-apis.asciidoc
View File

@ -205,9 +205,11 @@ include::licensing/delete-license.asciidoc[]
The Java High Level REST Client supports the following Machine Learning APIs:
* <<java-rest-high-x-pack-ml-put-job>>
* <<java-rest-high-x-pack-ml-delete-job>>
* <<java-rest-high-x-pack-ml-open-job>>
include::ml/put-job.asciidoc[]
include::ml/delete-job.asciidoc[]
include::ml/open-job.asciidoc[]
== Migration APIs

9
docs/plugins/integrations.asciidoc
View File

@ -17,14 +17,11 @@ Integrations are not plugins, but are external tools or modules that make it eas
* https://drupal.org/project/elasticsearch_connector[Drupal]:
Drupal Elasticsearch integration.
* https://wordpress.org/plugins/wpsolr-search-engine/[WPSOLR]:
Elasticsearch (and Apache Solr) WordPress Plugin
* http://searchbox-io.github.com/wp-elasticsearch/[Wp-Elasticsearch]:
* https://wordpress.org/plugins/elasticpress/[ElasticPress]:
Elasticsearch WordPress Plugin
* https://github.com/wallmanderco/elasticsearch-indexer[Elasticsearch Indexer]:
Elasticsearch WordPress Plugin
* https://wordpress.org/plugins/wpsolr-search-engine/[WPSOLR]:
Elasticsearch (and Apache Solr) WordPress Plugin
* https://doc.tiki.org/Elasticsearch[Tiki Wiki CMS Groupware]:
Tiki has native support for Elasticsearch. This provides faster & better

3
docs/reference/search/request-body.asciidoc
View File

@ -90,7 +90,8 @@ And here is a sample response:
Set to `false` to return an overall failure if the request would produce partial
results. Defaults to true, which will allow partial results in the case of timeouts
or partial failures.
or partial failures. This default can be controlled using the cluster-level setting
`search.default_allow_partial_results`.
`terminate_after`::

3
docs/reference/search/uri-request.asciidoc
View File

@ -125,5 +125,6 @@ more details on the different types of search that can be performed.
|`allow_partial_search_results` |Set to `false` to return an overall failure if the request would produce
partial results. Defaults to true, which will allow partial results in the case of timeouts
or partial failures..
or partial failures. This default can be controlled using the cluster-level setting
`search.default_allow_partial_results`.
|=======================================================================

10
docs/reference/setup/important-settings/heap-dump-path.asciidoc
View File

@ -8,8 +8,8 @@ distributions, and the `data` directory under the root of the
Elasticsearch installation for the <<zip-targz,tar and zip>> archive
distributions). If this path is not suitable for receiving heap dumps,
you should modify the entry `-XX:HeapDumpPath=...` in
<<jvm-options,`jvm.options`>>. If you specify a fixed filename instead
of a directory, the JVM will repeatedly use the same file; this is one
mechanism for preventing heap dumps from accumulating in the heap dump
path. Alternatively, you can configure a scheduled task via your OS to
remove heap dumps that are older than a configured age.
<<jvm-options,`jvm.options`>>. If you specify a directory, the JVM
will generate a filename for the heap dump based on the PID of the running
instance. If you specify a fixed filename instead of a directory, the file must
not exist when the JVM needs to perform a heap dump on an out of memory
exception, otherwise the heap dump will fail.

2
docs/reference/setup/important-settings/network-host.asciidoc
View File

@ -9,7 +9,7 @@ location on a single node. This can be useful for testing Elasticsearch's
ability to form clusters, but it is not a configuration recommended for
production.
In order to communicate and to form a cluster with nodes on other servers, your
In order to form a cluster with nodes on other servers, your
node will need to bind to a non-loopback address. While there are many
<<modules-network,network settings>>, usually all you need to configure is
`network.host`:

150
libs/core/src/main/java/org/elasticsearch/common/CharArrays.java Normal file
View File

@ -0,0 +1,150 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Objects;
/**
* Helper class similar to Arrays to handle conversions for Char arrays
*/
public final class CharArrays {
private CharArrays() {}
/**
* Decodes the provided byte[] to a UTF-8 char[]. This is done while avoiding
* conversions to String. The provided byte[] is not modified by this method, so
* the caller needs to take care of clearing the value if it is sensitive.
*/
public static char[] utf8BytesToChars(byte[] utf8Bytes) {
final ByteBuffer byteBuffer = ByteBuffer.wrap(utf8Bytes);
final CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer);
final char[] chars;
if (charBuffer.hasArray()) {
// there is no guarantee that the char buffers backing array is the right size
// so we need to make a copy
chars = Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit());
Arrays.fill(charBuffer.array(), (char) 0); // clear sensitive data
} else {
final int length = charBuffer.limit() - charBuffer.position();
chars = new char[length];
charBuffer.get(chars);
// if the buffer is not read only we can reset and fill with 0's
if (charBuffer.isReadOnly() == false) {
charBuffer.clear(); // reset
for (int i = 0; i < charBuffer.limit(); i++) {
charBuffer.put((char) 0);
}
}
}
return chars;
}
/**
* Encodes the provided char[] to a UTF-8 byte[]. This is done while avoiding
* conversions to String. The provided char[] is not modified by this method, so
* the caller needs to take care of clearing the value if it is sensitive.
*/
public static byte[] toUtf8Bytes(char[] chars) {
final CharBuffer charBuffer = CharBuffer.wrap(chars);
final ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(charBuffer);
final byte[] bytes;
if (byteBuffer.hasArray()) {
// there is no guarantee that the byte buffers backing array is the right size
// so we need to make a copy
bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
Arrays.fill(byteBuffer.array(), (byte) 0); // clear sensitive data
} else {
final int length = byteBuffer.limit() - byteBuffer.position();
bytes = new byte[length];
byteBuffer.get(bytes);
// if the buffer is not read only we can reset and fill with 0's
if (byteBuffer.isReadOnly() == false) {
byteBuffer.clear(); // reset
for (int i = 0; i < byteBuffer.limit(); i++) {
byteBuffer.put((byte) 0);
}
}
}
return bytes;
}
/**
* Tests if a char[] contains a sequence of characters that match the prefix. This is like
* {@link String#startsWith(String)} but does not require conversion of the char[] to a string.
*/
public static boolean charsBeginsWith(String prefix, char[] chars) {
if (chars == null || prefix == null) {
return false;
}
if (prefix.length() > chars.length) {
return false;
}
for (int i = 0; i < prefix.length(); i++) {
if (chars[i] != prefix.charAt(i)) {
return false;
}
}
return true;
}
/**
* Constant time equality check of char arrays to avoid potential timing attacks.
*/
public static boolean constantTimeEquals(char[] a, char[] b) {
Objects.requireNonNull(a, "char arrays must not be null for constantTimeEquals");
Objects.requireNonNull(b, "char arrays must not be null for constantTimeEquals");
if (a.length != b.length) {
return false;
}
int equals = 0;
for (int i = 0; i < a.length; i++) {
equals |= a[i] ^ b[i];
}
return equals == 0;
}
/**
* Constant time equality check of strings to avoid potential timing attacks.
*/
public static boolean constantTimeEquals(String a, String b) {
Objects.requireNonNull(a, "strings must not be null for constantTimeEquals");
Objects.requireNonNull(b, "strings must not be null for constantTimeEquals");
if (a.length() != b.length()) {
return false;
}
int equals = 0;
for (int i = 0; i < a.length(); i++) {
equals |= a.charAt(i) ^ b.charAt(i);
}
return equals == 0;
}
}

75
libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common;
import org.elasticsearch.test.ESTestCase;
import java.nio.charset.StandardCharsets;
public class CharArraysTests extends ESTestCase {
public void testCharsToBytes() {
final String originalValue = randomUnicodeOfCodepointLengthBetween(0, 32);
final byte[] expectedBytes = originalValue.getBytes(StandardCharsets.UTF_8);
final char[] valueChars = originalValue.toCharArray();
final byte[] convertedBytes = CharArrays.toUtf8Bytes(valueChars);
assertArrayEquals(expectedBytes, convertedBytes);
}
public void testBytesToUtf8Chars() {
final String originalValue = randomUnicodeOfCodepointLengthBetween(0, 32);
final byte[] bytes = originalValue.getBytes(StandardCharsets.UTF_8);
final char[] expectedChars = originalValue.toCharArray();
final char[] convertedChars = CharArrays.utf8BytesToChars(bytes);
assertArrayEquals(expectedChars, convertedChars);
}
public void testCharsBeginsWith() {
assertFalse(CharArrays.charsBeginsWith(randomAlphaOfLength(4), null));
assertFalse(CharArrays.charsBeginsWith(null, null));
assertFalse(CharArrays.charsBeginsWith(null, randomAlphaOfLength(4).toCharArray()));
assertFalse(CharArrays.charsBeginsWith(randomAlphaOfLength(2), randomAlphaOfLengthBetween(3, 8).toCharArray()));
final String prefix = randomAlphaOfLengthBetween(2, 4);
assertTrue(CharArrays.charsBeginsWith(prefix, prefix.toCharArray()));
final char[] prefixedValue = prefix.concat(randomAlphaOfLengthBetween(1, 12)).toCharArray();
assertTrue(CharArrays.charsBeginsWith(prefix, prefixedValue));
final String modifiedPrefix = randomBoolean() ? prefix.substring(1) : prefix.substring(0, prefix.length() - 1);
char[] nonMatchingValue;
do {
nonMatchingValue = modifiedPrefix.concat(randomAlphaOfLengthBetween(0, 12)).toCharArray();
} while (new String(nonMatchingValue).startsWith(prefix));
assertFalse(CharArrays.charsBeginsWith(prefix, nonMatchingValue));
assertTrue(CharArrays.charsBeginsWith(modifiedPrefix, nonMatchingValue));
}
public void testConstantTimeEquals() {
final String value = randomAlphaOfLengthBetween(0, 32);
assertTrue(CharArrays.constantTimeEquals(value, value));
assertTrue(CharArrays.constantTimeEquals(value.toCharArray(), value.toCharArray()));
final String other = randomAlphaOfLengthBetween(1, 32);
assertFalse(CharArrays.constantTimeEquals(value, other));
assertFalse(CharArrays.constantTimeEquals(value.toCharArray(), other.toCharArray()));
}
}

3
modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.painless;
import org.elasticsearch.painless.lookup.PainlessLookup;
import org.elasticsearch.painless.lookup.PainlessLookupUtility;
import org.elasticsearch.painless.lookup.def;
import java.lang.invoke.MethodType;
import java.lang.reflect.Field;
@ -190,7 +191,7 @@ public class ScriptClassInfo {
componentType = componentType.getComponentType();
}
if (painlessLookup.lookupPainlessClass(componentType) == null) {
if (componentType != def.class && painlessLookup.lookupPainlessClass(componentType) == null) {
throw new IllegalArgumentException(unknownErrorMessageSource.apply(componentType));
}

3
modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
View File

@ -26,6 +26,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.DEF_CLASS_NAME;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessConstructorKey;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessFieldKey;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey;
@ -47,7 +48,7 @@ public final class PainlessLookup {
public boolean isValidCanonicalClassName(String canonicalClassName) {
Objects.requireNonNull(canonicalClassName);
return canonicalClassNamesToClasses.containsKey(canonicalClassName);
return DEF_CLASS_NAME.equals(canonicalClassName) || canonicalClassNamesToClasses.containsKey(canonicalClassName);
}
public Class<?> canonicalTypeNameToType(String canonicalTypeName) {

5
modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
View File

@ -211,9 +211,6 @@ public final class PainlessLookupBuilder {
public PainlessLookupBuilder() {
canonicalClassNamesToClasses = new HashMap<>();
classesToPainlessClassBuilders = new HashMap<>();
canonicalClassNamesToClasses.put(DEF_CLASS_NAME, def.class);
classesToPainlessClassBuilders.put(def.class, new PainlessClassBuilder());
}
private Class<?> canonicalTypeNameToType(String canonicalTypeName) {
@ -225,7 +222,7 @@ public final class PainlessLookupBuilder {
type = type.getComponentType();
}
return classesToPainlessClassBuilders.containsKey(type);
return type == def.class || classesToPainlessClassBuilders.containsKey(type);
}
public void addPainlessClass(ClassLoader classLoader, String javaClassName, boolean importClassName) {

4
modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
View File

@ -82,7 +82,7 @@ public final class PainlessLookupUtility {
Objects.requireNonNull(canonicalTypeName);
Objects.requireNonNull(canonicalClassNamesToClasses);
Class<?> type = canonicalClassNamesToClasses.get(canonicalTypeName);
Class<?> type = DEF_CLASS_NAME.equals(canonicalTypeName) ? def.class : canonicalClassNamesToClasses.get(canonicalTypeName);
if (type != null) {
return type;
@ -105,7 +105,7 @@ public final class PainlessLookupUtility {
}
canonicalTypeName = canonicalTypeName.substring(0, canonicalTypeName.indexOf('['));
type = canonicalClassNamesToClasses.get(canonicalTypeName);
type = DEF_CLASS_NAME.equals(canonicalTypeName) ? def.class : canonicalClassNamesToClasses.get(canonicalTypeName);
if (type != null) {
char arrayBraces[] = new char[arrayDimensions];

128
server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java
View File

@ -19,142 +19,22 @@
package org.elasticsearch.action.admin.cluster.node.reload;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.nodes.BaseNodesRequest;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.SecureString;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import static org.elasticsearch.action.ValidateActions.addValidationError;
/**
* Request for a reload secure settings action
* Request for a reload secure settings action.
*/
public class NodesReloadSecureSettingsRequest extends BaseNodesRequest<NodesReloadSecureSettingsRequest> {
/**
* The password which is broadcasted to all nodes, but is never stored on
* persistent storage. The password is used to reread and decrypt the contents
* of the node's keystore (backing the implementation of
* {@code SecureSettings}).
*/
private SecureString secureSettingsPassword;
public NodesReloadSecureSettingsRequest() {
}
/**
* Reload secure settings only on certain nodes, based on the nodes ids
* specified. If none are passed, secure settings will be reloaded on all the
* nodes.
* Reload secure settings only on certain nodes, based on the nodes IDs specified. If none are passed, secure settings will be reloaded
* on all the nodes.
*/
public NodesReloadSecureSettingsRequest(String... nodesIds) {
public NodesReloadSecureSettingsRequest(final String... nodesIds) {
super(nodesIds);
}
@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if (secureSettingsPassword == null) {
validationException = addValidationError("secure settings password cannot be null (use empty string instead)",
validationException);
}
return validationException;
}
public SecureString secureSettingsPassword() {
return secureSettingsPassword;
}
public NodesReloadSecureSettingsRequest secureStorePassword(SecureString secureStorePassword) {
this.secureSettingsPassword = secureStorePassword;
return this;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
final byte[] passwordBytes = in.readByteArray();
try {
this.secureSettingsPassword = new SecureString(utf8BytesToChars(passwordBytes));
} finally {
Arrays.fill(passwordBytes, (byte) 0);
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
final byte[] passwordBytes = charsToUtf8Bytes(this.secureSettingsPassword.getChars());
try {
out.writeByteArray(passwordBytes);
} finally {
Arrays.fill(passwordBytes, (byte) 0);
}
}
/**
* Encodes the provided char[] to a UTF-8 byte[]. This is done while avoiding
* conversions to String. The provided char[] is not modified by this method, so
* the caller needs to take care of clearing the value if it is sensitive.
*/
private static byte[] charsToUtf8Bytes(char[] chars) {
final CharBuffer charBuffer = CharBuffer.wrap(chars);
final ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(charBuffer);
final byte[] bytes;
if (byteBuffer.hasArray()) {
// there is no guarantee that the byte buffers backing array is the right size
// so we need to make a copy
bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
Arrays.fill(byteBuffer.array(), (byte) 0); // clear sensitive data
} else {
final int length = byteBuffer.limit() - byteBuffer.position();
bytes = new byte[length];
byteBuffer.get(bytes);
// if the buffer is not read only we can reset and fill with 0's
if (byteBuffer.isReadOnly() == false) {
byteBuffer.clear(); // reset
for (int i = 0; i < byteBuffer.limit(); i++) {
byteBuffer.put((byte) 0);
}
}
}
return bytes;
}
/**
* Decodes the provided byte[] to a UTF-8 char[]. This is done while avoiding
* conversions to String. The provided byte[] is not modified by this method, so
* the caller needs to take care of clearing the value if it is sensitive.
*/
public static char[] utf8BytesToChars(byte[] utf8Bytes) {
final ByteBuffer byteBuffer = ByteBuffer.wrap(utf8Bytes);
final CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer);
final char[] chars;
if (charBuffer.hasArray()) {
// there is no guarantee that the char buffers backing array is the right size
// so we need to make a copy
chars = Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit());
Arrays.fill(charBuffer.array(), (char) 0); // clear sensitive data
} else {
final int length = charBuffer.limit() - charBuffer.position();
chars = new char[length];
charBuffer.get(chars);
// if the buffer is not read only we can reset and fill with 0's
if (charBuffer.isReadOnly() == false) {
charBuffer.clear(); // reset
for (int i = 0; i < charBuffer.limit(); i++) {
charBuffer.put((char) 0);
}
}
}
return chars;
}
}

49
server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java
View File

@ -19,19 +19,8 @@
package org.elasticsearch.action.admin.cluster.node.reload;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
import java.io.InputStream;
import java.util.Objects;
/**
* Builder for the reload secure settings nodes request
@ -39,46 +28,8 @@ import java.util.Objects;
public class NodesReloadSecureSettingsRequestBuilder extends NodesOperationRequestBuilder<NodesReloadSecureSettingsRequest,
NodesReloadSecureSettingsResponse, NodesReloadSecureSettingsRequestBuilder> {
public static final String SECURE_SETTINGS_PASSWORD_FIELD_NAME = "secure_settings_password";
public NodesReloadSecureSettingsRequestBuilder(ElasticsearchClient client, NodesReloadSecureSettingsAction action) {
super(client, action, new NodesReloadSecureSettingsRequest());
}
public NodesReloadSecureSettingsRequestBuilder setSecureStorePassword(SecureString secureStorePassword) {
request.secureStorePassword(secureStorePassword);
return this;
}
public NodesReloadSecureSettingsRequestBuilder source(BytesReference source, XContentType xContentType) throws IOException {
Objects.requireNonNull(xContentType);
// EMPTY is ok here because we never call namedObject
try (InputStream stream = source.streamInput();
XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY,
LoggingDeprecationHandler.INSTANCE, stream)) {
XContentParser.Token token;
token = parser.nextToken();
if (token != XContentParser.Token.START_OBJECT) {
throw new ElasticsearchParseException("expected an object, but found token [{}]", token);
}
token = parser.nextToken();
if (token != XContentParser.Token.FIELD_NAME || false == SECURE_SETTINGS_PASSWORD_FIELD_NAME.equals(parser.currentName())) {
throw new ElasticsearchParseException("expected a field named [{}], but found [{}]", SECURE_SETTINGS_PASSWORD_FIELD_NAME,
token);
}
token = parser.nextToken();
if (token != XContentParser.Token.VALUE_STRING) {
throw new ElasticsearchParseException("expected field [{}] to be of type string, but found [{}] instead",
SECURE_SETTINGS_PASSWORD_FIELD_NAME, token);
}
final String password = parser.text();
setSecureStorePassword(new SecureString(password.toCharArray()));
token = parser.nextToken();
if (token != XContentParser.Token.END_OBJECT) {
throw new ElasticsearchParseException("expected end of object, but found token [{}]", token);
}
}
return this;
}
}

6
server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
View File

@ -31,7 +31,6 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.KeyStoreWrapper;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.plugins.PluginsService;
@ -82,16 +81,13 @@ public class TransportNodesReloadSecureSettingsAction extends TransportNodesActi
@Override
protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation(NodeRequest nodeReloadRequest) {
final NodesReloadSecureSettingsRequest request = nodeReloadRequest.request;
final SecureString secureSettingsPassword = request.secureSettingsPassword();
try (KeyStoreWrapper keystore = KeyStoreWrapper.load(environment.configFile())) {
// reread keystore from config file
if (keystore == null) {
return new NodesReloadSecureSettingsResponse.NodeResponse(clusterService.localNode(),
new IllegalStateException("Keystore is missing"));
}
// decrypt the keystore using the password from the request
keystore.decrypt(secureSettingsPassword.getChars());
keystore.decrypt(new char[0]);
// add the keystore to the original node settings object
final Settings settingsWithKeystore = Settings.builder()
.put(environment.settings(), false)

11
server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java
View File

@ -59,7 +59,6 @@ public final class RestReloadSecureSettingsAction extends BaseRestHandler {
.cluster()
.prepareReloadSecureSettings()
.setTimeout(request.param("timeout"))
.source(request.requiredContent(), request.getXContentType())
.setNodesIds(nodesIds);
final NodesReloadSecureSettingsRequest nodesRequest = nodesRequestBuilder.request();
return channel -> nodesRequestBuilder
@ -68,12 +67,12 @@ public final class RestReloadSecureSettingsAction extends BaseRestHandler {
public RestResponse buildResponse(NodesReloadSecureSettingsResponse response, XContentBuilder builder)
throws Exception {
builder.startObject();
RestActions.buildNodesHeader(builder, channel.request(), response);
builder.field("cluster_name", response.getClusterName().value());
response.toXContent(builder, channel.request());
{
RestActions.buildNodesHeader(builder, channel.request(), response);
builder.field("cluster_name", response.getClusterName().value());
response.toXContent(builder, channel.request());
}
builder.endObject();
// clear password for the original request
nodesRequest.secureSettingsPassword().close();
return new BytesRestResponse(RestStatus.OK, builder);
}
});

98
server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
View File

@ -20,11 +20,9 @@
package org.elasticsearch.action.admin;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsResponse;
import org.elasticsearch.common.settings.KeyStoreWrapper;
import org.elasticsearch.common.settings.SecureSettings;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.plugins.Plugin;
@ -44,11 +42,11 @@ import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicReference;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.containsString;
public class ReloadSecureSettingsIT extends ESIntegTestCase {
@ -62,7 +60,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
Files.deleteIfExists(KeyStoreWrapper.keystorePath(environment.configFile()));
final int initialReloadCount = mockReloadablePlugin.getReloadCount();
final CountDownLatch latch = new CountDownLatch(1);
client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute(
client().admin().cluster().prepareReloadSecureSettings().execute(
new ActionListener<NodesReloadSecureSettingsResponse>() {
@Override
public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
@ -96,44 +94,6 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount));
}
public void testNullKeystorePassword() throws Exception {
final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class);
final MockReloadablePlugin mockReloadablePlugin = pluginsService.filterPlugins(MockReloadablePlugin.class)
.stream().findFirst().get();
final AtomicReference<AssertionError> reloadSettingsError = new AtomicReference<>();
final int initialReloadCount = mockReloadablePlugin.getReloadCount();
final CountDownLatch latch = new CountDownLatch(1);
client().admin().cluster().prepareReloadSecureSettings().execute(
new ActionListener<NodesReloadSecureSettingsResponse>() {
@Override
public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
try {
reloadSettingsError.set(new AssertionError("Null keystore password should fail"));
} finally {
latch.countDown();
}
}
@Override
public void onFailure(Exception e) {
try {
assertThat(e, instanceOf(ActionRequestValidationException.class));
assertThat(e.getMessage(), containsString("secure settings password cannot be null"));
} catch (final AssertionError ae) {
reloadSettingsError.set(ae);
} finally {
latch.countDown();
}
}
});
latch.await();
if (reloadSettingsError.get() != null) {
throw reloadSettingsError.get();
}
// in the null password case no reload should be triggered
assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount));
}
public void testInvalidKeystoreFile() throws Exception {
final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class);
final MockReloadablePlugin mockReloadablePlugin = pluginsService.filterPlugins(MockReloadablePlugin.class)
@ -149,7 +109,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
Files.copy(keystore, KeyStoreWrapper.keystorePath(environment.configFile()), StandardCopyOption.REPLACE_EXISTING);
}
final CountDownLatch latch = new CountDownLatch(1);
client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute(
client().admin().cluster().prepareReloadSecureSettings().execute(
new ActionListener<NodesReloadSecureSettingsResponse>() {
@Override
public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
@ -181,52 +141,6 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount));
}
public void testWrongKeystorePassword() throws Exception {
final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class);
final MockReloadablePlugin mockReloadablePlugin = pluginsService.filterPlugins(MockReloadablePlugin.class)
.stream().findFirst().get();
final Environment environment = internalCluster().getInstance(Environment.class);
final AtomicReference<AssertionError> reloadSettingsError = new AtomicReference<>();
final int initialReloadCount = mockReloadablePlugin.getReloadCount();
// "some" keystore should be present in this case
writeEmptyKeystore(environment, new char[0]);
final CountDownLatch latch = new CountDownLatch(1);
client().admin()
.cluster()
.prepareReloadSecureSettings()
.setSecureStorePassword(new SecureString(new char[] { 'W', 'r', 'o', 'n', 'g' }))
.execute(new ActionListener<NodesReloadSecureSettingsResponse>() {
@Override
public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
try {
assertThat(nodesReloadResponse, notNullValue());
final Map<String, NodesReloadSecureSettingsResponse.NodeResponse> nodesMap = nodesReloadResponse.getNodesMap();
assertThat(nodesMap.size(), equalTo(cluster().size()));
for (final NodesReloadSecureSettingsResponse.NodeResponse nodeResponse : nodesReloadResponse.getNodes()) {
assertThat(nodeResponse.reloadException(), notNullValue());
assertThat(nodeResponse.reloadException(), instanceOf(SecurityException.class));
}
} catch (final AssertionError e) {
reloadSettingsError.set(e);
} finally {
latch.countDown();
}
}
@Override
public void onFailure(Exception e) {
reloadSettingsError.set(new AssertionError("Nodes request failed", e));
latch.countDown();
}
});
latch.await();
if (reloadSettingsError.get() != null) {
throw reloadSettingsError.get();
}
// in the wrong password case no reload should be triggered
assertThat(mockReloadablePlugin.getReloadCount(), equalTo(initialReloadCount));
}
public void testMisbehavingPlugin() throws Exception {
final Environment environment = internalCluster().getInstance(Environment.class);
final PluginsService pluginsService = internalCluster().getInstance(PluginsService.class);
@ -247,7 +161,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
.get(Settings.builder().put(environment.settings()).setSecureSettings(secureSettings).build())
.toString();
final CountDownLatch latch = new CountDownLatch(1);
client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute(
client().admin().cluster().prepareReloadSecureSettings().execute(
new ActionListener<NodesReloadSecureSettingsResponse>() {
@Override
public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {
@ -314,7 +228,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
private void successfulReloadCall() throws InterruptedException {
final AtomicReference<AssertionError> reloadSettingsError = new AtomicReference<>();
final CountDownLatch latch = new CountDownLatch(1);
client().admin().cluster().prepareReloadSecureSettings().setSecureStorePassword(new SecureString(new char[0])).execute(
client().admin().cluster().prepareReloadSecureSettings().execute(
new ActionListener<NodesReloadSecureSettingsResponse>() {
@Override
public void onResponse(NodesReloadSecureSettingsResponse nodesReloadResponse) {

2
server/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.cluster.ack;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
@ -50,6 +51,7 @@ import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
@ClusterScope(minNumDataNodes = 2)
@AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/32767")
public class AckIT extends ESIntegTestCase {
@Override

3
server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.search.scroll;
import com.carrotsearch.hppc.IntHashSet;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
@ -198,6 +199,8 @@ public class DuelScrollIT extends ESIntegTestCase {
}
// no replicas, as they might be ordered differently
settings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
// we need to control refreshes as they might take different merges into account
settings.put("index.refresh_interval", -1);
assertAcked(prepareCreate("test").setSettings(settings.build()).get());
final int numDocs = randomIntBetween(10, 200);

2
x-pack/docs/en/security/configuring-es.asciidoc
View File

@ -9,7 +9,7 @@
password-protect your data as well as implement more advanced security measures
such as encrypting communications, role-based access control, IP filtering, and
auditing. For more information, see
{xpack-ref}/xpack-security.html[Securing the Elastic Stack].
{xpack-ref}/elasticsearch-security.html[Securing the Elastic Stack].
To use {security} in {es}:

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java
View File

@ -15,7 +15,7 @@ import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import org.elasticsearch.common.CharArrays;
import java.io.IOException;
import java.util.Arrays;

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequest.java
View File

@ -12,7 +12,7 @@ import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import org.elasticsearch.common.CharArrays;
import java.io.IOException;

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequest.java
View File

@ -8,12 +8,12 @@ package org.elasticsearch.xpack.core.security.action.user;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.common.CharArrays;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import java.io.IOException;
import java.util.Map;

9
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/BCrypt.java
View File

@ -14,6 +14,7 @@ package org.elasticsearch.xpack.core.security.authc.support;
// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
import org.elasticsearch.common.CharArrays;
import org.elasticsearch.common.settings.SecureString;
import java.security.SecureRandom;
@ -54,7 +55,7 @@ import java.security.SecureRandom;
* String stronger_salt = BCrypt.gensalt(12)<br>
* </code>
* <p>
* The amount of work increases exponentially (2**log_rounds), so
* The amount of work increases exponentially (2**log_rounds), so
* each increment is twice as much work. The default log_rounds is
* 10, and the valid range is 4 to 30.
*
@ -689,7 +690,11 @@ public class BCrypt {
// the next lines are the SecureString replacement for the above commented-out section
if (minor >= 'a') {
try (SecureString secureString = new SecureString(CharArrays.concat(password.getChars(), "\000".toCharArray()))) {
final char[] suffix = "\000".toCharArray();
final char[] result = new char[password.length() + suffix.length];
System.arraycopy(password.getChars(), 0, result, 0, password.length());
System.arraycopy(suffix, 0, result, password.length(), suffix.length);
try (SecureString secureString = new SecureString(result)) {
passwordb = CharArrays.toUtf8Bytes(secureString.getChars());
}
} else {

101
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CharArrays.java
View File

@ -1,101 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.security.authc.support;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
/**
* Helper class similar to Arrays to handle conversions for Char arrays
*/
public class CharArrays {
public static char[] utf8BytesToChars(byte[] utf8Bytes) {
ByteBuffer byteBuffer = ByteBuffer.wrap(utf8Bytes);
CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer);
char[] chars = Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit());
byteBuffer.clear();
charBuffer.clear();
return chars;
}
/**
* Like String.indexOf for for an array of chars
*/
static int indexOf(char[] array, char ch) {
for (int i = 0; (i < array.length); i++) {
if (array[i] == ch) {
return i;
}
}
return -1;
}
/**
* Converts the provided char[] to a UTF-8 byte[]. The provided char[] is not modified by this
* method, so the caller needs to take care of clearing the value if it is sensitive.
*/
public static byte[] toUtf8Bytes(char[] chars) {
CharBuffer charBuffer = CharBuffer.wrap(chars);
ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(charBuffer);
byte[] bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
Arrays.fill(byteBuffer.array(), (byte) 0); // clear sensitive data
return bytes;
}
public static boolean charsBeginsWith(String prefix, char[] chars) {
if (chars == null || prefix == null) {
return false;
}
if (prefix.length() > chars.length) {
return false;
}
for (int i = 0; i < prefix.length(); i++) {
if (chars[i] != prefix.charAt(i)) {
return false;
}
}
return true;
}
public static boolean constantTimeEquals(char[] a, char[] b) {
if (a.length != b.length) {
return false;
}
int equals = 0;
for (int i = 0; i < a.length; i++) {
equals |= a[i] ^ b[i];
}
return equals == 0;
}
public static boolean constantTimeEquals(String a, String b) {
if (a.length() != b.length()) {
return false;
}
int equals = 0;
for (int i = 0; i < a.length(); i++) {
equals |= a.charAt(i) ^ b.charAt(i);
}
return equals == 0;
}
public static char[] concat(char[] a, char[] b) {
final char[] result = new char[a.length + b.length];
System.arraycopy(a, 0, result, 0, a.length);
System.arraycopy(b, 0, result, a.length, b.length);
return result;
}
}

1
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.core.security.authc.support;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.CharArrays;
import org.elasticsearch.common.hash.MessageDigests;
import org.elasticsearch.common.settings.SecureString;

14
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.core.security.authc.support;
import org.elasticsearch.common.CharArrays;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.util.concurrent.ThreadContext;
@ -107,7 +108,7 @@ public class UsernamePasswordToken implements AuthenticationToken {
throw authenticationError("invalid basic authentication header encoding", e);
}
int i = CharArrays.indexOf(userpasswd, ':');
int i = indexOfColon(userpasswd);
if (i < 0) {
throw authenticationError("invalid basic authentication header value");
}
@ -121,4 +122,15 @@ public class UsernamePasswordToken implements AuthenticationToken {
context.putHeader(BASIC_AUTH_HEADER, basicAuthHeaderValue(token.username, token.password));
}
/**
* Like String.indexOf for for an array of chars
*/
private static int indexOfColon(char[] array) {
for (int i = 0; (i < array.length); i++) {
if (array[i] == ':') {
return i;
}
}
return -1;
}
}

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java
View File

@ -7,7 +7,7 @@
package org.elasticsearch.xpack.core.ssl;
import org.elasticsearch.common.hash.MessageDigests;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import org.elasticsearch.common.CharArrays;
import java.io.BufferedReader;
import java.io.IOException;

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java
View File

@ -13,7 +13,7 @@ import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.core.watcher.WatcherField;
import org.elasticsearch.xpack.core.security.SecurityField;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import org.elasticsearch.common.CharArrays;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;

2
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
View File

@ -100,7 +100,7 @@ public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedCon
if (aggHistogramInterval == null) {
builder.setFrequency(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)));
} else {
builder.setFrequency(TimeValue.timeValueMillis(randomIntBetween(1, 5) * aggHistogramInterval));
builder.setFrequency(TimeValue.timeValueSeconds(randomIntBetween(1, 5) * aggHistogramInterval));
}
}
if (randomBoolean()) {

3
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
View File

@ -80,9 +80,8 @@ public class DocumentSubsetReaderTests extends ESTestCase {
bitsetFilterCache.close();
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32457")
public void testSearch() throws Exception {
IndexWriter iw = new IndexWriter(directory, newIndexWriterConfig());
IndexWriter iw = new IndexWriter(directory, newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random())));
Document document = new Document();
document.add(new StringField("field", "value1", Field.Store.NO));

2
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java
View File

@ -32,7 +32,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings;
import org.elasticsearch.xpack.core.security.authc.ldap.ActiveDirectorySessionFactorySettings;
import org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings;
import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import org.elasticsearch.common.CharArrays;
import org.elasticsearch.xpack.core.ssl.SSLService;
import org.elasticsearch.xpack.security.authc.ldap.support.LdapMetaDataResolver;
import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;

2
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java
View File

@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmConfig;
import org.elasticsearch.xpack.core.security.authc.RealmSettings;
import org.elasticsearch.xpack.core.security.authc.ldap.LdapSessionFactorySettings;
import org.elasticsearch.xpack.core.security.authc.ldap.SearchGroupsResolverSettings;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import org.elasticsearch.common.CharArrays;
import org.elasticsearch.xpack.core.ssl.SSLService;
import org.elasticsearch.xpack.security.authc.ldap.support.LdapMetaDataResolver;
import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;

2
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java
View File

@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings;
import org.elasticsearch.xpack.core.security.authc.ldap.LdapUserSearchSessionFactorySettings;
import org.elasticsearch.xpack.core.security.authc.ldap.SearchGroupsResolverSettings;
import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import org.elasticsearch.common.CharArrays;
import org.elasticsearch.xpack.core.ssl.SSLService;
import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;
import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession.GroupsResolver;

2
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java
View File

@ -25,7 +25,7 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.security.authc.RealmConfig;
import org.elasticsearch.xpack.core.security.authc.RealmSettings;
import org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import org.elasticsearch.common.CharArrays;
import org.elasticsearch.xpack.core.ssl.SSLService;
import org.elasticsearch.xpack.security.authc.ldap.support.LdapMetaDataResolver;
import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;

2
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java
View File

@ -13,7 +13,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.test.NativeRealmIntegTestCase;
import org.elasticsearch.test.SecuritySettingsSource;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import org.elasticsearch.common.CharArrays;
import org.elasticsearch.xpack.core.security.client.SecurityClient;
import org.elasticsearch.xpack.security.support.SecurityIndexManager;
import org.junit.BeforeClass;

75
x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequest.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import java.util.Objects;
public class DeleteJobRequest extends ActionRequest {
private String jobId;
private boolean force;
public DeleteJobRequest(String jobId) {
this.jobId = Objects.requireNonNull(jobId, "[job_id] must not be null");
}
public String getJobId() {
return jobId;
}
public void setJobId(String jobId) {
this.jobId = Objects.requireNonNull(jobId, "[job_id] must not be null");
}
public boolean isForce() {
return force;
}
public void setForce(boolean force) {
this.force = force;
}
@Override
public ActionRequestValidationException validate() {
return null;
}
@Override
public int hashCode() {
return Objects.hash(jobId, force);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || obj.getClass() != getClass()) {
return false;
}
DeleteJobRequest other = (DeleteJobRequest) obj;
return Objects.equals(jobId, other.jobId) && Objects.equals(force, other.force);
}
}

60
x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponse.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
import java.util.Objects;
public class DeleteJobResponse extends AcknowledgedResponse {
public DeleteJobResponse(boolean acknowledged) {
super(acknowledged);
}
public DeleteJobResponse() {
}
public static DeleteJobResponse fromXContent(XContentParser parser) throws IOException {
AcknowledgedResponse response = AcknowledgedResponse.fromXContent(parser);
return new DeleteJobResponse(response.isAcknowledged());
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (other == null || getClass() != other.getClass()) {
return false;
}
DeleteJobResponse that = (DeleteJobResponse) other;
return isAcknowledged() == that.isAcknowledged();
}
@Override
public int hashCode() {
return Objects.hash(isAcknowledged());
}
}

45
x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobRequestTests.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml;
import org.elasticsearch.protocol.xpack.ml.job.config.JobTests;
import org.elasticsearch.test.ESTestCase;
public class DeleteJobRequestTests extends ESTestCase {
private DeleteJobRequest createTestInstance() {
return new DeleteJobRequest(JobTests.randomValidJobId());
}
public void test_WithNullJobId() {
NullPointerException ex = expectThrows(NullPointerException.class, () -> new DeleteJobRequest(null));
assertEquals("[job_id] must not be null", ex.getMessage());
ex = expectThrows(NullPointerException.class, () -> createTestInstance().setJobId(null));
assertEquals("[job_id] must not be null", ex.getMessage());
}
public void test_WithForce() {
DeleteJobRequest deleteJobRequest = createTestInstance();
assertFalse(deleteJobRequest.isForce());
deleteJobRequest.setForce(true);
assertTrue(deleteJobRequest.isForce());
}
}

42
x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/DeleteJobResponseTests.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
public class DeleteJobResponseTests extends AbstractXContentTestCase<DeleteJobResponse> {
@Override
protected DeleteJobResponse createTestInstance() {
return new DeleteJobResponse();
}
@Override
protected DeleteJobResponse doParseInstance(XContentParser parser) throws IOException {
return DeleteJobResponse.fromXContent(parser);
}
@Override
protected boolean supportsUnknownFields() {
return false;
}
}

1
x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
View File

@ -325,6 +325,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
}
}
@AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/32773")
public void testRollupIDSchemeAfterRestart() throws Exception {
assumeTrue("Rollup can be tested with 6.3.0 and onwards", oldClusterVersion.onOrAfter(Version.V_6_3_0));
assumeTrue("Rollup ID scheme changed in 6.4", oldClusterVersion.before(Version.V_6_4_0));

2
x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java
View File

@ -12,7 +12,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
import org.elasticsearch.xpack.core.security.authc.Realm;
import org.elasticsearch.xpack.core.security.authc.RealmConfig;
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
import org.elasticsearch.common.CharArrays;
import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.protocol.xpack.security.User;