honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into index-lifecycle

This commit is contained in:
Colin Goodheart-Smithe 2018-08-09 12:47:26 +01:00
parent 97ba7b83b4 823d40e19b
commit 0fe21136db
No known key found for this signature in database
GPG Key ID: F975E7BDD739B3C7
57 changed files with 2937 additions and 1110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java
View File

@ -26,7 +26,7 @@ import org.elasticsearch.action.ingest.GetPipelineResponse;
import org.elasticsearch.action.ingest.PutPipelineRequest;
import org.elasticsearch.action.ingest.SimulatePipelineRequest;
import org.elasticsearch.action.ingest.SimulatePipelineResponse;
import org.elasticsearch.action.ingest.WritePipelineResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import java.io.IOException;
@ -54,9 +54,9 @@ public final class IngestClient {
* @return the response
* @throws IOException in case there is a problem sending the request or parsing back the response
*/
public WritePipelineResponse putPipeline(PutPipelineRequest request, RequestOptions options) throws IOException {
public AcknowledgedResponse putPipeline(PutPipelineRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::putPipeline, options,
WritePipelineResponse::fromXContent, emptySet());
AcknowledgedResponse::fromXContent, emptySet());
}
/**
@ -67,9 +67,9 @@ public final class IngestClient {
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener the listener to be notified upon request completion
*/
public void putPipelineAsync(PutPipelineRequest request, RequestOptions options, ActionListener<WritePipelineResponse> listener) {
public void putPipelineAsync(PutPipelineRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::putPipeline, options,
WritePipelineResponse::fromXContent, listener, emptySet());
AcknowledgedResponse::fromXContent, listener, emptySet());
}
/**
@ -109,9 +109,9 @@ public final class IngestClient {
* @return the response
* @throws IOException in case there is a problem sending the request or parsing back the response
*/
public WritePipelineResponse deletePipeline(DeletePipelineRequest request, RequestOptions options) throws IOException {
public AcknowledgedResponse deletePipeline(DeletePipelineRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::deletePipeline, options,
WritePipelineResponse::fromXContent, emptySet());
AcknowledgedResponse::fromXContent, emptySet());
}
/**
@ -123,9 +123,9 @@ public final class IngestClient {
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener the listener to be notified upon request completion
*/
public void deletePipelineAsync(DeletePipelineRequest request, RequestOptions options, ActionListener<WritePipelineResponse> listener) {
public void deletePipelineAsync(DeletePipelineRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::deletePipeline, options,
WritePipelineResponse::fromXContent, listener, emptySet());
AcknowledgedResponse::fromXContent, listener, emptySet());
}
/**

6
client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java
View File

@ -28,7 +28,7 @@ import org.elasticsearch.action.ingest.SimulateDocumentResult;
import org.elasticsearch.action.ingest.SimulateDocumentVerboseResult;
import org.elasticsearch.action.ingest.SimulatePipelineRequest;
import org.elasticsearch.action.ingest.SimulatePipelineResponse;
import org.elasticsearch.action.ingest.WritePipelineResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
@ -50,7 +50,7 @@ public class IngestClientIT extends ESRestHighLevelClientTestCase {
BytesReference.bytes(pipelineBuilder),
pipelineBuilder.contentType());
WritePipelineResponse putPipelineResponse =
AcknowledgedResponse putPipelineResponse =
execute(request, highLevelClient().ingest()::putPipeline, highLevelClient().ingest()::putPipelineAsync);
assertTrue(putPipelineResponse.isAcknowledged());
}
@ -86,7 +86,7 @@ public class IngestClientIT extends ESRestHighLevelClientTestCase {
DeletePipelineRequest request = new DeletePipelineRequest(id);
WritePipelineResponse response =
AcknowledgedResponse response =
execute(request, highLevelClient().ingest()::deletePipeline, highLevelClient().ingest()::deletePipelineAsync);
assertTrue(response.isAcknowledged());
}

18
client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java
View File

@ -31,7 +31,7 @@ import org.elasticsearch.action.ingest.SimulateDocumentVerboseResult;
import org.elasticsearch.action.ingest.SimulatePipelineRequest;
import org.elasticsearch.action.ingest.SimulatePipelineResponse;
import org.elasticsearch.action.ingest.SimulateProcessorResult;
import org.elasticsearch.action.ingest.WritePipelineResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ESRestHighLevelClientTestCase;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
@ -93,7 +93,7 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
// end::put-pipeline-request-masterTimeout
// tag::put-pipeline-execute
WritePipelineResponse response = client.ingest().putPipeline(request, RequestOptions.DEFAULT); // <1>
AcknowledgedResponse response = client.ingest().putPipeline(request, RequestOptions.DEFAULT); // <1>
// end::put-pipeline-execute
// tag::put-pipeline-response
@ -117,10 +117,10 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
);
// tag::put-pipeline-execute-listener
ActionListener<WritePipelineResponse> listener =
new ActionListener<WritePipelineResponse>() {
ActionListener<AcknowledgedResponse> listener =
new ActionListener<AcknowledgedResponse>() {
@Override
public void onResponse(WritePipelineResponse response) {
public void onResponse(AcknowledgedResponse response) {
// <1>
}
@ -236,7 +236,7 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
// end::delete-pipeline-request-masterTimeout
// tag::delete-pipeline-execute
WritePipelineResponse response = client.ingest().deletePipeline(request, RequestOptions.DEFAULT); // <1>
AcknowledgedResponse response = client.ingest().deletePipeline(request, RequestOptions.DEFAULT); // <1>
// end::delete-pipeline-execute
// tag::delete-pipeline-response
@ -257,10 +257,10 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
DeletePipelineRequest request = new DeletePipelineRequest("my-pipeline-id");
// tag::delete-pipeline-execute-listener
ActionListener<WritePipelineResponse> listener =
new ActionListener<WritePipelineResponse>() {
ActionListener<AcknowledgedResponse> listener =
new ActionListener<AcknowledgedResponse>() {
@Override
public void onResponse(WritePipelineResponse response) {
public void onResponse(AcknowledgedResponse response) {
// <1>
}

138
modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.painless;
import org.elasticsearch.painless.Locals.LocalMethod;
import org.elasticsearch.painless.lookup.PainlessClass;
import org.elasticsearch.painless.lookup.PainlessLookup;
import org.elasticsearch.painless.lookup.PainlessLookupUtility;
import org.elasticsearch.painless.lookup.PainlessMethod;
@ -38,6 +37,8 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
/**
* Support for dynamic type (def).
* <p>
@ -167,52 +168,6 @@ public final class Def {
}
}
/**
* Looks up method entry for a dynamic method call.
* <p>
* A dynamic method call for variable {@code x} of type {@code def} looks like:
* {@code x.method(args...)}
* <p>
* This method traverses {@code recieverClass}'s class hierarchy (including interfaces)
* until it finds a matching whitelisted method. If one is not found, it throws an exception.
* Otherwise it returns the matching method.
* <p>
* @params painlessLookup the whitelist
* @param receiverClass Class of the object to invoke the method on.
* @param name Name of the method.
* @param arity arity of method
* @return matching method to invoke. never returns null.
* @throws IllegalArgumentException if no matching whitelisted method was found.
*/
static PainlessMethod lookupMethodInternal(PainlessLookup painlessLookup, Class<?> receiverClass, String name, int arity) {
String key = PainlessLookupUtility.buildPainlessMethodKey(name, arity);
// check whitelist for matching method
for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
PainlessClass struct = painlessLookup.lookupPainlessClass(clazz);
if (struct != null) {
PainlessMethod method = struct.methods.get(key);
if (method != null) {
return method;
}
}
for (Class<?> iface : clazz.getInterfaces()) {
struct = painlessLookup.lookupPainlessClass(iface);
if (struct != null) {
PainlessMethod method = struct.methods.get(key);
if (method != null) {
return method;
}
}
}
}
throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] with [" + arity + "] arguments " +
"for class [" + receiverClass.getCanonicalName() + "].");
}
/**
* Looks up handle for a dynamic method call, with lambda replacement
* <p>
@ -241,7 +196,14 @@ public final class Def {
int numArguments = callSiteType.parameterCount();
// simple case: no lambdas
if (recipeString.isEmpty()) {
return lookupMethodInternal(painlessLookup, receiverClass, name, numArguments - 1).methodHandle;
PainlessMethod painlessMethod = painlessLookup.lookupRuntimePainlessMethod(receiverClass, name, numArguments - 1);
if (painlessMethod == null) {
throw new IllegalArgumentException("dynamic method " +
"[" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + (numArguments - 1) + "] not found");
}
return painlessMethod.methodHandle;
}
// convert recipe string to a bitset for convenience (the code below should be refactored...)
@ -264,7 +226,13 @@ public final class Def {
// lookup the method with the proper arity, then we know everything (e.g. interface types of parameters).
// based on these we can finally link any remaining lambdas that were deferred.
PainlessMethod method = lookupMethodInternal(painlessLookup, receiverClass, name, arity);
PainlessMethod method = painlessLookup.lookupRuntimePainlessMethod(receiverClass, name, arity);
if (method == null) {
throw new IllegalArgumentException(
"dynamic method [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + arity + "] not found");
}
MethodHandle handle = method.methodHandle;
int replaced = 0;
@ -332,15 +300,23 @@ public final class Def {
static MethodHandle lookupReference(PainlessLookup painlessLookup, Map<String, LocalMethod> localMethods,
MethodHandles.Lookup methodHandlesLookup, String interfaceClass, Class<?> receiverClass, String name) throws Throwable {
Class<?> interfaceType = painlessLookup.canonicalTypeNameToType(interfaceClass);
if (interfaceType == null) {
throw new IllegalArgumentException("type [" + interfaceClass + "] not found");
}
PainlessMethod interfaceMethod = painlessLookup.lookupFunctionalInterfacePainlessMethod(interfaceType);
if (interfaceMethod == null) {
throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface");
}
int arity = interfaceMethod.typeParameters.size();
PainlessMethod implMethod = lookupMethodInternal(painlessLookup, receiverClass, name, arity);
PainlessMethod implMethod = painlessLookup.lookupRuntimePainlessMethod(receiverClass, name, arity);
if (implMethod == null) {
throw new IllegalArgumentException(
"dynamic method [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + arity + "] not found");
}
return lookupReferenceInternal(painlessLookup, localMethods, methodHandlesLookup,
interfaceType, PainlessLookupUtility.typeToCanonicalTypeName(implMethod.targetClass),
implMethod.javaMethod.getName(), 1);
interfaceType, PainlessLookupUtility.typeToCanonicalTypeName(implMethod.targetClass),
implMethod.javaMethod.getName(), 1);
}
/** Returns a method handle to an implementation of clazz, given method reference signature. */
@ -389,27 +365,12 @@ public final class Def {
*/
static MethodHandle lookupGetter(PainlessLookup painlessLookup, Class<?> receiverClass, String name) {
// first try whitelist
for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
PainlessClass struct = painlessLookup.lookupPainlessClass(clazz);
MethodHandle getter = painlessLookup.lookupRuntimeGetterMethodHandle(receiverClass, name);
if (struct != null) {
MethodHandle handle = struct.getterMethodHandles.get(name);
if (handle != null) {
return handle;
}
}
for (final Class<?> iface : clazz.getInterfaces()) {
struct = painlessLookup.lookupPainlessClass(iface);
if (struct != null) {
MethodHandle handle = struct.getterMethodHandles.get(name);
if (handle != null) {
return handle;
}
}
}
if (getter != null) {
return getter;
}
// special case: arrays, maps, and lists
if (receiverClass.isArray() && "length".equals(name)) {
// arrays expose .length as a read-only getter
@ -426,12 +387,12 @@ public final class Def {
int index = Integer.parseInt(name);
return MethodHandles.insertArguments(LIST_GET, 1, index);
} catch (NumberFormatException exception) {
throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
throw new IllegalArgumentException("Illegal list shortcut value [" + name + "].");
}
}
throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
"for class [" + receiverClass.getCanonicalName() + "].");
throw new IllegalArgumentException(
"dynamic getter [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "] not found");
}
/**
@ -460,27 +421,12 @@ public final class Def {
*/
static MethodHandle lookupSetter(PainlessLookup painlessLookup, Class<?> receiverClass, String name) {
// first try whitelist
for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
PainlessClass struct = painlessLookup.lookupPainlessClass(clazz);
MethodHandle setter = painlessLookup.lookupRuntimeSetterMethodHandle(receiverClass, name);
if (struct != null) {
MethodHandle handle = struct.setterMethodHandles.get(name);
if (handle != null) {
return handle;
}
}
for (final Class<?> iface : clazz.getInterfaces()) {
struct = painlessLookup.lookupPainlessClass(iface);
if (struct != null) {
MethodHandle handle = struct.setterMethodHandles.get(name);
if (handle != null) {
return handle;
}
}
}
if (setter != null) {
return setter;
}
// special case: maps, and lists
if (Map.class.isAssignableFrom(receiverClass)) {
// maps allow access like mymap.key
@ -494,12 +440,12 @@ public final class Def {
int index = Integer.parseInt(name);
return MethodHandles.insertArguments(LIST_SET, 1, index);
} catch (final NumberFormatException exception) {
throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
throw new IllegalArgumentException("Illegal list shortcut value [" + name + "].");
}
}
throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
"for class [" + receiverClass.getCanonicalName() + "].");
throw new IllegalArgumentException(
"dynamic getter [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "] not found");
}
/**

37
modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
View File

@ -67,11 +67,11 @@ public class FunctionRef {
PainlessMethod interfaceMethod;
try {
try {
interfaceMethod = painlessLookup.lookupFunctionalInterfacePainlessMethod(targetClass);
} catch (IllegalArgumentException iae) {
interfaceMethod = painlessLookup.lookupFunctionalInterfacePainlessMethod(targetClass);
if (interfaceMethod == null) {
throw new IllegalArgumentException("cannot convert function reference [" + typeName + "::" + methodName + "] " +
"to a non-functional interface [" + targetClassName + "]", iae);
"to a non-functional interface [" + targetClassName + "]");
}
String interfaceMethodName = interfaceMethod.javaMethod.getName();
@ -116,14 +116,12 @@ public class FunctionRef {
throw new IllegalStateException("internal error");
}
PainlessConstructor painlessConstructor;
PainlessConstructor painlessConstructor = painlessLookup.lookupPainlessConstructor(typeName, interfaceTypeParametersSize);
try {
painlessConstructor = painlessLookup.lookupPainlessConstructor(typeName, interfaceTypeParametersSize);
} catch (IllegalArgumentException iae) {
if (painlessConstructor == null) {
throw new IllegalArgumentException("function reference [" + typeName + "::new/" + interfaceTypeParametersSize + "] " +
"matching [" + targetClassName + ", " + interfaceMethodName + "/" + interfaceTypeParametersSize + "] " +
"not found", iae);
"not found");
}
delegateClassName = painlessConstructor.javaConstructor.getDeclaringClass().getName();
@ -140,24 +138,21 @@ public class FunctionRef {
}
boolean captured = numberOfCaptures == 1;
PainlessMethod painlessMethod;
PainlessMethod painlessMethod =
painlessLookup.lookupPainlessMethod(typeName, true, methodName, interfaceTypeParametersSize);
try {
painlessMethod = painlessLookup.lookupPainlessMethod(typeName, true, methodName, interfaceTypeParametersSize);
if (painlessMethod == null) {
painlessMethod = painlessLookup.lookupPainlessMethod(typeName, false, methodName,
captured ? interfaceTypeParametersSize : interfaceTypeParametersSize - 1);
if (captured) {
throw new IllegalStateException("internal error");
}
} catch (IllegalArgumentException staticIAE) {
try {
painlessMethod = painlessLookup.lookupPainlessMethod(typeName, false, methodName,
captured ? interfaceTypeParametersSize : interfaceTypeParametersSize - 1);
} catch (IllegalArgumentException iae) {
if (painlessMethod == null) {
throw new IllegalArgumentException(
"function reference " + "[" + typeName + "::" + methodName + "/" + interfaceTypeParametersSize + "] " +
"matching [" + targetClassName + ", " + interfaceMethodName + "/" + interfaceTypeParametersSize + "] " +
"not found", iae);
"not found");
}
} else if (captured) {
throw new IllegalStateException("internal error");
}
delegateClassName = painlessMethod.javaMethod.getDeclaringClass().getName();

114
modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
View File

@ -19,15 +19,17 @@
package org.elasticsearch.painless.lookup;
import java.lang.invoke.MethodHandle;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessConstructorKey;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessFieldKey;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToBoxedType;
public final class PainlessLookup {
@ -67,6 +69,10 @@ public final class PainlessLookup {
Class<?> targetClass = canonicalTypeNameToType(targetClassName);
if (targetClass == null) {
return null;
}
return lookupPainlessConstructor(targetClass, constructorArity);
}
@ -77,15 +83,13 @@ public final class PainlessLookup {
String painlessConstructorKey = buildPainlessConstructorKey(constructorArity);
if (targetPainlessClass == null) {
throw new IllegalArgumentException("target class [" + typeToCanonicalTypeName(targetClass) + "] " +
"not found for constructor [" + painlessConstructorKey + "]");
return null;
}
PainlessConstructor painlessConstructor = targetPainlessClass.constructors.get(painlessConstructorKey);
if (painlessConstructor == null) {
throw new IllegalArgumentException(
"constructor [" + typeToCanonicalTypeName(targetClass) + ", " + painlessConstructorKey + "] not found");
return null;
}
return painlessConstructor;
@ -96,6 +100,10 @@ public final class PainlessLookup {
Class<?> targetClass = canonicalTypeNameToType(targetClassName);
if (targetClass == null) {
return null;
}
return lookupPainlessMethod(targetClass, isStatic, methodName, methodArity);
}
@ -104,27 +112,19 @@ public final class PainlessLookup {
Objects.requireNonNull(methodName);
if (targetClass.isPrimitive()) {
targetClass = PainlessLookupUtility.typeToBoxedType(targetClass);
targetClass = typeToBoxedType(targetClass);
}
PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetClass);
String painlessMethodKey = buildPainlessMethodKey(methodName, methodArity);
if (targetPainlessClass == null) {
throw new IllegalArgumentException(
"target class [" + typeToCanonicalTypeName(targetClass) + "] not found for method [" + painlessMethodKey + "]");
return null;
}
PainlessMethod painlessMethod = isStatic ?
return isStatic ?
targetPainlessClass.staticMethods.get(painlessMethodKey) :
targetPainlessClass.methods.get(painlessMethodKey);
if (painlessMethod == null) {
throw new IllegalArgumentException(
"method [" + typeToCanonicalTypeName(targetClass) + ", " + painlessMethodKey + "] not found");
}
return painlessMethod;
}
public PainlessField lookupPainlessField(String targetClassName, boolean isStatic, String fieldName) {
@ -132,6 +132,10 @@ public final class PainlessLookup {
Class<?> targetClass = canonicalTypeNameToType(targetClassName);
if (targetClass == null) {
return null;
}
return lookupPainlessField(targetClass, isStatic, fieldName);
}
@ -143,8 +147,7 @@ public final class PainlessLookup {
String painlessFieldKey = buildPainlessFieldKey(fieldName);
if (targetPainlessClass == null) {
throw new IllegalArgumentException(
"target class [" + typeToCanonicalTypeName(targetClass) + "] not found for field [" + painlessFieldKey + "]");
return null;
}
PainlessField painlessField = isStatic ?
@ -152,8 +155,7 @@ public final class PainlessLookup {
targetPainlessClass.fields.get(painlessFieldKey);
if (painlessField == null) {
throw new IllegalArgumentException(
"field [" + typeToCanonicalTypeName(targetClass) + ", " + painlessFieldKey + "] not found");
return null;
}
return painlessField;
@ -163,15 +165,77 @@ public final class PainlessLookup {
PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetClass);
if (targetPainlessClass == null) {
throw new IllegalArgumentException("target class [" + typeToCanonicalTypeName(targetClass) + "] not found");
return null;
}
PainlessMethod functionalInterfacePainlessMethod = targetPainlessClass.functionalInterfaceMethod;
return targetPainlessClass.functionalInterfaceMethod;
}
if (functionalInterfacePainlessMethod == null) {
throw new IllegalArgumentException("target class [" + typeToCanonicalTypeName(targetClass) + "] is not a functional interface");
public PainlessMethod lookupRuntimePainlessMethod(Class<?> originalTargetClass, String methodName, int methodArity) {
Objects.requireNonNull(originalTargetClass);
Objects.requireNonNull(methodName);
String painlessMethodKey = buildPainlessMethodKey(methodName, methodArity);
Function<PainlessClass, PainlessMethod> objectLookup = targetPainlessClass -> targetPainlessClass.methods.get(painlessMethodKey);
return lookupRuntimePainlessObject(originalTargetClass, objectLookup);
}
public MethodHandle lookupRuntimeGetterMethodHandle(Class<?> originalTargetClass, String getterName) {
Objects.requireNonNull(originalTargetClass);
Objects.requireNonNull(getterName);
Function<PainlessClass, MethodHandle> objectLookup = targetPainlessClass -> targetPainlessClass.getterMethodHandles.get(getterName);
return lookupRuntimePainlessObject(originalTargetClass, objectLookup);
}
public MethodHandle lookupRuntimeSetterMethodHandle(Class<?> originalTargetClass, String setterName) {
Objects.requireNonNull(originalTargetClass);
Objects.requireNonNull(setterName);
Function<PainlessClass, MethodHandle> objectLookup = targetPainlessClass -> targetPainlessClass.setterMethodHandles.get(setterName);
return lookupRuntimePainlessObject(originalTargetClass, objectLookup);
}
private <T> T lookupRuntimePainlessObject(
Class<?> originalTargetClass, Function<PainlessClass, T> objectLookup) {
Class<?> currentTargetClass = originalTargetClass;
while (currentTargetClass != null) {
PainlessClass targetPainlessClass = classesToPainlessClasses.get(currentTargetClass);
if (targetPainlessClass != null) {
T painlessObject = objectLookup.apply(targetPainlessClass);
if (painlessObject != null) {
return painlessObject;
}
}
currentTargetClass = currentTargetClass.getSuperclass();
}
return functionalInterfacePainlessMethod;
currentTargetClass = originalTargetClass;
while (currentTargetClass != null) {
for (Class<?> targetInterface : currentTargetClass.getInterfaces()) {
PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetInterface);
if (targetPainlessClass != null) {
T painlessObject = objectLookup.apply(targetPainlessClass);
if (painlessObject != null) {
return painlessObject;
}
}
}
currentTargetClass = currentTargetClass.getSuperclass();
}
return null;
}
}

68
modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
View File

@ -220,8 +220,12 @@ public final class PainlessLookupBuilder {
return PainlessLookupUtility.canonicalTypeNameToType(canonicalTypeName, canonicalClassNamesToClasses);
}
private void validateType(Class<?> type) {
PainlessLookupUtility.validateType(type, classesToPainlessClassBuilders.keySet());
private boolean isValidType(Class<?> type) {
while (type.getComponentType() != null) {
type = type.getComponentType();
}
return classesToPainlessClassBuilders.containsKey(type);
}
public void addPainlessClass(ClassLoader classLoader, String javaClassName, boolean importClassName) {
@ -325,13 +329,14 @@ public final class PainlessLookupBuilder {
List<Class<?>> typeParameters = new ArrayList<>(typeNameParameters.size());
for (String typeNameParameter : typeNameParameters) {
try {
Class<?> typeParameter = canonicalTypeNameToType(typeNameParameter);
typeParameters.add(typeParameter);
} catch (IllegalArgumentException iae) {
Class<?> typeParameter = canonicalTypeNameToType(typeNameParameter);
if (typeParameter == null) {
throw new IllegalArgumentException("type parameter [" + typeNameParameter + "] not found " +
"for constructor [[" + targetCanonicalClassName + "], " + typeNameParameters + "]", iae);
"for constructor [[" + targetCanonicalClassName + "], " + typeNameParameters + "]");
}
typeParameters.add(typeParameter);
}
addPainlessConstructor(targetClass, typeParameters);
@ -357,11 +362,9 @@ public final class PainlessLookupBuilder {
List<Class<?>> javaTypeParameters = new ArrayList<>(typeParametersSize);
for (Class<?> typeParameter : typeParameters) {
try {
validateType(typeParameter);
} catch (IllegalArgumentException iae) {
if (isValidType(typeParameter) == false) {
throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] not found " +
"for constructor [[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "]", iae);
"for constructor [[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "]");
}
javaTypeParameters.add(typeToJavaType(typeParameter));
@ -435,22 +438,21 @@ public final class PainlessLookupBuilder {
List<Class<?>> typeParameters = new ArrayList<>(typeNameParameters.size());
for (String typeNameParameter : typeNameParameters) {
try {
Class<?> typeParameter = canonicalTypeNameToType(typeNameParameter);
typeParameters.add(typeParameter);
} catch (IllegalArgumentException iae) {
Class<?> typeParameter = canonicalTypeNameToType(typeNameParameter);
if (typeParameter == null) {
throw new IllegalArgumentException("parameter type [" + typeNameParameter + "] not found for method " +
"[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]", iae);
"[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]");
}
typeParameters.add(typeParameter);
}
Class<?> returnType;
Class<?> returnType = canonicalTypeNameToType(returnCanonicalTypeName);
try {
returnType = canonicalTypeNameToType(returnCanonicalTypeName);
} catch (IllegalArgumentException iae) {
if (returnType == null) {
throw new IllegalArgumentException("parameter type [" + returnCanonicalTypeName + "] not found for method " +
"[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]", iae);
"[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]");
}
addPainlessMethod(targetClass, augmentedClass, methodName, returnType, typeParameters);
@ -490,22 +492,18 @@ public final class PainlessLookupBuilder {
}
for (Class<?> typeParameter : typeParameters) {
try {
validateType(typeParameter);
} catch (IllegalArgumentException iae) {
if (isValidType(typeParameter) == false) {
throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] " +
"not found for method [[" + targetCanonicalClassName + "], [" + methodName + "], " +
typesToCanonicalTypeNames(typeParameters) + "]", iae);
typesToCanonicalTypeNames(typeParameters) + "]");
}
javaTypeParameters.add(typeToJavaType(typeParameter));
}
try {
validateType(returnType);
} catch (IllegalArgumentException iae) {
if (isValidType(returnType) == false) {
throw new IllegalArgumentException("return type [" + typeToCanonicalTypeName(returnType) + "] not found for method " +
"[[" + targetCanonicalClassName + "], [" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]", iae);
"[[" + targetCanonicalClassName + "], [" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]");
}
Method javaMethod;
@ -620,11 +618,9 @@ public final class PainlessLookupBuilder {
throw new IllegalArgumentException("class [" + targetCanonicalClassName + "] not found");
}
Class<?> typeParameter;
Class<?> typeParameter = canonicalTypeNameToType(typeNameParameter);
try {
typeParameter = canonicalTypeNameToType(typeNameParameter);
} catch (IllegalArgumentException iae) {
if (typeParameter == null) {
throw new IllegalArgumentException("type parameter [" + typeNameParameter + "] not found " +
"for field [[" + targetCanonicalClassName + "], [" + fieldName + "]");
}
@ -656,11 +652,9 @@ public final class PainlessLookupBuilder {
throw new IllegalArgumentException("class [" + targetCanonicalClassName + "] not found");
}
try {
validateType(typeParameter);
} catch (IllegalArgumentException iae) {
if (isValidType(typeParameter) == false) {
throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] not found " +
"for field [[" + targetCanonicalClassName + "], [" + fieldName + "]", iae);
"for field [[" + targetCanonicalClassName + "], [" + fieldName + "]");
}
Field javaField;

84
modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.painless.lookup;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -101,45 +100,47 @@ public final class PainlessLookupUtility {
canonicalTypeName.charAt(arrayIndex++) == ']') {
++arrayDimensions;
} else {
throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found");
return null;
}
}
canonicalTypeName = canonicalTypeName.substring(0, canonicalTypeName.indexOf('['));
type = canonicalClassNamesToClasses.get(canonicalTypeName);
char arrayBraces[] = new char[arrayDimensions];
Arrays.fill(arrayBraces, '[');
String javaTypeName = new String(arrayBraces);
if (type != null) {
char arrayBraces[] = new char[arrayDimensions];
Arrays.fill(arrayBraces, '[');
String javaTypeName = new String(arrayBraces);
if (type == boolean.class) {
javaTypeName += "Z";
} else if (type == byte.class) {
javaTypeName += "B";
} else if (type == short.class) {
javaTypeName += "S";
} else if (type == char.class) {
javaTypeName += "C";
} else if (type == int.class) {
javaTypeName += "I";
} else if (type == long.class) {
javaTypeName += "J";
} else if (type == float.class) {
javaTypeName += "F";
} else if (type == double.class) {
javaTypeName += "D";
} else {
javaTypeName += "L" + type.getName() + ";";
}
if (type == boolean.class) {
javaTypeName += "Z";
} else if (type == byte.class) {
javaTypeName += "B";
} else if (type == short.class) {
javaTypeName += "S";
} else if (type == char.class) {
javaTypeName += "C";
} else if (type == int.class) {
javaTypeName += "I";
} else if (type == long.class) {
javaTypeName += "J";
} else if (type == float.class) {
javaTypeName += "F";
} else if (type == double.class) {
javaTypeName += "D";
} else {
javaTypeName += "L" + type.getName() + ";";
}
try {
return Class.forName(javaTypeName);
} catch (ClassNotFoundException cnfe) {
throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found", cnfe);
try {
return Class.forName(javaTypeName);
} catch (ClassNotFoundException cnfe) {
throw new IllegalStateException("internal error", cnfe);
}
}
}
throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found");
return null;
}
/**
@ -152,7 +153,9 @@ public final class PainlessLookupUtility {
String canonicalTypeName = type.getCanonicalName();
if (canonicalTypeName.startsWith(def.class.getCanonicalName())) {
if (canonicalTypeName == null) {
canonicalTypeName = ANONYMOUS_CLASS_NAME;
} else if (canonicalTypeName.startsWith(def.class.getCanonicalName())) {
canonicalTypeName = canonicalTypeName.replace(def.class.getCanonicalName(), DEF_CLASS_NAME);
}
@ -252,22 +255,6 @@ public final class PainlessLookupUtility {
return type;
}
/**
* Ensures a type exists based on the terminology specified as part of {@link PainlessLookupUtility}. Throws an
* {@link IllegalArgumentException} if the type does not exist.
*/
public static void validateType(Class<?> type, Collection<Class<?>> classes) {
String canonicalTypeName = typeToCanonicalTypeName(type);
while (type.getComponentType() != null) {
type = type.getComponentType();
}
if (classes.contains(type) == false) {
throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found");
}
}
/**
* Converts a type to its boxed type equivalent if one exists based on the terminology specified as part of
* {@link PainlessLookupUtility}. Otherwise, this behaves as an identity function.
@ -357,6 +344,11 @@ public final class PainlessLookupUtility {
return fieldName;
}
/**
* The name for an anonymous class.
*/
public static final String ANONYMOUS_CLASS_NAME = "$anonymous";
/**
* The def type name as specified in the source for a script.
*/

6
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
View File

@ -49,9 +49,9 @@ public final class EExplicit extends AExpression {
@Override
void analyze(Locals locals) {
try {
actual = locals.getPainlessLookup().canonicalTypeNameToType(type);
} catch (IllegalArgumentException exception) {
actual = locals.getPainlessLookup().canonicalTypeNameToType(type);
if (actual == null) {
throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
}

7
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
View File

@ -54,12 +54,11 @@ public final class EInstanceof extends AExpression {
@Override
void analyze(Locals locals) {
Class<?> clazz;
// ensure the specified type is part of the definition
try {
clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
} catch (IllegalArgumentException exception) {
Class<?> clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
if (clazz == null) {
throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
}

19
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
View File

@ -33,6 +33,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
/**
* Represents a list initialization shortcut.
*/
@ -63,16 +65,17 @@ public final class EListInit extends AExpression {
actual = ArrayList.class;
try {
constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, 0);
} catch (IllegalArgumentException iae) {
throw createError(iae);
constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, 0);
if (constructor == null) {
throw createError(new IllegalArgumentException(
"constructor [" + typeToCanonicalTypeName(actual) + ", <init>/0] not found"));
}
try {
method = locals.getPainlessLookup().lookupPainlessMethod(actual, false, "add", 1);
} catch (IllegalArgumentException iae) {
throw createError(iae);
method = locals.getPainlessLookup().lookupPainlessMethod(actual, false, "add", 1);
if (method == null) {
throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(actual) + ", add/1] not found"));
}
for (int index = 0; index < values.size(); ++index) {

19
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
View File

@ -33,6 +33,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Set;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
/**
* Represents a map initialization shortcut.
*/
@ -69,16 +71,17 @@ public final class EMapInit extends AExpression {
actual = HashMap.class;
try {
constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, 0);
} catch (IllegalArgumentException iae) {
throw createError(iae);
constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, 0);
if (constructor == null) {
throw createError(new IllegalArgumentException(
"constructor [" + typeToCanonicalTypeName(actual) + ", <init>/0] not found"));
}
try {
method = locals.getPainlessLookup().lookupPainlessMethod(actual, false, "put", 2);
} catch (IllegalArgumentException iae) {
throw createError(iae);
method = locals.getPainlessLookup().lookupPainlessMethod(actual, false, "put", 2);
if (method == null) {
throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(actual) + ", put/2] not found"));
}
if (keys.size() != values.size()) {

10
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
View File

@ -54,15 +54,13 @@ public final class ENewArray extends AExpression {
@Override
void analyze(Locals locals) {
if (!read) {
throw createError(new IllegalArgumentException("A newly created array must be read from."));
if (!read) {
throw createError(new IllegalArgumentException("A newly created array must be read from."));
}
Class<?> clazz;
Class<?> clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
try {
clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
} catch (IllegalArgumentException exception) {
if (clazz == null) {
throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
}

17
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
View File

@ -32,6 +32,8 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
/**
* Represents and object instantiation.
*/
@ -58,16 +60,17 @@ public final class ENewObj extends AExpression {
@Override
void analyze(Locals locals) {
try {
actual = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
} catch (IllegalArgumentException exception) {
actual = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
if (actual == null) {
throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
}
try {
constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, arguments.size());
} catch (IllegalArgumentException iae) {
throw createError(iae);
constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, arguments.size());
if (constructor == null) {
throw createError(new IllegalArgumentException(
"constructor [" + typeToCanonicalTypeName(actual) + ", <init>/" + arguments.size() + "] not found"));
}
Class<?>[] types = new Class<?>[constructor.typeParameters.size()];

6
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
View File

@ -47,9 +47,9 @@ public final class EStatic extends AExpression {
@Override
void analyze(Locals locals) {
try {
actual = locals.getPainlessLookup().canonicalTypeNameToType(type);
} catch (IllegalArgumentException exception) {
actual = locals.getPainlessLookup().canonicalTypeNameToType(type);
if (actual == null) {
throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
}
}

16
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
View File

@ -30,6 +30,8 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
/**
* Represents a method call and defers to a child subnode.
*/
@ -67,13 +69,15 @@ public final class PCallInvoke extends AExpression {
if (prefix.actual == def.class) {
sub = new PSubDefCall(location, name, arguments);
} else {
try {
PainlessMethod method =
locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, prefix instanceof EStatic, name, arguments.size());
sub = new PSubCallInvoke(location, method, prefix.actual, arguments);
} catch (IllegalArgumentException iae) {
throw createError(iae);
PainlessMethod method =
locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, prefix instanceof EStatic, name, arguments.size());
if (method == null) {
throw createError(new IllegalArgumentException(
"method [" + typeToCanonicalTypeName(prefix.actual) + ", " + name + "/" + arguments.size() + "] not found"));
}
sub = new PSubCallInvoke(location, method, prefix.actual, arguments);
}
if (nullSafe) {

37
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.painless.Globals;
import org.elasticsearch.painless.Locals;
import org.elasticsearch.painless.Location;
import org.elasticsearch.painless.MethodWriter;
import org.elasticsearch.painless.lookup.PainlessField;
import org.elasticsearch.painless.lookup.PainlessLookupUtility;
import org.elasticsearch.painless.lookup.PainlessMethod;
import org.elasticsearch.painless.lookup.def;
@ -32,6 +33,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
/**
* Represents a field load/store and defers to a child subnode.
*/
@ -65,31 +68,22 @@ public final class PField extends AStoreable {
} else if (prefix.actual == def.class) {
sub = new PSubDefField(location, value);
} else {
try {
sub = new PSubField(location,
locals.getPainlessLookup().lookupPainlessField(prefix.actual, prefix instanceof EStatic, value));
} catch (IllegalArgumentException fieldIAE) {
PainlessField field = locals.getPainlessLookup().lookupPainlessField(prefix.actual, prefix instanceof EStatic, value);
if (field == null) {
PainlessMethod getter;
PainlessMethod setter;
try {
getter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
"get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
if (getter == null) {
getter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
"get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
} catch (IllegalArgumentException getIAE) {
try {
getter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
"is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
} catch (IllegalArgumentException isIAE) {
getter = null;
}
"is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
}
try {
setter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
"set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
} catch (IllegalArgumentException setIAE) {
setter = null;
}
setter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
"set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
if (getter != null || setter != null) {
sub = new PSubShortcut(location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), getter, setter);
@ -107,8 +101,11 @@ public final class PField extends AStoreable {
}
if (sub == null) {
throw createError(fieldIAE);
throw createError(new IllegalArgumentException(
"field [" + typeToCanonicalTypeName(prefix.actual) + ", " + value + "] not found"));
}
} else {
sub = new PSubField(location, field);
}
}

8
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
View File

@ -57,12 +57,8 @@ final class PSubListShortcut extends AStoreable {
void analyze(Locals locals) {
String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
try {
getter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
setter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "set", 2);
} catch (IllegalArgumentException iae) {
throw createError(iae);
}
getter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
setter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "set", 2);
if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1 ||
getter.typeParameters.get(0) != int.class)) {

8
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
View File

@ -56,12 +56,8 @@ final class PSubMapShortcut extends AStoreable {
void analyze(Locals locals) {
String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
try {
getter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
setter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "put", 2);
} catch (IllegalArgumentException iae) {
throw createError(iae);
}
getter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
setter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "put", 2);
if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1)) {
throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + canonicalClassName + "]."));

6
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
View File

@ -64,11 +64,9 @@ public final class SCatch extends AStatement {
@Override
void analyze(Locals locals) {
Class<?> clazz;
Class<?> clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
try {
clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
} catch (IllegalArgumentException exception) {
if (clazz == null) {
throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
}

6
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
View File

@ -59,11 +59,9 @@ public final class SDeclaration extends AStatement {
@Override
void analyze(Locals locals) {
Class<?> clazz;
Class<?> clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
try {
clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
} catch (IllegalArgumentException exception) {
if (clazz == null) {
throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
}

6
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
View File

@ -68,11 +68,9 @@ public class SEach extends AStatement {
expression.expected = expression.actual;
expression = expression.cast(locals);
Class<?> clazz;
Class<?> clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
try {
clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type);
} catch (IllegalArgumentException exception) {
if (clazz == null) {
throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
}

16
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
View File

@ -115,9 +115,9 @@ public final class SFunction extends AStatement {
}
void generateSignature(PainlessLookup painlessLookup) {
try {
returnType = painlessLookup.canonicalTypeNameToType(rtnTypeStr);
} catch (IllegalArgumentException exception) {
returnType = painlessLookup.canonicalTypeNameToType(rtnTypeStr);
if (returnType == null) {
throw createError(new IllegalArgumentException("Illegal return type [" + rtnTypeStr + "] for function [" + name + "]."));
}
@ -129,16 +129,16 @@ public final class SFunction extends AStatement {
List<Class<?>> paramTypes = new ArrayList<>();
for (int param = 0; param < this.paramTypeStrs.size(); ++param) {
try {
Class<?> paramType = painlessLookup.canonicalTypeNameToType(this.paramTypeStrs.get(param));
paramClasses[param] = PainlessLookupUtility.typeToJavaType(paramType);
paramTypes.add(paramType);
parameters.add(new Parameter(location, paramNameStrs.get(param), paramType));
} catch (IllegalArgumentException exception) {
if (paramType == null) {
throw createError(new IllegalArgumentException(
"Illegal parameter type [" + this.paramTypeStrs.get(param) + "] for function [" + name + "]."));
}
paramClasses[param] = PainlessLookupUtility.typeToJavaType(paramType);
paramTypes.add(paramType);
parameters.add(new Parameter(location, paramNameStrs.get(param), paramType));
}
typeParameters = paramTypes;

10
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
View File

@ -40,6 +40,7 @@ import java.util.Set;
import static org.elasticsearch.painless.WriterConstants.ITERATOR_HASNEXT;
import static org.elasticsearch.painless.WriterConstants.ITERATOR_NEXT;
import static org.elasticsearch.painless.WriterConstants.ITERATOR_TYPE;
import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
/**
* Represents a for-each loop for iterables.
@ -76,10 +77,11 @@ final class SSubEachIterable extends AStatement {
if (expression.actual == def.class) {
method = null;
} else {
try {
method = locals.getPainlessLookup().lookupPainlessMethod(expression.actual, false, "iterator", 0);
} catch (IllegalArgumentException iae) {
throw createError(iae);
method = locals.getPainlessLookup().lookupPainlessMethod(expression.actual, false, "iterator", 0);
if (method == null) {
throw createError(new IllegalArgumentException(
"method [" + typeToCanonicalTypeName(expression.actual) + ", iterator/0] not found"));
}
}

2
modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
View File

@ -134,7 +134,7 @@ public class DefBootstrapTests extends ESTestCase {
final IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> {
Integer.toString((int)handle.invokeExact(new Object()));
});
assertEquals("Unable to find dynamic method [size] with [0] arguments for class [java.lang.Object].", iae.getMessage());
assertEquals("dynamic method [java.lang.Object, size/0] not found", iae.getMessage());
assertTrue("Does not fail inside ClassValue.computeValue()", Arrays.stream(iae.getStackTrace()).anyMatch(e -> {
return e.getMethodName().equals("computeValue") &&
e.getClassName().startsWith("org.elasticsearch.painless.DefBootstrap$PIC$");

2
modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
View File

@ -37,7 +37,7 @@ public class OverloadTests extends ScriptTestCase {
IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
exec("def x = 'abc123abc'; return x.indexOf('c', 3, 'bogus');");
});
assertTrue(expected.getMessage().contains("dynamic method [indexOf]"));
assertTrue(expected.getMessage().contains("dynamic method [java.lang.String, indexOf/3] not found"));
}
public void testConstructor() {

2
modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
View File

@ -219,7 +219,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
exec("def x = 'test'; return x.getClass().toString()");
});
assertTrue(expected.getMessage().contains("Unable to find dynamic method"));
assertTrue(expected.getMessage().contains("dynamic method [java.lang.String, getClass/0] not found"));
}
public void testDynamicNPE() {

9
rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
View File

@ -7,7 +7,6 @@ setup:
test:
properties:
numeric_group: { type: integer }
group_alias: { type: alias, path: numeric_group }
- do:
index:
@ -354,6 +353,14 @@ setup:
- skip:
version: " - 6.3.99"
reason: Field aliases were introduced in 6.4.0.
- do:
indices.put_mapping:
index: test
type: test
body:
test:
properties:
group_alias: { type: alias, path: numeric_group }
- do:
search:
index: test

7
server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineAction.java
View File

@ -20,8 +20,9 @@
package org.elasticsearch.action.ingest;
import org.elasticsearch.action.Action;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
public class DeletePipelineAction extends Action<WritePipelineResponse> {
public class DeletePipelineAction extends Action<AcknowledgedResponse> {
public static final DeletePipelineAction INSTANCE = new DeletePipelineAction();
public static final String NAME = "cluster:admin/ingest/pipeline/delete";
@ -31,7 +32,7 @@ public class DeletePipelineAction extends Action<WritePipelineResponse> {
}
@Override
public WritePipelineResponse newResponse() {
return new WritePipelineResponse();
public AcknowledgedResponse newResponse() {
return new AcknowledgedResponse();
}
}

3
server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java
View File

@ -20,9 +20,10 @@
package org.elasticsearch.action.ingest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ElasticsearchClient;
public class DeletePipelineRequestBuilder extends ActionRequestBuilder<DeletePipelineRequest, WritePipelineResponse> {
public class DeletePipelineRequestBuilder extends ActionRequestBuilder<DeletePipelineRequest, AcknowledgedResponse> {
public DeletePipelineRequestBuilder(ElasticsearchClient client, DeletePipelineAction action) {
super(client, action, new DeletePipelineRequest());

9
server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.action.ingest;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.support.master.TransportMasterNodeAction;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
@ -34,7 +35,7 @@ import org.elasticsearch.node.NodeService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
public class DeletePipelineTransportAction extends TransportMasterNodeAction<DeletePipelineRequest, WritePipelineResponse> {
public class DeletePipelineTransportAction extends TransportMasterNodeAction<DeletePipelineRequest, AcknowledgedResponse> {
private final PipelineStore pipelineStore;
private final ClusterService clusterService;
@ -54,12 +55,12 @@ public class DeletePipelineTransportAction extends TransportMasterNodeAction<Del
}
@Override
protected WritePipelineResponse newResponse() {
return new WritePipelineResponse();
protected AcknowledgedResponse newResponse() {
return new AcknowledgedResponse();
}
@Override
protected void masterOperation(DeletePipelineRequest request, ClusterState state, ActionListener<WritePipelineResponse> listener) throws Exception {
protected void masterOperation(DeletePipelineRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) throws Exception {
pipelineStore.delete(clusterService, request, listener);
}

7
server/src/main/java/org/elasticsearch/action/ingest/PutPipelineAction.java
View File

@ -20,8 +20,9 @@
package org.elasticsearch.action.ingest;
import org.elasticsearch.action.Action;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
public class PutPipelineAction extends Action<WritePipelineResponse> {
public class PutPipelineAction extends Action<AcknowledgedResponse> {
public static final PutPipelineAction INSTANCE = new PutPipelineAction();
public static final String NAME = "cluster:admin/ingest/pipeline/put";
@ -31,7 +32,7 @@ public class PutPipelineAction extends Action<WritePipelineResponse> {
}
@Override
public WritePipelineResponse newResponse() {
return new WritePipelineResponse();
public AcknowledgedResponse newResponse() {
return new AcknowledgedResponse();
}
}

3
server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java
View File

@ -20,11 +20,12 @@
package org.elasticsearch.action.ingest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentType;
public class PutPipelineRequestBuilder extends ActionRequestBuilder<PutPipelineRequest, WritePipelineResponse> {
public class PutPipelineRequestBuilder extends ActionRequestBuilder<PutPipelineRequest, AcknowledgedResponse> {
public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action) {
super(client, action, new PutPipelineRequest());

9
server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.support.master.TransportMasterNodeAction;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.cluster.ClusterState;
@ -43,7 +44,7 @@ import org.elasticsearch.transport.TransportService;
import java.util.HashMap;
import java.util.Map;
public class PutPipelineTransportAction extends TransportMasterNodeAction<PutPipelineRequest, WritePipelineResponse> {
public class PutPipelineTransportAction extends TransportMasterNodeAction<PutPipelineRequest, AcknowledgedResponse> {
private final PipelineStore pipelineStore;
private final ClusterService clusterService;
@ -66,12 +67,12 @@ public class PutPipelineTransportAction extends TransportMasterNodeAction<PutPip
}
@Override
protected WritePipelineResponse newResponse() {
return new WritePipelineResponse();
protected AcknowledgedResponse newResponse() {
return new AcknowledgedResponse();
}
@Override
protected void masterOperation(PutPipelineRequest request, ClusterState state, ActionListener<WritePipelineResponse> listener) throws Exception {
protected void masterOperation(PutPipelineRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) throws Exception {
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
nodesInfoRequest.clear();
nodesInfoRequest.ingest(true);

38
server/src/main/java/org/elasticsearch/action/ingest/WritePipelineResponse.java
View File

@ -1,38 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.ingest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentParser;
public class WritePipelineResponse extends AcknowledgedResponse implements ToXContentObject {
WritePipelineResponse() {
}
public WritePipelineResponse(boolean acknowledged) {
super(acknowledged);
}
public static WritePipelineResponse fromXContent(XContentParser parser) {
return new WritePipelineResponse(parseAcknowledged(parser));
}
}

17
server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
View File

@ -34,10 +34,9 @@ import java.util.Objects;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
/**
* Abstract class that allows to mark action responses that support acknowledgements.
* Facilitates consistency across different api.
* A response that indicates that a request has been acknowledged
*/
public abstract class AcknowledgedResponse extends ActionResponse implements ToXContentObject {
public class AcknowledgedResponse extends ActionResponse implements ToXContentObject {
private static final ParseField ACKNOWLEDGED = new ParseField("acknowledged");
@ -48,11 +47,10 @@ public abstract class AcknowledgedResponse extends ActionResponse implements ToX
protected boolean acknowledged;
protected AcknowledgedResponse() {
public AcknowledgedResponse() {
}
protected AcknowledgedResponse(boolean acknowledged) {
public AcknowledgedResponse(boolean acknowledged) {
this.acknowledged = acknowledged;
}
@ -100,10 +98,15 @@ public abstract class AcknowledgedResponse extends ActionResponse implements ToX
ObjectParser.ValueType.BOOLEAN);
}
protected static boolean parseAcknowledged(XContentParser parser) {
@Deprecated
public static boolean parseAcknowledged(XContentParser parser) {
return ACKNOWLEDGED_FLAG_PARSER.apply(parser, null);
}
public static AcknowledgedResponse fromXContent(XContentParser parser) throws IOException {
return new AcknowledgedResponse(ACKNOWLEDGED_FLAG_PARSER.apply(parser, null));
}
@Override
public boolean equals(Object o) {
if (this == o) {

10
server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
View File

@ -113,7 +113,7 @@ import org.elasticsearch.action.ingest.PutPipelineRequestBuilder;
import org.elasticsearch.action.ingest.SimulatePipelineRequest;
import org.elasticsearch.action.ingest.SimulatePipelineRequestBuilder;
import org.elasticsearch.action.ingest.SimulatePipelineResponse;
import org.elasticsearch.action.ingest.WritePipelineResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.tasks.TaskId;
@ -574,12 +574,12 @@ public interface ClusterAdminClient extends ElasticsearchClient {
/**
* Stores an ingest pipeline
*/
void putPipeline(PutPipelineRequest request, ActionListener<WritePipelineResponse> listener);
void putPipeline(PutPipelineRequest request, ActionListener<AcknowledgedResponse> listener);
/**
* Stores an ingest pipeline
*/
ActionFuture<WritePipelineResponse> putPipeline(PutPipelineRequest request);
ActionFuture<AcknowledgedResponse> putPipeline(PutPipelineRequest request);
/**
* Stores an ingest pipeline
@ -596,12 +596,12 @@ public interface ClusterAdminClient extends ElasticsearchClient {
/**
* Deletes a stored ingest pipeline
*/
void deletePipeline(DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener);
void deletePipeline(DeletePipelineRequest request, ActionListener<AcknowledgedResponse> listener);
/**
* Deletes a stored ingest pipeline
*/
ActionFuture<WritePipelineResponse> deletePipeline(DeletePipelineRequest request);
ActionFuture<AcknowledgedResponse> deletePipeline(DeletePipelineRequest request);
/**
* Deletes a stored ingest pipeline

10
server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
View File

@ -307,7 +307,6 @@ import org.elasticsearch.action.ingest.SimulatePipelineAction;
import org.elasticsearch.action.ingest.SimulatePipelineRequest;
import org.elasticsearch.action.ingest.SimulatePipelineRequestBuilder;
import org.elasticsearch.action.ingest.SimulatePipelineResponse;
import org.elasticsearch.action.ingest.WritePipelineResponse;
import org.elasticsearch.action.search.ClearScrollAction;
import org.elasticsearch.action.search.ClearScrollRequest;
import org.elasticsearch.action.search.ClearScrollRequestBuilder;
@ -325,6 +324,7 @@ import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.action.support.ThreadedActionListener;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.termvectors.MultiTermVectorsAction;
import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
import org.elasticsearch.action.termvectors.MultiTermVectorsRequestBuilder;
@ -1082,12 +1082,12 @@ public abstract class AbstractClient extends AbstractComponent implements Client
}
@Override
public void putPipeline(PutPipelineRequest request, ActionListener<WritePipelineResponse> listener) {
public void putPipeline(PutPipelineRequest request, ActionListener<AcknowledgedResponse> listener) {
execute(PutPipelineAction.INSTANCE, request, listener);
}
@Override
public ActionFuture<WritePipelineResponse> putPipeline(PutPipelineRequest request) {
public ActionFuture<AcknowledgedResponse> putPipeline(PutPipelineRequest request) {
return execute(PutPipelineAction.INSTANCE, request);
}
@ -1102,12 +1102,12 @@ public abstract class AbstractClient extends AbstractComponent implements Client
}
@Override
public void deletePipeline(DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener) {
public void deletePipeline(DeletePipelineRequest request, ActionListener<AcknowledgedResponse> listener) {
execute(DeletePipelineAction.INSTANCE, request, listener);
}
@Override
public ActionFuture<WritePipelineResponse> deletePipeline(DeletePipelineRequest request) {
public ActionFuture<AcknowledgedResponse> deletePipeline(DeletePipelineRequest request) {
return execute(DeletePipelineAction.INSTANCE, request);
}

1
server/src/main/java/org/elasticsearch/common/time/CompoundDateTimeFormatter.java
View File

@ -70,4 +70,5 @@ public class CompoundDateTimeFormatter {
public String format(TemporalAccessor accessor) {
return printer.format(accessor);
}
}

1559
server/src/main/java/org/elasticsearch/common/time/DateFormatters.java
View File

File diff suppressed because it is too large Load Diff

18
server/src/main/java/org/elasticsearch/ingest/PipelineStore.java
View File

@ -25,7 +25,7 @@ import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ingest.DeletePipelineRequest;
import org.elasticsearch.action.ingest.PutPipelineRequest;
import org.elasticsearch.action.ingest.WritePipelineResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
@ -121,13 +121,13 @@ public class PipelineStore extends AbstractComponent implements ClusterStateAppl
/**
* Deletes the pipeline specified by id in the request.
*/
public void delete(ClusterService clusterService, DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener) {
public void delete(ClusterService clusterService, DeletePipelineRequest request, ActionListener<AcknowledgedResponse> listener) {
clusterService.submitStateUpdateTask("delete-pipeline-" + request.getId(),
new AckedClusterStateUpdateTask<WritePipelineResponse>(request, listener) {
new AckedClusterStateUpdateTask<AcknowledgedResponse>(request, listener) {
@Override
protected WritePipelineResponse newResponse(boolean acknowledged) {
return new WritePipelineResponse(acknowledged);
protected AcknowledgedResponse newResponse(boolean acknowledged) {
return new AcknowledgedResponse(acknowledged);
}
@Override
@ -169,15 +169,15 @@ public class PipelineStore extends AbstractComponent implements ClusterStateAppl
* Stores the specified pipeline definition in the request.
*/
public void put(ClusterService clusterService, Map<DiscoveryNode, IngestInfo> ingestInfos, PutPipelineRequest request,
ActionListener<WritePipelineResponse> listener) throws Exception {
ActionListener<AcknowledgedResponse> listener) throws Exception {
// validates the pipeline and processor configuration before submitting a cluster update task:
validatePipeline(ingestInfos, request);
clusterService.submitStateUpdateTask("put-pipeline-" + request.getId(),
new AckedClusterStateUpdateTask<WritePipelineResponse>(request, listener) {
new AckedClusterStateUpdateTask<AcknowledgedResponse>(request, listener) {
@Override
protected WritePipelineResponse newResponse(boolean acknowledged) {
return new WritePipelineResponse(acknowledged);
protected AcknowledgedResponse newResponse(boolean acknowledged) {
return new AcknowledgedResponse(acknowledged);
}
@Override

5
server/src/main/java/org/elasticsearch/rest/action/RestMainAction.java
View File

@ -67,4 +67,9 @@ public class RestMainAction extends BaseRestHandler {
response.toXContent(builder, request);
return new BytesRestResponse(RestStatus.OK, builder);
}
@Override
public boolean canTripCircuitBreaker() {
return false;
}
}

83
server/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java
View File

@ -1,83 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.ingest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractStreamableXContentTestCase;
import java.io.IOException;
import static org.hamcrest.CoreMatchers.equalTo;
public class WritePipelineResponseTests extends AbstractStreamableXContentTestCase<WritePipelineResponse> {
public void testSerializationWithoutError() throws IOException {
boolean isAcknowledged = randomBoolean();
WritePipelineResponse response;
response = new WritePipelineResponse(isAcknowledged);
BytesStreamOutput out = new BytesStreamOutput();
response.writeTo(out);
StreamInput streamInput = out.bytes().streamInput();
WritePipelineResponse otherResponse = new WritePipelineResponse();
otherResponse.readFrom(streamInput);
assertThat(otherResponse.isAcknowledged(), equalTo(response.isAcknowledged()));
}
public void testSerializationWithError() throws IOException {
WritePipelineResponse response = new WritePipelineResponse();
BytesStreamOutput out = new BytesStreamOutput();
response.writeTo(out);
StreamInput streamInput = out.bytes().streamInput();
WritePipelineResponse otherResponse = new WritePipelineResponse();
otherResponse.readFrom(streamInput);
assertThat(otherResponse.isAcknowledged(), equalTo(response.isAcknowledged()));
}
public void testToXContent() {
WritePipelineResponse response = new WritePipelineResponse(true);
String output = Strings.toString(response);
assertEquals("{\"acknowledged\":true}", output);
}
@Override
protected WritePipelineResponse doParseInstance(XContentParser parser) {
return WritePipelineResponse.fromXContent(parser);
}
@Override
protected WritePipelineResponse createTestInstance() {
return new WritePipelineResponse(randomBoolean());
}
@Override
protected WritePipelineResponse createBlankInstance() {
return new WritePipelineResponse();
}
@Override
protected WritePipelineResponse mutateInstance(WritePipelineResponse response) {
return new WritePipelineResponse(response.isAcknowledged() == false);
}
}

10
server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java
View File

@ -453,10 +453,12 @@ public class JavaJodaTimeDuellingTests extends ESTestCase {
}
private void assertSamePrinterOutput(String format, ZonedDateTime javaDate, DateTime jodaDate) {
assertThat(jodaDate.getMillis(), is(javaDate.toEpochSecond() * 1000));
String javaTimeOut = DateFormatters.forPattern("dateOptionalTime").format(javaDate);
String jodaTimeOut = Joda.forPattern("dateOptionalTime").printer().print(jodaDate);
assertThat(javaTimeOut, is(jodaTimeOut));
assertThat(jodaDate.getMillis(), is(javaDate.toInstant().toEpochMilli()));
String javaTimeOut = DateFormatters.forPattern(format).format(javaDate);
String jodaTimeOut = Joda.forPattern(format).printer().print(jodaDate);
String message = String.format(Locale.ROOT, "expected string representation to be equal for format [%s]: joda [%s], java [%s]",
format, jodaTimeOut, javaTimeOut);
assertThat(message, javaTimeOut, is(jodaTimeOut));
}
private void assertSameDate(String input, String format) {

8
server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
View File

@ -34,7 +34,7 @@ import org.elasticsearch.action.ingest.PutPipelineRequest;
import org.elasticsearch.action.ingest.SimulateDocumentBaseResult;
import org.elasticsearch.action.ingest.SimulatePipelineRequest;
import org.elasticsearch.action.ingest.SimulatePipelineResponse;
import org.elasticsearch.action.ingest.WritePipelineResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.bytes.BytesReference;
@ -126,7 +126,7 @@ public class IngestClientIT extends ESIntegTestCase {
assertThat(simulateDocumentBaseResult.getFailure(), nullValue());
// cleanup
WritePipelineResponse deletePipelineResponse = client().admin().cluster().prepareDeletePipeline("_id").get();
AcknowledgedResponse deletePipelineResponse = client().admin().cluster().prepareDeletePipeline("_id").get();
assertTrue(deletePipelineResponse.isAcknowledged());
}
@ -172,7 +172,7 @@ public class IngestClientIT extends ESIntegTestCase {
}
// cleanup
WritePipelineResponse deletePipelineResponse = client().admin().cluster().prepareDeletePipeline("_id").get();
AcknowledgedResponse deletePipelineResponse = client().admin().cluster().prepareDeletePipeline("_id").get();
assertTrue(deletePipelineResponse.isAcknowledged());
}
@ -246,7 +246,7 @@ public class IngestClientIT extends ESIntegTestCase {
assertThat(doc.get("processed"), equalTo(true));
DeletePipelineRequest deletePipelineRequest = new DeletePipelineRequest("_id");
WritePipelineResponse response = client().admin().cluster().deletePipeline(deletePipelineRequest).get();
AcknowledgedResponse response = client().admin().cluster().deletePipeline(deletePipelineRequest).get();
assertThat(response.isAcknowledged(), is(true));
getResponse = client().admin().cluster().prepareGetPipeline("_id").get();

4
server/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
View File

@ -20,7 +20,7 @@
package org.elasticsearch.ingest;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.ingest.WritePipelineResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.node.NodeService;
@ -95,7 +95,7 @@ public class IngestProcessorNotInstalledOnAllNodesIT extends ESIntegTestCase {
installPlugin = true;
String node1 = internalCluster().startNode();
WritePipelineResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get();
AcknowledgedResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get();
assertThat(response.isAcknowledged(), is(true));
Pipeline pipeline = internalCluster().getInstance(NodeService.class, node1).getIngestService().getPipelineStore().get("_id");
assertThat(pipeline, notNullValue());

9
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java
View File

@ -50,6 +50,7 @@ import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.Rewriteable;
import org.elasticsearch.index.query.TermsQueryBuilder;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.search.NestedHelper;
import org.elasticsearch.index.shard.IndexSearcherWrapper;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardUtils;
@ -73,6 +74,7 @@ import java.util.List;
import java.util.Map;
import java.util.function.Function;
import static org.apache.lucene.search.BooleanClause.Occur.FILTER;
import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
/**
@ -139,6 +141,13 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
Query roleQuery = queryShardContext.toFilter(queryBuilder).query();
filter.add(roleQuery, SHOULD);
if (queryShardContext.getMapperService().hasNested()) {
NestedHelper nestedHelper = new NestedHelper(queryShardContext.getMapperService());
if (nestedHelper.mightMatchNestedDocs(roleQuery)) {
roleQuery = new BooleanQuery.Builder()
.add(roleQuery, FILTER)
.add(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()), FILTER)
.build();
}
// If access is allowed on root doc then also access is allowed on all nested docs of that root document:
BitSetProducer rootDocs = queryShardContext.bitsetFilter(
Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()));

5
x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
View File

@ -15,7 +15,6 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.ingest.PutPipelineRequest;
import org.elasticsearch.action.ingest.WritePipelineResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterChangedEvent;
@ -39,6 +38,7 @@ import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.ingest.IngestMetadata;
import org.elasticsearch.ingest.PipelineConfiguration;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
import org.elasticsearch.xpack.core.XPackClient;
@ -46,7 +46,6 @@ import org.elasticsearch.xpack.core.XPackSettings;
import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
import org.elasticsearch.xpack.core.watcher.watch.Watch;
@ -385,7 +384,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
* }
* </code></pre>
*/
private void putIngestPipeline(final String pipelineId, final ActionListener<WritePipelineResponse> listener) {
private void putIngestPipeline(final String pipelineId, final ActionListener<AcknowledgedResponse> listener) {
final String pipelineName = pipelineName(pipelineId);
final BytesReference pipeline = BytesReference.bytes(loadPipeline(pipelineId, XContentType.JSON));
final PutPipelineRequest request = new PutPipelineRequest(pipelineName, pipeline, XContentType.JSON);

17
x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
View File

@ -97,7 +97,8 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
return super.configUsers() +
"user1:" + usersPasswdHashed + "\n" +
"user2:" + usersPasswdHashed + "\n" +
"user3:" + usersPasswdHashed + "\n";
"user3:" + usersPasswdHashed + "\n" +
"user4:" + usersPasswdHashed + "\n";
}
@Override
@ -105,7 +106,8 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
return super.configUsersRoles() +
"role1:user1,user2,user3\n" +
"role2:user1,user3\n" +
"role3:user2,user3\n";
"role3:user2,user3\n" +
"role4:user4\n";
}
@Override
@ -131,7 +133,14 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
" indices:\n" +
" - names: '*'\n" +
" privileges: [ ALL ]\n" +
" query: '{\"term\" : {\"field2\" : \"value2\"}}'"; // <-- query defined as json in a string
" query: '{\"term\" : {\"field2\" : \"value2\"}}'\n" + // <-- query defined as json in a string
"role4:\n" +
" cluster: [ all ]\n" +
" indices:\n" +
" - names: '*'\n" +
" privileges: [ ALL ]\n" +
// query that can match nested documents
" query: '{\"bool\": { \"must_not\": { \"term\" : {\"field1\" : \"value2\"}}}}'";
}
@Override
@ -869,7 +878,7 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
refresh("test");
SearchResponse response = client()
.filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
.filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
.prepareSearch("test")
.setQuery(QueryBuilders.nestedQuery("nested_field", QueryBuilders.termQuery("nested_field.field2", "value2"),
ScoreMode.None).innerHit(new InnerHitBuilder()))

400
x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfig.java Normal file
View File

@ -0,0 +1,400 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml.job.config;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
/**
* Analysis configuration options that describe which fields are
* analyzed and which functions are used to detect anomalies.
* <p>
* The configuration can contain multiple detectors, a new anomaly detector will
* be created for each detector configuration. The fields
* <code>bucketSpan, summaryCountFieldName and categorizationFieldName</code>
* apply to all detectors.
* <p>
* If a value has not been set it will be <code>null</code>
* Object wrappers are used around integral types &amp; booleans so they can take
* <code>null</code> values.
*/
public class AnalysisConfig implements ToXContentObject {
/**
* Serialisation names
*/
public static final ParseField ANALYSIS_CONFIG = new ParseField("analysis_config");
public static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
public static final ParseField CATEGORIZATION_FIELD_NAME = new ParseField("categorization_field_name");
public static final ParseField CATEGORIZATION_FILTERS = new ParseField("categorization_filters");
public static final ParseField CATEGORIZATION_ANALYZER = CategorizationAnalyzerConfig.CATEGORIZATION_ANALYZER;
public static final ParseField LATENCY = new ParseField("latency");
public static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name");
public static final ParseField DETECTORS = new ParseField("detectors");
public static final ParseField INFLUENCERS = new ParseField("influencers");
public static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets");
public static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window");
public static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields");
@SuppressWarnings("unchecked")
public static final ConstructingObjectParser<Builder, Void> PARSER = new ConstructingObjectParser<>(ANALYSIS_CONFIG.getPreferredName(),
true, a -> new AnalysisConfig.Builder((List<Detector>) a[0]));
static {
PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(),
(p, c) -> (Detector.PARSER).apply(p, c).build(), DETECTORS);
PARSER.declareString((builder, val) ->
builder.setBucketSpan(TimeValue.parseTimeValue(val, BUCKET_SPAN.getPreferredName())), BUCKET_SPAN);
PARSER.declareString(Builder::setCategorizationFieldName, CATEGORIZATION_FIELD_NAME);
PARSER.declareStringArray(Builder::setCategorizationFilters, CATEGORIZATION_FILTERS);
// This one is nasty - the syntax for analyzers takes either names or objects at many levels, hence it's not
// possible to simply declare whether the field is a string or object and a completely custom parser is required
PARSER.declareField(Builder::setCategorizationAnalyzerConfig,
(p, c) -> CategorizationAnalyzerConfig.buildFromXContentFragment(p),
CATEGORIZATION_ANALYZER, ObjectParser.ValueType.OBJECT_OR_STRING);
PARSER.declareString((builder, val) ->
builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
PARSER.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
PARSER.declareStringArray(Builder::setInfluencers, INFLUENCERS);
PARSER.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS);
PARSER.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW);
PARSER.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);
}
/**
* These values apply to all detectors
*/
private final TimeValue bucketSpan;
private final String categorizationFieldName;
private final List<String> categorizationFilters;
private final CategorizationAnalyzerConfig categorizationAnalyzerConfig;
private final TimeValue latency;
private final String summaryCountFieldName;
private final List<Detector> detectors;
private final List<String> influencers;
private final Boolean overlappingBuckets;
private final Long resultFinalizationWindow;
private final Boolean multivariateByFields;
private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List<String> categorizationFilters,
CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName,
List<Detector> detectors, List<String> influencers, Boolean overlappingBuckets, Long resultFinalizationWindow,
Boolean multivariateByFields) {
this.detectors = Collections.unmodifiableList(detectors);
this.bucketSpan = bucketSpan;
this.latency = latency;
this.categorizationFieldName = categorizationFieldName;
this.categorizationAnalyzerConfig = categorizationAnalyzerConfig;
this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters);
this.summaryCountFieldName = summaryCountFieldName;
this.influencers = Collections.unmodifiableList(influencers);
this.overlappingBuckets = overlappingBuckets;
this.resultFinalizationWindow = resultFinalizationWindow;
this.multivariateByFields = multivariateByFields;
}
/**
* The analysis bucket span
*
* @return The bucketspan or <code>null</code> if not set
*/
public TimeValue getBucketSpan() {
return bucketSpan;
}
public String getCategorizationFieldName() {
return categorizationFieldName;
}
public List<String> getCategorizationFilters() {
return categorizationFilters;
}
public CategorizationAnalyzerConfig getCategorizationAnalyzerConfig() {
return categorizationAnalyzerConfig;
}
/**
* The latency interval during which out-of-order records should be handled.
*
* @return The latency interval or <code>null</code> if not set
*/
public TimeValue getLatency() {
return latency;
}
/**
* The name of the field that contains counts for pre-summarised input
*
* @return The field name or <code>null</code> if not set
*/
public String getSummaryCountFieldName() {
return summaryCountFieldName;
}
/**
* The list of analysis detectors. In a valid configuration the list should
* contain at least 1 {@link Detector}
*
* @return The Detectors used in this job
*/
public List<Detector> getDetectors() {
return detectors;
}
/**
* The list of influence field names
*/
public List<String> getInfluencers() {
return influencers;
}
public Boolean getOverlappingBuckets() {
return overlappingBuckets;
}
public Long getResultFinalizationWindow() {
return resultFinalizationWindow;
}
public Boolean getMultivariateByFields() {
return multivariateByFields;
}
private static void addIfNotNull(Set<String> fields, String field) {
if (field != null) {
fields.add(field);
}
}
public List<String> fields() {
return collectNonNullAndNonEmptyDetectorFields(Detector::getFieldName);
}
private List<String> collectNonNullAndNonEmptyDetectorFields(
Function<Detector, String> fieldGetter) {
Set<String> fields = new HashSet<>();
for (Detector d : getDetectors()) {
addIfNotNull(fields, fieldGetter.apply(d));
}
// remove empty strings
fields.remove("");
return new ArrayList<>(fields);
}
public List<String> byFields() {
return collectNonNullAndNonEmptyDetectorFields(Detector::getByFieldName);
}
public List<String> overFields() {
return collectNonNullAndNonEmptyDetectorFields(Detector::getOverFieldName);
}
public List<String> partitionFields() {
return collectNonNullAndNonEmptyDetectorFields(Detector::getPartitionFieldName);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
if (bucketSpan != null) {
builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan.getStringRep());
}
if (categorizationFieldName != null) {
builder.field(CATEGORIZATION_FIELD_NAME.getPreferredName(), categorizationFieldName);
}
if (categorizationFilters != null) {
builder.field(CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters);
}
if (categorizationAnalyzerConfig != null) {
// This cannot be builder.field(CATEGORIZATION_ANALYZER.getPreferredName(), categorizationAnalyzerConfig, params);
// because that always writes categorizationAnalyzerConfig as an object, and in the case of a global analyzer it
// gets written as a single string.
categorizationAnalyzerConfig.toXContent(builder, params);
}
if (latency != null) {
builder.field(LATENCY.getPreferredName(), latency.getStringRep());
}
if (summaryCountFieldName != null) {
builder.field(SUMMARY_COUNT_FIELD_NAME.getPreferredName(), summaryCountFieldName);
}
builder.startArray(DETECTORS.getPreferredName());
for (Detector detector : detectors) {
detector.toXContent(builder, params);
}
builder.endArray();
builder.field(INFLUENCERS.getPreferredName(), influencers);
if (overlappingBuckets != null) {
builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets);
}
if (resultFinalizationWindow != null) {
builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow);
}
if (multivariateByFields != null) {
builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields);
}
builder.endObject();
return builder;
}
@Override
public boolean equals(Object object) {
if (this == object) {
return true;
}
if (object == null || getClass() != object.getClass()) {
return false;
}
AnalysisConfig that = (AnalysisConfig) object;
return Objects.equals(latency, that.latency) &&
Objects.equals(bucketSpan, that.bucketSpan) &&
Objects.equals(categorizationFieldName, that.categorizationFieldName) &&
Objects.equals(categorizationFilters, that.categorizationFilters) &&
Objects.equals(categorizationAnalyzerConfig, that.categorizationAnalyzerConfig) &&
Objects.equals(summaryCountFieldName, that.summaryCountFieldName) &&
Objects.equals(detectors, that.detectors) &&
Objects.equals(influencers, that.influencers) &&
Objects.equals(overlappingBuckets, that.overlappingBuckets) &&
Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) &&
Objects.equals(multivariateByFields, that.multivariateByFields);
}
@Override
public int hashCode() {
return Objects.hash(
bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency,
summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow,
multivariateByFields);
}
public static class Builder {
private List<Detector> detectors;
private TimeValue bucketSpan;
private TimeValue latency;
private String categorizationFieldName;
private List<String> categorizationFilters;
private CategorizationAnalyzerConfig categorizationAnalyzerConfig;
private String summaryCountFieldName;
private List<String> influencers = new ArrayList<>();
private Boolean overlappingBuckets;
private Long resultFinalizationWindow;
private Boolean multivariateByFields;
public Builder(List<Detector> detectors) {
setDetectors(detectors);
}
public Builder(AnalysisConfig analysisConfig) {
this.detectors = new ArrayList<>(analysisConfig.detectors);
this.bucketSpan = analysisConfig.bucketSpan;
this.latency = analysisConfig.latency;
this.categorizationFieldName = analysisConfig.categorizationFieldName;
this.categorizationFilters = analysisConfig.categorizationFilters == null ? null
: new ArrayList<>(analysisConfig.categorizationFilters);
this.categorizationAnalyzerConfig = analysisConfig.categorizationAnalyzerConfig;
this.summaryCountFieldName = analysisConfig.summaryCountFieldName;
this.influencers = new ArrayList<>(analysisConfig.influencers);
this.overlappingBuckets = analysisConfig.overlappingBuckets;
this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow;
this.multivariateByFields = analysisConfig.multivariateByFields;
}
public void setDetectors(List<Detector> detectors) {
Objects.requireNonNull(detectors, "[" + DETECTORS.getPreferredName() + "] must not be null");
// We always assign sequential IDs to the detectors that are correct for this analysis config
int detectorIndex = 0;
List<Detector> sequentialIndexDetectors = new ArrayList<>(detectors.size());
for (Detector origDetector : detectors) {
Detector.Builder builder = new Detector.Builder(origDetector);
builder.setDetectorIndex(detectorIndex++);
sequentialIndexDetectors.add(builder.build());
}
this.detectors = sequentialIndexDetectors;
}
public void setDetector(int detectorIndex, Detector detector) {
detectors.set(detectorIndex, detector);
}
public void setBucketSpan(TimeValue bucketSpan) {
this.bucketSpan = bucketSpan;
}
public void setLatency(TimeValue latency) {
this.latency = latency;
}
public void setCategorizationFieldName(String categorizationFieldName) {
this.categorizationFieldName = categorizationFieldName;
}
public void setCategorizationFilters(List<String> categorizationFilters) {
this.categorizationFilters = categorizationFilters;
}
public void setCategorizationAnalyzerConfig(CategorizationAnalyzerConfig categorizationAnalyzerConfig) {
this.categorizationAnalyzerConfig = categorizationAnalyzerConfig;
}
public void setSummaryCountFieldName(String summaryCountFieldName) {
this.summaryCountFieldName = summaryCountFieldName;
}
public void setInfluencers(List<String> influencers) {
this.influencers = Objects.requireNonNull(influencers, INFLUENCERS.getPreferredName());
}
public void setOverlappingBuckets(Boolean overlappingBuckets) {
this.overlappingBuckets = overlappingBuckets;
}
public void setResultFinalizationWindow(Long resultFinalizationWindow) {
this.resultFinalizationWindow = resultFinalizationWindow;
}
public void setMultivariateByFields(Boolean multivariateByFields) {
this.multivariateByFields = multivariateByFields;
}
public AnalysisConfig build() {
return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig,
latency, summaryCountFieldName, detectors, influencers, overlappingBuckets,
resultFinalizationWindow, multivariateByFields);
}
}
}

567
x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java
View File

@ -19,7 +19,572 @@
package org.elasticsearch.protocol.xpack.ml.job.config;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
public class Job {
import java.io.IOException;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
* This class represents a configured and created Job. The creation time is set
* to the time the object was constructed and the finished time and last
* data time fields are {@code null} until the job has seen some data or it is
* finished respectively.
*/
public class Job implements ToXContentObject {
public static final String ANOMALY_DETECTOR_JOB_TYPE = "anomaly_detector";
/*
* Field names used in serialization
*/
public static final ParseField ID = new ParseField("job_id");
public static final ParseField JOB_TYPE = new ParseField("job_type");
public static final ParseField GROUPS = new ParseField("groups");
public static final ParseField ANALYSIS_CONFIG = AnalysisConfig.ANALYSIS_CONFIG;
public static final ParseField ANALYSIS_LIMITS = new ParseField("analysis_limits");
public static final ParseField CREATE_TIME = new ParseField("create_time");
public static final ParseField CUSTOM_SETTINGS = new ParseField("custom_settings");
public static final ParseField DATA_DESCRIPTION = new ParseField("data_description");
public static final ParseField DESCRIPTION = new ParseField("description");
public static final ParseField FINISHED_TIME = new ParseField("finished_time");
public static final ParseField LAST_DATA_TIME = new ParseField("last_data_time");
public static final ParseField ESTABLISHED_MODEL_MEMORY = new ParseField("established_model_memory");
public static final ParseField MODEL_PLOT_CONFIG = new ParseField("model_plot_config");
public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days");
public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval");
public static final ParseField MODEL_SNAPSHOT_RETENTION_DAYS = new ParseField("model_snapshot_retention_days");
public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days");
public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id");
public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name");
public static final ParseField DELETED = new ParseField("deleted");
public static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("job_details", true, Builder::new);
static {
PARSER.declareString(Builder::setId, ID);
PARSER.declareString(Builder::setJobType, JOB_TYPE);
PARSER.declareStringArray(Builder::setGroups, GROUPS);
PARSER.declareStringOrNull(Builder::setDescription, DESCRIPTION);
PARSER.declareField(Builder::setCreateTime,
(p) -> TimeUtil.parseTimeField(p, CREATE_TIME.getPreferredName()),
CREATE_TIME,
ValueType.VALUE);
PARSER.declareField(Builder::setFinishedTime,
(p) -> TimeUtil.parseTimeField(p, FINISHED_TIME.getPreferredName()),
FINISHED_TIME,
ValueType.VALUE);
PARSER.declareField(Builder::setLastDataTime,
(p) -> TimeUtil.parseTimeField(p, LAST_DATA_TIME.getPreferredName()),
LAST_DATA_TIME,
ValueType.VALUE);
PARSER.declareLong(Builder::setEstablishedModelMemory, ESTABLISHED_MODEL_MEMORY);
PARSER.declareObject(Builder::setAnalysisConfig, AnalysisConfig.PARSER, ANALYSIS_CONFIG);
PARSER.declareObject(Builder::setAnalysisLimits, AnalysisLimits.PARSER, ANALYSIS_LIMITS);
PARSER.declareObject(Builder::setDataDescription, DataDescription.PARSER, DATA_DESCRIPTION);
PARSER.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.PARSER, MODEL_PLOT_CONFIG);
PARSER.declareLong(Builder::setRenormalizationWindowDays, RENORMALIZATION_WINDOW_DAYS);
PARSER.declareString((builder, val) -> builder.setBackgroundPersistInterval(
TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL);
PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS);
PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS);
PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), CUSTOM_SETTINGS, ValueType.OBJECT);
PARSER.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID);
PARSER.declareString(Builder::setResultsIndexName, RESULTS_INDEX_NAME);
PARSER.declareBoolean(Builder::setDeleted, DELETED);
}
private final String jobId;
private final String jobType;
private final List<String> groups;
private final String description;
private final Date createTime;
private final Date finishedTime;
private final Date lastDataTime;
private final Long establishedModelMemory;
private final AnalysisConfig analysisConfig;
private final AnalysisLimits analysisLimits;
private final DataDescription dataDescription;
private final ModelPlotConfig modelPlotConfig;
private final Long renormalizationWindowDays;
private final TimeValue backgroundPersistInterval;
private final Long modelSnapshotRetentionDays;
private final Long resultsRetentionDays;
private final Map<String, Object> customSettings;
private final String modelSnapshotId;
private final String resultsIndexName;
private final boolean deleted;
private Job(String jobId, String jobType, List<String> groups, String description, Date createTime,
Date finishedTime, Date lastDataTime, Long establishedModelMemory,
AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription,
ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval,
Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map<String, Object> customSettings,
String modelSnapshotId, String resultsIndexName, boolean deleted) {
this.jobId = jobId;
this.jobType = jobType;
this.groups = Collections.unmodifiableList(groups);
this.description = description;
this.createTime = createTime;
this.finishedTime = finishedTime;
this.lastDataTime = lastDataTime;
this.establishedModelMemory = establishedModelMemory;
this.analysisConfig = analysisConfig;
this.analysisLimits = analysisLimits;
this.dataDescription = dataDescription;
this.modelPlotConfig = modelPlotConfig;
this.renormalizationWindowDays = renormalizationWindowDays;
this.backgroundPersistInterval = backgroundPersistInterval;
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
this.resultsRetentionDays = resultsRetentionDays;
this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings);
this.modelSnapshotId = modelSnapshotId;
this.resultsIndexName = resultsIndexName;
this.deleted = deleted;
}
/**
* Return the Job Id.
*
* @return The job Id string
*/
public String getId() {
return jobId;
}
public String getJobType() {
return jobType;
}
public List<String> getGroups() {
return groups;
}
/**
* Private version of getResultsIndexName so that a job can be built from another
* job and pass index name validation
*
* @return The job's index name, minus prefix
*/
private String getResultsIndexNameNoPrefix() {
return resultsIndexName;
}
/**
* The job description
*
* @return job description
*/
public String getDescription() {
return description;
}
/**
* The Job creation time. This name is preferred when serialising to the
* REST API.
*
* @return The date the job was created
*/
public Date getCreateTime() {
return createTime;
}
/**
* The time the job was finished or <code>null</code> if not finished.
*
* @return The date the job was last retired or <code>null</code>
*/
public Date getFinishedTime() {
return finishedTime;
}
/**
* The last time data was uploaded to the job or <code>null</code> if no
* data has been seen.
*
* @return The date at which the last data was processed
*/
public Date getLastDataTime() {
return lastDataTime;
}
/**
* The established model memory of the job, or <code>null</code> if model
* memory has not reached equilibrium yet.
*
* @return The established model memory of the job
*/
public Long getEstablishedModelMemory() {
return establishedModelMemory;
}
/**
* The analysis configuration object
*
* @return The AnalysisConfig
*/
public AnalysisConfig getAnalysisConfig() {
return analysisConfig;
}
/**
* The analysis options object
*
* @return The AnalysisLimits
*/
public AnalysisLimits getAnalysisLimits() {
return analysisLimits;
}
public ModelPlotConfig getModelPlotConfig() {
return modelPlotConfig;
}
/**
* If not set the input data is assumed to be csv with a '_time' field in
* epoch format.
*
* @return A DataDescription or <code>null</code>
* @see DataDescription
*/
public DataDescription getDataDescription() {
return dataDescription;
}
/**
* The duration of the renormalization window in days
*
* @return renormalization window in days
*/
public Long getRenormalizationWindowDays() {
return renormalizationWindowDays;
}
/**
* The background persistence interval
*
* @return background persistence interval
*/
public TimeValue getBackgroundPersistInterval() {
return backgroundPersistInterval;
}
public Long getModelSnapshotRetentionDays() {
return modelSnapshotRetentionDays;
}
public Long getResultsRetentionDays() {
return resultsRetentionDays;
}
public Map<String, Object> getCustomSettings() {
return customSettings;
}
public String getModelSnapshotId() {
return modelSnapshotId;
}
public boolean isDeleted() {
return deleted;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
final String humanReadableSuffix = "_string";
builder.field(ID.getPreferredName(), jobId);
builder.field(JOB_TYPE.getPreferredName(), jobType);
if (groups.isEmpty() == false) {
builder.field(GROUPS.getPreferredName(), groups);
}
if (description != null) {
builder.field(DESCRIPTION.getPreferredName(), description);
}
builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime());
if (finishedTime != null) {
builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix,
finishedTime.getTime());
}
if (lastDataTime != null) {
builder.timeField(LAST_DATA_TIME.getPreferredName(), LAST_DATA_TIME.getPreferredName() + humanReadableSuffix,
lastDataTime.getTime());
}
if (establishedModelMemory != null) {
builder.field(ESTABLISHED_MODEL_MEMORY.getPreferredName(), establishedModelMemory);
}
builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params);
if (analysisLimits != null) {
builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params);
}
if (dataDescription != null) {
builder.field(DATA_DESCRIPTION.getPreferredName(), dataDescription, params);
}
if (modelPlotConfig != null) {
builder.field(MODEL_PLOT_CONFIG.getPreferredName(), modelPlotConfig, params);
}
if (renormalizationWindowDays != null) {
builder.field(RENORMALIZATION_WINDOW_DAYS.getPreferredName(), renormalizationWindowDays);
}
if (backgroundPersistInterval != null) {
builder.field(BACKGROUND_PERSIST_INTERVAL.getPreferredName(), backgroundPersistInterval.getStringRep());
}
if (modelSnapshotRetentionDays != null) {
builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
}
if (resultsRetentionDays != null) {
builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
}
if (customSettings != null) {
builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings);
}
if (modelSnapshotId != null) {
builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId);
}
if (resultsIndexName != null) {
builder.field(RESULTS_INDEX_NAME.getPreferredName(), resultsIndexName);
}
if (params.paramAsBoolean("all", false)) {
builder.field(DELETED.getPreferredName(), deleted);
}
builder.endObject();
return builder;
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (other == null || getClass() != other.getClass()) {
return false;
}
Job that = (Job) other;
return Objects.equals(this.jobId, that.jobId)
&& Objects.equals(this.jobType, that.jobType)
&& Objects.equals(this.groups, that.groups)
&& Objects.equals(this.description, that.description)
&& Objects.equals(this.createTime, that.createTime)
&& Objects.equals(this.finishedTime, that.finishedTime)
&& Objects.equals(this.lastDataTime, that.lastDataTime)
&& Objects.equals(this.establishedModelMemory, that.establishedModelMemory)
&& Objects.equals(this.analysisConfig, that.analysisConfig)
&& Objects.equals(this.analysisLimits, that.analysisLimits)
&& Objects.equals(this.dataDescription, that.dataDescription)
&& Objects.equals(this.modelPlotConfig, that.modelPlotConfig)
&& Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays)
&& Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval)
&& Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
&& Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
&& Objects.equals(this.customSettings, that.customSettings)
&& Objects.equals(this.modelSnapshotId, that.modelSnapshotId)
&& Objects.equals(this.resultsIndexName, that.resultsIndexName)
&& Objects.equals(this.deleted, that.deleted);
}
@Override
public int hashCode() {
return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime, lastDataTime, establishedModelMemory,
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
modelSnapshotId, resultsIndexName, deleted);
}
@Override
public final String toString() {
return Strings.toString(this);
}
public static class Builder {
private String id;
private String jobType = ANOMALY_DETECTOR_JOB_TYPE;
private List<String> groups = Collections.emptyList();
private String description;
private AnalysisConfig analysisConfig;
private AnalysisLimits analysisLimits;
private DataDescription dataDescription;
private Date createTime;
private Date finishedTime;
private Date lastDataTime;
private Long establishedModelMemory;
private ModelPlotConfig modelPlotConfig;
private Long renormalizationWindowDays;
private TimeValue backgroundPersistInterval;
private Long modelSnapshotRetentionDays;
private Long resultsRetentionDays;
private Map<String, Object> customSettings;
private String modelSnapshotId;
private String resultsIndexName;
private boolean deleted;
public Builder() {
}
public Builder(String id) {
this.id = id;
}
public Builder(Job job) {
this.id = job.getId();
this.jobType = job.getJobType();
this.groups = job.getGroups();
this.description = job.getDescription();
this.analysisConfig = job.getAnalysisConfig();
this.analysisLimits = job.getAnalysisLimits();
this.dataDescription = job.getDataDescription();
this.createTime = job.getCreateTime();
this.finishedTime = job.getFinishedTime();
this.lastDataTime = job.getLastDataTime();
this.establishedModelMemory = job.getEstablishedModelMemory();
this.modelPlotConfig = job.getModelPlotConfig();
this.renormalizationWindowDays = job.getRenormalizationWindowDays();
this.backgroundPersistInterval = job.getBackgroundPersistInterval();
this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays();
this.resultsRetentionDays = job.getResultsRetentionDays();
this.customSettings = job.getCustomSettings();
this.modelSnapshotId = job.getModelSnapshotId();
this.resultsIndexName = job.getResultsIndexNameNoPrefix();
this.deleted = job.isDeleted();
}
public Builder setId(String id) {
this.id = id;
return this;
}
public String getId() {
return id;
}
public Builder setJobType(String jobType) {
this.jobType = jobType;
return this;
}
public Builder setGroups(List<String> groups) {
this.groups = groups == null ? Collections.emptyList() : groups;
return this;
}
public Builder setCustomSettings(Map<String, Object> customSettings) {
this.customSettings = customSettings;
return this;
}
public Builder setDescription(String description) {
this.description = description;
return this;
}
public Builder setAnalysisConfig(AnalysisConfig.Builder configBuilder) {
analysisConfig = Objects.requireNonNull(configBuilder, ANALYSIS_CONFIG.getPreferredName()).build();
return this;
}
public Builder setAnalysisLimits(AnalysisLimits analysisLimits) {
this.analysisLimits = Objects.requireNonNull(analysisLimits, ANALYSIS_LIMITS.getPreferredName());
return this;
}
Builder setCreateTime(Date createTime) {
this.createTime = createTime;
return this;
}
Builder setFinishedTime(Date finishedTime) {
this.finishedTime = finishedTime;
return this;
}
/**
* Set the wall clock time of the last data upload
*
* @param lastDataTime Wall clock time
*/
public Builder setLastDataTime(Date lastDataTime) {
this.lastDataTime = lastDataTime;
return this;
}
public Builder setEstablishedModelMemory(Long establishedModelMemory) {
this.establishedModelMemory = establishedModelMemory;
return this;
}
public Builder setDataDescription(DataDescription.Builder description) {
dataDescription = Objects.requireNonNull(description, DATA_DESCRIPTION.getPreferredName()).build();
return this;
}
public Builder setModelPlotConfig(ModelPlotConfig modelPlotConfig) {
this.modelPlotConfig = modelPlotConfig;
return this;
}
public Builder setBackgroundPersistInterval(TimeValue backgroundPersistInterval) {
this.backgroundPersistInterval = backgroundPersistInterval;
return this;
}
public Builder setRenormalizationWindowDays(Long renormalizationWindowDays) {
this.renormalizationWindowDays = renormalizationWindowDays;
return this;
}
public Builder setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) {
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
return this;
}
public Builder setResultsRetentionDays(Long resultsRetentionDays) {
this.resultsRetentionDays = resultsRetentionDays;
return this;
}
public Builder setModelSnapshotId(String modelSnapshotId) {
this.modelSnapshotId = modelSnapshotId;
return this;
}
public Builder setResultsIndexName(String resultsIndexName) {
this.resultsIndexName = resultsIndexName;
return this;
}
public Builder setDeleted(boolean deleted) {
this.deleted = deleted;
return this;
}
/**
* Builds a job.
*
* @return The job
*/
public Job build() {
Objects.requireNonNull(id, "[" + ID.getPreferredName() + "] must not be null");
Objects.requireNonNull(jobType, "[" + JOB_TYPE.getPreferredName() + "] must not be null");
return new Job(
id, jobType, groups, description, createTime, finishedTime, lastDataTime, establishedModelMemory,
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
modelSnapshotId, resultsIndexName, deleted);
}
}
}

1
x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCounts.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.protocol.xpack.ml.job.config.Job;
import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
import java.io.IOException;
import java.util.Date;

1
x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStats.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.protocol.xpack.ml.job.config.Job;
import org.elasticsearch.protocol.xpack.ml.job.results.Result;
import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
import java.io.IOException;
import java.util.Date;

1
x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshot.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.protocol.xpack.ml.job.config.Job;
import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
import java.io.IOException;
import java.util.Date;

6
x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/TimeUtil.java → x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/util/TimeUtil.java
View File

@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml.job.process;
package org.elasticsearch.protocol.xpack.ml.job.util;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.common.xcontent.XContentParser;
@ -25,7 +25,7 @@ import java.io.IOException;
import java.time.format.DateTimeFormatter;
import java.util.Date;
final class TimeUtil {
public final class TimeUtil {
/**
* Parse out a Date object given the current parser and field name.
@ -35,7 +35,7 @@ final class TimeUtil {
* @return parsed Date object
* @throws IOException from XContentParser
*/
static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
return new Date(parser.longValue());
} else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {

268
x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfigTests.java Normal file
View File

@ -0,0 +1,268 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml.job.config;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class AnalysisConfigTests extends AbstractXContentTestCase<AnalysisConfig> {
public static AnalysisConfig.Builder createRandomized() {
boolean isCategorization = randomBoolean();
List<Detector> detectors = new ArrayList<>();
int numDetectors = randomIntBetween(1, 10);
for (int i = 0; i < numDetectors; i++) {
Detector.Builder builder = new Detector.Builder("count", null);
builder.setPartitionFieldName(isCategorization ? "mlcategory" : "part");
detectors.add(builder.build());
}
AnalysisConfig.Builder builder = new AnalysisConfig.Builder(detectors);
if (randomBoolean()) {
TimeValue bucketSpan = TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000));
builder.setBucketSpan(bucketSpan);
}
if (isCategorization) {
builder.setCategorizationFieldName(randomAlphaOfLength(10));
if (randomBoolean()) {
builder.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false)));
} else {
CategorizationAnalyzerConfig.Builder analyzerBuilder = new CategorizationAnalyzerConfig.Builder();
if (rarely()) {
analyzerBuilder.setAnalyzer(randomAlphaOfLength(10));
} else {
if (randomBoolean()) {
for (String pattern : generateRandomStringArray(3, 40, false)) {
Map<String, Object> charFilter = new HashMap<>();
charFilter.put("type", "pattern_replace");
charFilter.put("pattern", pattern);
analyzerBuilder.addCharFilter(charFilter);
}
}
Map<String, Object> tokenizer = new HashMap<>();
tokenizer.put("type", "pattern");
tokenizer.put("pattern", randomAlphaOfLength(10));
analyzerBuilder.setTokenizer(tokenizer);
if (randomBoolean()) {
for (String pattern : generateRandomStringArray(4, 40, false)) {
Map<String, Object> tokenFilter = new HashMap<>();
tokenFilter.put("type", "pattern_replace");
tokenFilter.put("pattern", pattern);
analyzerBuilder.addTokenFilter(tokenFilter);
}
}
}
builder.setCategorizationAnalyzerConfig(analyzerBuilder.build());
}
}
if (randomBoolean()) {
builder.setLatency(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)));
}
if (randomBoolean()) {
builder.setMultivariateByFields(randomBoolean());
}
if (randomBoolean()) {
builder.setOverlappingBuckets(randomBoolean());
}
if (randomBoolean()) {
builder.setResultFinalizationWindow(randomNonNegativeLong());
}
builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false)));
return builder;
}
@Override
protected AnalysisConfig createTestInstance() {
return createRandomized().build();
}
@Override
protected AnalysisConfig doParseInstance(XContentParser parser) {
return AnalysisConfig.PARSER.apply(parser, null).build();
}
@Override
protected boolean supportsUnknownFields() {
return false;
}
public void testBuilder_WithNullDetectors() {
AnalysisConfig.Builder builder = new AnalysisConfig.Builder(new ArrayList<>());
NullPointerException ex = expectThrows(NullPointerException.class, () -> builder.setDetectors(null));
assertEquals("[detectors] must not be null", ex.getMessage());
}
public void testEquals_GivenSameReference() {
AnalysisConfig config = createRandomized().build();
assertTrue(config.equals(config));
}
public void testEquals_GivenDifferentClass() {
assertFalse(createRandomized().build().equals("a string"));
}
public void testEquals_GivenNull() {
assertFalse(createRandomized().build().equals(null));
}
public void testEquals_GivenEqualConfig() {
AnalysisConfig config1 = createValidCategorizationConfig().build();
AnalysisConfig config2 = createValidCategorizationConfig().build();
assertTrue(config1.equals(config2));
assertTrue(config2.equals(config1));
assertEquals(config1.hashCode(), config2.hashCode());
}
public void testEquals_GivenDifferentBucketSpan() {
AnalysisConfig.Builder builder = createConfigBuilder();
builder.setBucketSpan(TimeValue.timeValueSeconds(1800));
AnalysisConfig config1 = builder.build();
builder = createConfigBuilder();
builder.setBucketSpan(TimeValue.timeValueHours(1));
AnalysisConfig config2 = builder.build();
assertFalse(config1.equals(config2));
assertFalse(config2.equals(config1));
}
public void testEquals_GivenCategorizationField() {
AnalysisConfig.Builder builder = createValidCategorizationConfig();
builder.setCategorizationFieldName("foo");
AnalysisConfig config1 = builder.build();
builder = createValidCategorizationConfig();
builder.setCategorizationFieldName("bar");
AnalysisConfig config2 = builder.build();
assertFalse(config1.equals(config2));
assertFalse(config2.equals(config1));
}
public void testEquals_GivenDifferentDetector() {
AnalysisConfig config1 = createConfigWithDetectors(Collections.singletonList(new Detector.Builder("min", "low_count").build()));
AnalysisConfig config2 = createConfigWithDetectors(Collections.singletonList(new Detector.Builder("min", "high_count").build()));
assertFalse(config1.equals(config2));
assertFalse(config2.equals(config1));
}
public void testEquals_GivenDifferentInfluencers() {
AnalysisConfig.Builder builder = createConfigBuilder();
builder.setInfluencers(Collections.singletonList("foo"));
AnalysisConfig config1 = builder.build();
builder = createConfigBuilder();
builder.setInfluencers(Collections.singletonList("bar"));
AnalysisConfig config2 = builder.build();
assertFalse(config1.equals(config2));
assertFalse(config2.equals(config1));
}
public void testEquals_GivenDifferentLatency() {
AnalysisConfig.Builder builder = createConfigBuilder();
builder.setLatency(TimeValue.timeValueSeconds(1800));
AnalysisConfig config1 = builder.build();
builder = createConfigBuilder();
builder.setLatency(TimeValue.timeValueSeconds(1801));
AnalysisConfig config2 = builder.build();
assertFalse(config1.equals(config2));
assertFalse(config2.equals(config1));
}
public void testEquals_GivenSummaryCountField() {
AnalysisConfig.Builder builder = createConfigBuilder();
builder.setSummaryCountFieldName("foo");
AnalysisConfig config1 = builder.build();
builder = createConfigBuilder();
builder.setSummaryCountFieldName("bar");
AnalysisConfig config2 = builder.build();
assertFalse(config1.equals(config2));
assertFalse(config2.equals(config1));
}
public void testEquals_GivenMultivariateByField() {
AnalysisConfig.Builder builder = createConfigBuilder();
builder.setMultivariateByFields(true);
AnalysisConfig config1 = builder.build();
builder = createConfigBuilder();
builder.setMultivariateByFields(false);
AnalysisConfig config2 = builder.build();
assertFalse(config1.equals(config2));
assertFalse(config2.equals(config1));
}
public void testEquals_GivenDifferentCategorizationFilters() {
AnalysisConfig.Builder configBuilder1 = createValidCategorizationConfig();
AnalysisConfig.Builder configBuilder2 = createValidCategorizationConfig();
configBuilder1.setCategorizationFilters(Arrays.asList("foo", "bar"));
configBuilder2.setCategorizationFilters(Arrays.asList("foo", "foobar"));
AnalysisConfig config1 = configBuilder1.build();
AnalysisConfig config2 = configBuilder2.build();
assertFalse(config1.equals(config2));
assertFalse(config2.equals(config1));
}
private static AnalysisConfig createConfigWithDetectors(List<Detector> detectors) {
return new AnalysisConfig.Builder(detectors).build();
}
private static AnalysisConfig.Builder createConfigBuilder() {
return new AnalysisConfig.Builder(Collections.singletonList(new Detector.Builder("min", "count").build()));
}
private static AnalysisConfig.Builder createValidCategorizationConfig() {
Detector.Builder detector = new Detector.Builder("count", null);
detector.setByFieldName("mlcategory");
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
analysisConfig.setBucketSpan(TimeValue.timeValueHours(1));
analysisConfig.setLatency(TimeValue.ZERO);
analysisConfig.setCategorizationFieldName("msg");
return analysisConfig;
}
@Override
protected NamedXContentRegistry xContentRegistry() {
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
return new NamedXContentRegistry(searchModule.getNamedXContents());
}
}

276
x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/JobTests.java Normal file
View File

@ -0,0 +1,276 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml.job.config;
import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class JobTests extends AbstractXContentTestCase<Job> {
private static final String FUTURE_JOB = "{\n" +
" \"job_id\": \"farequote\",\n" +
" \"create_time\": 1234567890000,\n" +
" \"tomorrows_technology_today\": \"wow\",\n" +
" \"analysis_config\": {\n" +
" \"bucket_span\": \"1h\",\n" +
" \"something_new\": \"gasp\",\n" +
" \"detectors\": [{\"function\": \"metric\", \"field_name\": \"responsetime\", \"by_field_name\": \"airline\"}]\n" +
" },\n" +
" \"data_description\": {\n" +
" \"time_field\": \"time\",\n" +
" \"the_future\": 123\n" +
" }\n" +
"}";
@Override
protected Job createTestInstance() {
return createRandomizedJob();
}
@Override
protected Job doParseInstance(XContentParser parser) {
return Job.PARSER.apply(parser, null).build();
}
@Override
protected boolean supportsUnknownFields() {
return false;
}
public void testFutureMetadataParse() throws IOException {
XContentParser parser = XContentFactory.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, FUTURE_JOB);
// The parser should tolerate unknown fields
assertNotNull(Job.PARSER.apply(parser, null).build());
}
public void testEquals_GivenDifferentClass() {
Job job = buildJobBuilder("foo").build();
assertFalse(job.equals("a string"));
}
public void testEquals_GivenDifferentIds() {
Date createTime = new Date();
Job.Builder builder = buildJobBuilder("foo");
builder.setCreateTime(createTime);
Job job1 = builder.build();
builder.setId("bar");
Job job2 = builder.build();
assertFalse(job1.equals(job2));
}
public void testEquals_GivenDifferentRenormalizationWindowDays() {
Date date = new Date();
Job.Builder jobDetails1 = new Job.Builder("foo");
jobDetails1.setDataDescription(new DataDescription.Builder());
jobDetails1.setAnalysisConfig(createAnalysisConfig());
jobDetails1.setRenormalizationWindowDays(3L);
jobDetails1.setCreateTime(date);
Job.Builder jobDetails2 = new Job.Builder("foo");
jobDetails2.setDataDescription(new DataDescription.Builder());
jobDetails2.setRenormalizationWindowDays(4L);
jobDetails2.setAnalysisConfig(createAnalysisConfig());
jobDetails2.setCreateTime(date);
assertFalse(jobDetails1.build().equals(jobDetails2.build()));
}
public void testEquals_GivenDifferentBackgroundPersistInterval() {
Date date = new Date();
Job.Builder jobDetails1 = new Job.Builder("foo");
jobDetails1.setDataDescription(new DataDescription.Builder());
jobDetails1.setAnalysisConfig(createAnalysisConfig());
jobDetails1.setBackgroundPersistInterval(TimeValue.timeValueSeconds(10000L));
jobDetails1.setCreateTime(date);
Job.Builder jobDetails2 = new Job.Builder("foo");
jobDetails2.setDataDescription(new DataDescription.Builder());
jobDetails2.setBackgroundPersistInterval(TimeValue.timeValueSeconds(8000L));
jobDetails2.setAnalysisConfig(createAnalysisConfig());
jobDetails2.setCreateTime(date);
assertFalse(jobDetails1.build().equals(jobDetails2.build()));
}
public void testEquals_GivenDifferentModelSnapshotRetentionDays() {
Date date = new Date();
Job.Builder jobDetails1 = new Job.Builder("foo");
jobDetails1.setDataDescription(new DataDescription.Builder());
jobDetails1.setAnalysisConfig(createAnalysisConfig());
jobDetails1.setModelSnapshotRetentionDays(10L);
jobDetails1.setCreateTime(date);
Job.Builder jobDetails2 = new Job.Builder("foo");
jobDetails2.setDataDescription(new DataDescription.Builder());
jobDetails2.setModelSnapshotRetentionDays(8L);
jobDetails2.setAnalysisConfig(createAnalysisConfig());
jobDetails2.setCreateTime(date);
assertFalse(jobDetails1.build().equals(jobDetails2.build()));
}
public void testEquals_GivenDifferentResultsRetentionDays() {
Date date = new Date();
Job.Builder jobDetails1 = new Job.Builder("foo");
jobDetails1.setDataDescription(new DataDescription.Builder());
jobDetails1.setAnalysisConfig(createAnalysisConfig());
jobDetails1.setCreateTime(date);
jobDetails1.setResultsRetentionDays(30L);
Job.Builder jobDetails2 = new Job.Builder("foo");
jobDetails2.setDataDescription(new DataDescription.Builder());
jobDetails2.setResultsRetentionDays(4L);
jobDetails2.setAnalysisConfig(createAnalysisConfig());
jobDetails2.setCreateTime(date);
assertFalse(jobDetails1.build().equals(jobDetails2.build()));
}
public void testEquals_GivenDifferentCustomSettings() {
Job.Builder jobDetails1 = buildJobBuilder("foo");
Map<String, Object> customSettings1 = new HashMap<>();
customSettings1.put("key1", "value1");
jobDetails1.setCustomSettings(customSettings1);
Job.Builder jobDetails2 = buildJobBuilder("foo");
Map<String, Object> customSettings2 = new HashMap<>();
customSettings2.put("key2", "value2");
jobDetails2.setCustomSettings(customSettings2);
assertFalse(jobDetails1.build().equals(jobDetails2.build()));
}
public void testCopyConstructor() {
for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
Job job = createTestInstance();
Job copy = new Job.Builder(job).build();
assertEquals(job, copy);
}
}
public void testBuilder_WithNullID() {
Job.Builder builder = new Job.Builder("anything").setId(null);
NullPointerException ex = expectThrows(NullPointerException.class, builder::build);
assertEquals("[job_id] must not be null", ex.getMessage());
}
public void testBuilder_WithNullJobType() {
Job.Builder builder = new Job.Builder("anything").setJobType(null);
NullPointerException ex = expectThrows(NullPointerException.class, builder::build);
assertEquals("[job_type] must not be null", ex.getMessage());
}
public static Job.Builder buildJobBuilder(String id, Date date) {
Job.Builder builder = new Job.Builder(id);
builder.setCreateTime(date);
AnalysisConfig.Builder ac = createAnalysisConfig();
DataDescription.Builder dc = new DataDescription.Builder();
builder.setAnalysisConfig(ac);
builder.setDataDescription(dc);
return builder;
}
public static Job.Builder buildJobBuilder(String id) {
return buildJobBuilder(id, new Date());
}
public static String randomValidJobId() {
CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray());
return generator.ofCodePointsLength(random(), 10, 10);
}
public static AnalysisConfig.Builder createAnalysisConfig() {
Detector.Builder d1 = new Detector.Builder("info_content", "domain");
d1.setOverFieldName("client");
Detector.Builder d2 = new Detector.Builder("min", "field");
return new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build()));
}
public static Job createRandomizedJob() {
String jobId = randomValidJobId();
Job.Builder builder = new Job.Builder(jobId);
if (randomBoolean()) {
builder.setDescription(randomAlphaOfLength(10));
}
if (randomBoolean()) {
int groupsNum = randomIntBetween(0, 10);
List<String> groups = new ArrayList<>(groupsNum);
for (int i = 0; i < groupsNum; i++) {
groups.add(randomValidJobId());
}
builder.setGroups(groups);
}
builder.setCreateTime(new Date(randomNonNegativeLong()));
if (randomBoolean()) {
builder.setFinishedTime(new Date(randomNonNegativeLong()));
}
if (randomBoolean()) {
builder.setLastDataTime(new Date(randomNonNegativeLong()));
}
if (randomBoolean()) {
builder.setEstablishedModelMemory(randomNonNegativeLong());
}
builder.setAnalysisConfig(AnalysisConfigTests.createRandomized());
builder.setAnalysisLimits(AnalysisLimitsTests.createRandomized());
DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setFormat(randomFrom(DataDescription.DataFormat.values()));
builder.setDataDescription(dataDescription);
if (randomBoolean()) {
builder.setModelPlotConfig(new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10)));
}
if (randomBoolean()) {
builder.setRenormalizationWindowDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setBackgroundPersistInterval(TimeValue.timeValueHours(randomIntBetween(1, 24)));
}
if (randomBoolean()) {
builder.setModelSnapshotRetentionDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setResultsRetentionDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10)));
}
if (randomBoolean()) {
builder.setModelSnapshotId(randomAlphaOfLength(10));
}
if (randomBoolean()) {
builder.setResultsIndexName(randomValidJobId());
}
return builder.build();
}
@Override
protected NamedXContentRegistry xContentRegistry() {
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
return new NamedXContentRegistry(searchModule.getNamedXContents());
}
}