From b938960602a559cf9e1133779cc9d7e2b0ec401a Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 2 Aug 2018 12:33:25 -0700 Subject: [PATCH 01/11] Painless: Move Some Lookup Logic to PainlessLookup (#32565) Renames existing methods in PainlessLookup. Adds lookupPainlessClass, lookupPainlessMethod, and lookupPainlessField to PainlessLookup. This consolidates the logic necessary to look these things up into a single place and begins the clean up of some of the nodes that were looking each of these things up individually. This also has the added benefit of improved consistency in error messaging. --- .../org/elasticsearch/painless/Compiler.java | 2 +- .../java/org/elasticsearch/painless/Def.java | 18 +-- .../elasticsearch/painless/FunctionRef.java | 12 +- .../painless/PainlessExplainError.java | 2 +- .../painless/ScriptClassInfo.java | 2 +- .../painless/antlr/EnhancedPainlessLexer.java | 2 +- .../painless/lookup/PainlessLookup.java | 116 +++++++++++++++--- .../painless/node/EExplicit.java | 2 +- .../painless/node/EFunctionRef.java | 2 +- .../painless/node/EInstanceof.java | 2 +- .../elasticsearch/painless/node/ELambda.java | 2 +- .../painless/node/EListInit.java | 19 ++- .../elasticsearch/painless/node/EMapInit.java | 19 ++- .../painless/node/ENewArray.java | 2 +- .../elasticsearch/painless/node/ENewObj.java | 53 ++++---- .../elasticsearch/painless/node/EStatic.java | 2 +- .../painless/node/PCallInvoke.java | 28 ++--- .../elasticsearch/painless/node/PField.java | 49 ++++---- .../painless/node/PSubListShortcut.java | 10 +- .../painless/node/PSubMapShortcut.java | 10 +- .../elasticsearch/painless/node/SCatch.java | 2 +- .../painless/node/SDeclaration.java | 2 +- .../elasticsearch/painless/node/SEach.java | 2 +- .../painless/node/SFunction.java | 4 +- .../painless/node/SSubEachIterable.java | 10 +- .../elasticsearch/painless/OverloadTests.java | 6 +- .../painless/PainlessDocGenerator.java | 7 +- .../elasticsearch/painless/RegexTests.java | 2 +- .../painless/node/NodeToStringTests.java | 6 +- 29 files changed, 230 insertions(+), 165 deletions(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java index 807b4409d7a..2096d7d7970 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java @@ -96,7 +96,7 @@ final class Compiler { if (statefulFactoryClass != null && statefulFactoryClass.getName().equals(name)) { return statefulFactoryClass; } - Class found = painlessLookup.getClassFromBinaryName(name); + Class found = painlessLookup.canonicalTypeNameToType(name.replace('$', '.')); return found != null ? found : super.findClass(name); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java index 91c25b7cfec..10806b64d0e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java @@ -187,7 +187,7 @@ public final class Def { String key = PainlessLookupUtility.buildPainlessMethodKey(name, arity); // check whitelist for matching method for (Class clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) { - PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz); + PainlessClass struct = painlessLookup.lookupPainlessClass(clazz); if (struct != null) { PainlessMethod method = struct.methods.get(key); @@ -197,7 +197,7 @@ public final class Def { } for (Class iface : clazz.getInterfaces()) { - struct = painlessLookup.getPainlessStructFromJavaClass(iface); + struct = painlessLookup.lookupPainlessClass(iface); if (struct != null) { PainlessMethod method = struct.methods.get(key); @@ -326,8 +326,8 @@ public final class Def { */ static MethodHandle lookupReference(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup, String interfaceClass, Class receiverClass, String name) throws Throwable { - Class interfaceType = painlessLookup.getJavaClassFromPainlessType(interfaceClass); - PainlessMethod interfaceMethod = painlessLookup.getPainlessStructFromJavaClass(interfaceType).functionalMethod; + Class interfaceType = painlessLookup.canonicalTypeNameToType(interfaceClass); + PainlessMethod interfaceMethod = painlessLookup.lookupPainlessClass(interfaceType).functionalMethod; if (interfaceMethod == null) { throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface"); } @@ -345,7 +345,7 @@ public final class Def { final FunctionRef ref; if ("this".equals(type)) { // user written method - PainlessMethod interfaceMethod = painlessLookup.getPainlessStructFromJavaClass(clazz).functionalMethod; + PainlessMethod interfaceMethod = painlessLookup.lookupPainlessClass(clazz).functionalMethod; if (interfaceMethod == null) { throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + "to [" + PainlessLookupUtility.typeToCanonicalTypeName(clazz) + "], not a functional interface"); @@ -419,7 +419,7 @@ public final class Def { static MethodHandle lookupGetter(PainlessLookup painlessLookup, Class receiverClass, String name) { // first try whitelist for (Class clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) { - PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz); + PainlessClass struct = painlessLookup.lookupPainlessClass(clazz); if (struct != null) { MethodHandle handle = struct.getterMethodHandles.get(name); @@ -429,7 +429,7 @@ public final class Def { } for (final Class iface : clazz.getInterfaces()) { - struct = painlessLookup.getPainlessStructFromJavaClass(iface); + struct = painlessLookup.lookupPainlessClass(iface); if (struct != null) { MethodHandle handle = struct.getterMethodHandles.get(name); @@ -490,7 +490,7 @@ public final class Def { static MethodHandle lookupSetter(PainlessLookup painlessLookup, Class receiverClass, String name) { // first try whitelist for (Class clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) { - PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz); + PainlessClass struct = painlessLookup.lookupPainlessClass(clazz); if (struct != null) { MethodHandle handle = struct.setterMethodHandles.get(name); @@ -500,7 +500,7 @@ public final class Def { } for (final Class iface : clazz.getInterfaces()) { - struct = painlessLookup.getPainlessStructFromJavaClass(iface); + struct = painlessLookup.lookupPainlessClass(iface); if (struct != null) { MethodHandle handle = struct.setterMethodHandles.get(name); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java index d4671f05b6c..cc558489446 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java @@ -90,10 +90,10 @@ public class FunctionRef { PainlessLookup painlessLookup, Class expected, String type, String call, int numCaptures) { if ("new".equals(call)) { - return new FunctionRef(expected, painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod, + return new FunctionRef(expected, painlessLookup.lookupPainlessClass(expected).functionalMethod, lookup(painlessLookup, expected, type), numCaptures); } else { - return new FunctionRef(expected, painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod, + return new FunctionRef(expected, painlessLookup.lookupPainlessClass(expected).functionalMethod, lookup(painlessLookup, expected, type, call, numCaptures > 0), numCaptures); } } @@ -230,14 +230,14 @@ public class FunctionRef { private static PainlessConstructor lookup(PainlessLookup painlessLookup, Class expected, String type) { // check its really a functional interface // for e.g. Comparable - PainlessMethod method = painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod; + PainlessMethod method = painlessLookup.lookupPainlessClass(expected).functionalMethod; if (method == null) { throw new IllegalArgumentException("Cannot convert function reference [" + type + "::new] " + "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"); } // lookup requested constructor - PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(painlessLookup.getJavaClassFromPainlessType(type)); + PainlessClass struct = painlessLookup.lookupPainlessClass(painlessLookup.canonicalTypeNameToType(type)); PainlessConstructor impl = struct.constructors.get(PainlessLookupUtility.buildPainlessConstructorKey(method.typeParameters.size())); if (impl == null) { @@ -254,14 +254,14 @@ public class FunctionRef { String type, String call, boolean receiverCaptured) { // check its really a functional interface // for e.g. Comparable - PainlessMethod method = painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod; + PainlessMethod method = painlessLookup.lookupPainlessClass(expected).functionalMethod; if (method == null) { throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"); } // lookup requested method - PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(painlessLookup.getJavaClassFromPainlessType(type)); + PainlessClass struct = painlessLookup.lookupPainlessClass(painlessLookup.canonicalTypeNameToType(type)); final PainlessMethod impl; // look for a static impl first PainlessMethod staticImpl = diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java index 7bef028c7d1..e4988103bc6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java @@ -57,7 +57,7 @@ public class PainlessExplainError extends Error { if (objectToExplain != null) { toString = objectToExplain.toString(); javaClassName = objectToExplain.getClass().getName(); - PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(objectToExplain.getClass()); + PainlessClass struct = painlessLookup.lookupPainlessClass(objectToExplain.getClass()); if (struct != null) { painlessClassName = PainlessLookupUtility.typeToCanonicalTypeName(objectToExplain.getClass()); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java index 6d4b4552696..345db46f887 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java @@ -190,7 +190,7 @@ public class ScriptClassInfo { componentType = componentType.getComponentType(); } - if (painlessLookup.getPainlessStructFromJavaClass(componentType) == null) { + if (painlessLookup.lookupPainlessClass(componentType) == null) { throw new IllegalArgumentException(unknownErrorMessageSource.apply(componentType)); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java index f1db35636b4..9279093cf31 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java @@ -75,7 +75,7 @@ final class EnhancedPainlessLexer extends PainlessLexer { @Override protected boolean isType(String name) { - return painlessLookup.isSimplePainlessType(name); + return painlessLookup.isValidCanonicalClassName(name); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java index 67c04498a58..786248f7269 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java @@ -19,41 +19,119 @@ package org.elasticsearch.painless.lookup; -import java.util.Collection; import java.util.Collections; import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessConstructorKey; +import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessFieldKey; +import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey; +import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName; -/** - * The entire API for Painless. Also used as a whitelist for checking for legal - * methods and fields during at both compile-time and runtime. - */ public final class PainlessLookup { - public Collection> getStructs() { - return classesToPainlessClasses.keySet(); - } - private final Map> canonicalClassNamesToClasses; private final Map, PainlessClass> classesToPainlessClasses; PainlessLookup(Map> canonicalClassNamesToClasses, Map, PainlessClass> classesToPainlessClasses) { + Objects.requireNonNull(canonicalClassNamesToClasses); + Objects.requireNonNull(classesToPainlessClasses); + this.canonicalClassNamesToClasses = Collections.unmodifiableMap(canonicalClassNamesToClasses); this.classesToPainlessClasses = Collections.unmodifiableMap(classesToPainlessClasses); } - public Class getClassFromBinaryName(String painlessType) { - return canonicalClassNamesToClasses.get(painlessType.replace('$', '.')); + public boolean isValidCanonicalClassName(String canonicalClassName) { + Objects.requireNonNull(canonicalClassName); + + return canonicalClassNamesToClasses.containsKey(canonicalClassName); } - public boolean isSimplePainlessType(String painlessType) { - return canonicalClassNamesToClasses.containsKey(painlessType); - } + public Class canonicalTypeNameToType(String painlessType) { + Objects.requireNonNull(painlessType); - public PainlessClass getPainlessStructFromJavaClass(Class clazz) { - return classesToPainlessClasses.get(clazz); - } - - public Class getJavaClassFromPainlessType(String painlessType) { return PainlessLookupUtility.canonicalTypeNameToType(painlessType, canonicalClassNamesToClasses); } + + public Set> getClasses() { + return classesToPainlessClasses.keySet(); + } + + public PainlessClass lookupPainlessClass(Class targetClass) { + return classesToPainlessClasses.get(targetClass); + } + + public PainlessConstructor lookupPainlessConstructor(Class targetClass, int constructorArity) { + Objects.requireNonNull(targetClass); + + PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetClass); + String painlessConstructorKey = buildPainlessConstructorKey(constructorArity); + + if (targetPainlessClass == null) { + throw new IllegalArgumentException("target class [" + typeToCanonicalTypeName(targetClass) + "] " + + "not found for constructor [" + painlessConstructorKey + "]"); + } + + PainlessConstructor painlessConstructor = targetPainlessClass.constructors.get(painlessConstructorKey); + + if (painlessConstructor == null) { + throw new IllegalArgumentException( + "constructor [" + typeToCanonicalTypeName(targetClass) + ", " + painlessConstructorKey + "] not found"); + } + + return painlessConstructor; + } + + public PainlessMethod lookupPainlessMethod(Class targetClass, boolean isStatic, String methodName, int methodArity) { + Objects.requireNonNull(targetClass); + Objects.requireNonNull(methodName); + + if (targetClass.isPrimitive()) { + targetClass = PainlessLookupUtility.typeToBoxedType(targetClass); + } + + PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetClass); + String painlessMethodKey = buildPainlessMethodKey(methodName, methodArity); + + if (targetPainlessClass == null) { + throw new IllegalArgumentException( + "target class [" + typeToCanonicalTypeName(targetClass) + "] not found for method [" + painlessMethodKey + "]"); + } + + PainlessMethod painlessMethod = isStatic ? + targetPainlessClass.staticMethods.get(painlessMethodKey) : + targetPainlessClass.methods.get(painlessMethodKey); + + if (painlessMethod == null) { + throw new IllegalArgumentException( + "method [" + typeToCanonicalTypeName(targetClass) + ", " + painlessMethodKey + "] not found"); + } + + return painlessMethod; + } + + public PainlessField lookupPainlessField(Class targetClass, boolean isStatic, String fieldName) { + Objects.requireNonNull(targetClass); + Objects.requireNonNull(fieldName); + + PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetClass); + String painlessFieldKey = buildPainlessFieldKey(fieldName); + + if (targetPainlessClass == null) { + throw new IllegalArgumentException( + "target class [" + typeToCanonicalTypeName(targetClass) + "] not found for field [" + painlessFieldKey + "]"); + } + + PainlessField painlessField = isStatic ? + targetPainlessClass.staticFields.get(painlessFieldKey) : + targetPainlessClass.fields.get(painlessFieldKey); + + if (painlessField == null) { + throw new IllegalArgumentException( + "field [" + typeToCanonicalTypeName(targetClass) + ", " + painlessFieldKey + "] not found"); + } + + return painlessField; + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java index d19068f8fa6..c58d51e45cb 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java @@ -50,7 +50,7 @@ public final class EExplicit extends AExpression { @Override void analyze(Locals locals) { try { - actual = locals.getPainlessLookup().getJavaClassFromPainlessType(type); + actual = locals.getPainlessLookup().canonicalTypeNameToType(type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java index 782991e2958..ead2e0c5f70 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java @@ -66,7 +66,7 @@ public final class EFunctionRef extends AExpression implements ILambda { try { if ("this".equals(type)) { // user's own function - PainlessMethod interfaceMethod = locals.getPainlessLookup().getPainlessStructFromJavaClass(expected).functionalMethod; + PainlessMethod interfaceMethod = locals.getPainlessLookup().lookupPainlessClass(expected).functionalMethod; if (interfaceMethod == null) { throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java index 2fa8ca8ca95..8585b7fc0bb 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java @@ -58,7 +58,7 @@ public final class EInstanceof extends AExpression { // ensure the specified type is part of the definition try { - clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); + clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java index 6fc4a3a6480..e84ab006501 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java @@ -121,7 +121,7 @@ public final class ELambda extends AExpression implements ILambda { } } else { // we know the method statically, infer return type and any unknown/def types - interfaceMethod = locals.getPainlessLookup().getPainlessStructFromJavaClass(expected).functionalMethod; + interfaceMethod = locals.getPainlessLookup().lookupPainlessClass(expected).functionalMethod; if (interfaceMethod == null) { throw createError(new IllegalArgumentException("Cannot pass lambda to " + "[" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java index 01a4878266e..bd931558b62 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java @@ -24,7 +24,6 @@ import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; import org.elasticsearch.painless.lookup.PainlessConstructor; -import org.elasticsearch.painless.lookup.PainlessLookupUtility; import org.elasticsearch.painless.lookup.PainlessMethod; import org.elasticsearch.painless.lookup.def; import org.objectweb.asm.Type; @@ -64,18 +63,16 @@ public final class EListInit extends AExpression { actual = ArrayList.class; - constructor = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors.get( - PainlessLookupUtility.buildPainlessConstructorKey(0)); - - if (constructor == null) { - throw createError(new IllegalStateException("Illegal tree structure.")); + try { + constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, 0); + } catch (IllegalArgumentException iae) { + throw createError(iae); } - method = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).methods - .get(PainlessLookupUtility.buildPainlessMethodKey("add", 1)); - - if (method == null) { - throw createError(new IllegalStateException("Illegal tree structure.")); + try { + method = locals.getPainlessLookup().lookupPainlessMethod(actual, false, "add", 1); + } catch (IllegalArgumentException iae) { + throw createError(iae); } for (int index = 0; index < values.size(); ++index) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java index 73afe7f0dc5..91332672c05 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java @@ -24,7 +24,6 @@ import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; import org.elasticsearch.painless.lookup.PainlessConstructor; -import org.elasticsearch.painless.lookup.PainlessLookupUtility; import org.elasticsearch.painless.lookup.PainlessMethod; import org.elasticsearch.painless.lookup.def; import org.objectweb.asm.Type; @@ -70,18 +69,16 @@ public final class EMapInit extends AExpression { actual = HashMap.class; - constructor = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors.get( - PainlessLookupUtility.buildPainlessConstructorKey(0)); - - if (constructor == null) { - throw createError(new IllegalStateException("Illegal tree structure.")); + try { + constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, 0); + } catch (IllegalArgumentException iae) { + throw createError(iae); } - method = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).methods - .get(PainlessLookupUtility.buildPainlessMethodKey("put", 2)); - - if (method == null) { - throw createError(new IllegalStateException("Illegal tree structure.")); + try { + method = locals.getPainlessLookup().lookupPainlessMethod(actual, false, "put", 2); + } catch (IllegalArgumentException iae) { + throw createError(iae); } if (keys.size() != values.size()) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java index f9bd4cebc3f..e0a49ebd615 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java @@ -61,7 +61,7 @@ public final class ENewArray extends AExpression { Class clazz; try { - clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); + clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java index 4e08f257386..55ba60feb3e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java @@ -23,7 +23,6 @@ import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; -import org.elasticsearch.painless.lookup.PainlessClass; import org.elasticsearch.painless.lookup.PainlessConstructor; import org.elasticsearch.painless.lookup.PainlessLookupUtility; import org.objectweb.asm.Type; @@ -60,38 +59,36 @@ public final class ENewObj extends AExpression { @Override void analyze(Locals locals) { try { - actual = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); + actual = locals.getPainlessLookup().canonicalTypeNameToType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } - PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual); - constructor = struct.constructors.get(PainlessLookupUtility.buildPainlessConstructorKey(arguments.size())); - - if (constructor != null) { - Class[] types = new Class[constructor.typeParameters.size()]; - constructor.typeParameters.toArray(types); - - if (constructor.typeParameters.size() != arguments.size()) { - throw createError(new IllegalArgumentException( - "When calling constructor on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "] " + - "expected [" + constructor.typeParameters.size() + "] arguments, but found [" + arguments.size() + "].")); - } - - for (int argument = 0; argument < arguments.size(); ++argument) { - AExpression expression = arguments.get(argument); - - expression.expected = types[argument]; - expression.internal = true; - expression.analyze(locals); - arguments.set(argument, expression.cast(locals)); - } - - statement = true; - } else { - throw createError(new IllegalArgumentException( - "Unknown new call on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "].")); + try { + constructor = locals.getPainlessLookup().lookupPainlessConstructor(actual, arguments.size()); + } catch (IllegalArgumentException iae) { + throw createError(iae); } + + Class[] types = new Class[constructor.typeParameters.size()]; + constructor.typeParameters.toArray(types); + + if (constructor.typeParameters.size() != arguments.size()) { + throw createError(new IllegalArgumentException( + "When calling constructor on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "] " + + "expected [" + constructor.typeParameters.size() + "] arguments, but found [" + arguments.size() + "].")); + } + + for (int argument = 0; argument < arguments.size(); ++argument) { + AExpression expression = arguments.get(argument); + + expression.expected = types[argument]; + expression.internal = true; + expression.analyze(locals); + arguments.set(argument, expression.cast(locals)); + } + + statement = true; } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java index a556b3ad315..e5909d93e9d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java @@ -48,7 +48,7 @@ public final class EStatic extends AExpression { @Override void analyze(Locals locals) { try { - actual = locals.getPainlessLookup().getJavaClassFromPainlessType(type); + actual = locals.getPainlessLookup().canonicalTypeNameToType(type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java index 56bc18eadbd..9406b4ca411 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java @@ -23,8 +23,6 @@ import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; -import org.elasticsearch.painless.lookup.PainlessClass; -import org.elasticsearch.painless.lookup.PainlessLookupUtility; import org.elasticsearch.painless.lookup.PainlessMethod; import org.elasticsearch.painless.lookup.def; @@ -66,26 +64,16 @@ public final class PCallInvoke extends AExpression { prefix.expected = prefix.actual; prefix = prefix.cast(locals); - if (prefix.actual.isArray()) { - throw createError(new IllegalArgumentException("Illegal call [" + name + "] on array type.")); - } - - PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual); - - if (prefix.actual.isPrimitive()) { - struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(PainlessLookupUtility.typeToBoxedType(prefix.actual)); - } - - String methodKey = PainlessLookupUtility.buildPainlessMethodKey(name, arguments.size()); - PainlessMethod method = prefix instanceof EStatic ? struct.staticMethods.get(methodKey) : struct.methods.get(methodKey); - - if (method != null) { - sub = new PSubCallInvoke(location, method, prefix.actual, arguments); - } else if (prefix.actual == def.class) { + if (prefix.actual == def.class) { sub = new PSubDefCall(location, name, arguments); } else { - throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + "] arguments " + - "on type [" + PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual) + "].")); + try { + PainlessMethod method = + locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, prefix instanceof EStatic, name, arguments.size()); + sub = new PSubCallInvoke(location, method, prefix.actual, arguments); + } catch (IllegalArgumentException iae) { + throw createError(iae); + } } if (nullSafe) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java index b322d5b1f28..59cbfd405b7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java @@ -23,8 +23,6 @@ import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; -import org.elasticsearch.painless.lookup.PainlessClass; -import org.elasticsearch.painless.lookup.PainlessField; import org.elasticsearch.painless.lookup.PainlessLookupUtility; import org.elasticsearch.painless.lookup.PainlessMethod; import org.elasticsearch.painless.lookup.def; @@ -67,26 +65,34 @@ public final class PField extends AStoreable { } else if (prefix.actual == def.class) { sub = new PSubDefField(location, value); } else { - PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual); - PainlessField field = prefix instanceof EStatic ? struct.staticFields.get(value) : struct.fields.get(value); + try { + sub = new PSubField(location, + locals.getPainlessLookup().lookupPainlessField(prefix.actual, prefix instanceof EStatic, value)); + } catch (IllegalArgumentException fieldIAE) { + PainlessMethod getter; + PainlessMethod setter; - if (field != null) { - sub = new PSubField(location, field); - } else { - PainlessMethod getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey( - "get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)); - - if (getter == null) { - getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey( - "is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)); + try { + getter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false, + "get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0); + } catch (IllegalArgumentException getIAE) { + try { + getter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false, + "is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0); + } catch (IllegalArgumentException isIAE) { + getter = null; + } } - PainlessMethod setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey( - "set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1)); + try { + setter = locals.getPainlessLookup().lookupPainlessMethod(prefix.actual, false, + "set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0); + } catch (IllegalArgumentException setIAE) { + setter = null; + } if (getter != null || setter != null) { - sub = new PSubShortcut( - location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), getter, setter); + sub = new PSubShortcut(location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), getter, setter); } else { EConstant index = new EConstant(location, value); index.analyze(locals); @@ -99,12 +105,11 @@ public final class PField extends AStoreable { sub = new PSubListShortcut(location, prefix.actual, index); } } - } - } - if (sub == null) { - throw createError(new IllegalArgumentException( - "Unknown field [" + value + "] for type [" + PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual) + "].")); + if (sub == null) { + throw createError(fieldIAE); + } + } } if (nullSafe) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java index 0738f55c2cf..838756fcc67 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java @@ -24,7 +24,6 @@ import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; import org.elasticsearch.painless.WriterConstants; -import org.elasticsearch.painless.lookup.PainlessClass; import org.elasticsearch.painless.lookup.PainlessLookupUtility; import org.elasticsearch.painless.lookup.PainlessMethod; @@ -56,11 +55,14 @@ final class PSubListShortcut extends AStoreable { @Override void analyze(Locals locals) { - PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(targetClass); String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass); - getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("get", 1)); - setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("set", 2)); + try { + getter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1); + setter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "set", 2); + } catch (IllegalArgumentException iae) { + throw createError(iae); + } if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1 || getter.typeParameters.get(0) != int.class)) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java index 04ccbc9f534..27a3f69775a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java @@ -23,7 +23,6 @@ import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; -import org.elasticsearch.painless.lookup.PainlessClass; import org.elasticsearch.painless.lookup.PainlessLookupUtility; import org.elasticsearch.painless.lookup.PainlessMethod; @@ -55,11 +54,14 @@ final class PSubMapShortcut extends AStoreable { @Override void analyze(Locals locals) { - PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(targetClass); String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass); - getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("get", 1)); - setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("put", 2)); + try { + getter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1); + setter = locals.getPainlessLookup().lookupPainlessMethod(targetClass, false, "put", 2); + } catch (IllegalArgumentException iae) { + throw createError(iae); + } if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1)) { throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + canonicalClassName + "].")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java index 8a703c80cba..04b0462b533 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java @@ -67,7 +67,7 @@ public final class SCatch extends AStatement { Class clazz; try { - clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); + clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java index fb92c20e89e..f3774885cfd 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java @@ -62,7 +62,7 @@ public final class SDeclaration extends AStatement { Class clazz; try { - clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); + clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java index 9ff57e6b913..a83f501df32 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java @@ -71,7 +71,7 @@ public class SEach extends AStatement { Class clazz; try { - clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); + clazz = locals.getPainlessLookup().canonicalTypeNameToType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java index d61a424f83d..8230b543697 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java @@ -121,7 +121,7 @@ public final class SFunction extends AStatement { void generateSignature(PainlessLookup painlessLookup) { try { - returnType = painlessLookup.getJavaClassFromPainlessType(rtnTypeStr); + returnType = painlessLookup.canonicalTypeNameToType(rtnTypeStr); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Illegal return type [" + rtnTypeStr + "] for function [" + name + "].")); } @@ -135,7 +135,7 @@ public final class SFunction extends AStatement { for (int param = 0; param < this.paramTypeStrs.size(); ++param) { try { - Class paramType = painlessLookup.getJavaClassFromPainlessType(this.paramTypeStrs.get(param)); + Class paramType = painlessLookup.canonicalTypeNameToType(this.paramTypeStrs.get(param)); paramClasses[param] = PainlessLookupUtility.typeToJavaType(paramType); paramTypes.add(paramType); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java index 5450f690f6c..577d1d51d09 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java @@ -76,12 +76,10 @@ final class SSubEachIterable extends AStatement { if (expression.actual == def.class) { method = null; } else { - method = locals.getPainlessLookup().getPainlessStructFromJavaClass(expression.actual).methods - .get(PainlessLookupUtility.buildPainlessMethodKey("iterator", 0)); - - if (method == null) { - throw createError(new IllegalArgumentException("Unable to create iterator for the type " + - "[" + PainlessLookupUtility.typeToCanonicalTypeName(expression.actual) + "].")); + try { + method = locals.getPainlessLookup().lookupPainlessMethod(expression.actual, false, "iterator", 0); + } catch (IllegalArgumentException iae) { + throw createError(iae); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java index fce827e686c..1b90d582999 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java @@ -23,12 +23,12 @@ package org.elasticsearch.painless; public class OverloadTests extends ScriptTestCase { public void testMethod() { - assertEquals(2, exec("return 'abc123abc'.indexOf('c');")); - assertEquals(8, exec("return 'abc123abc'.indexOf('c', 3);")); + //assertEquals(2, exec("return 'abc123abc'.indexOf('c');")); + //assertEquals(8, exec("return 'abc123abc'.indexOf('c', 3);")); IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("return 'abc123abc'.indexOf('c', 3, 'bogus');"); }); - assertTrue(expected.getMessage().contains("[indexOf] with [3] arguments")); + assertTrue(expected.getMessage().contains("[java.lang.String, indexOf/3]")); } public void testMethodDynamic() { diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java index ad29d702177..1460d5f2359 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java @@ -45,9 +45,9 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.function.Consumer; +import java.util.stream.Collectors; import static java.util.Comparator.comparing; -import static java.util.stream.Collectors.toList; /** * Generates an API reference from the method and type whitelists in {@link PainlessLookup}. @@ -74,9 +74,10 @@ public class PainlessDocGenerator { Files.newOutputStream(indexPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE), false, StandardCharsets.UTF_8.name())) { emitGeneratedWarning(indexStream); - List> classes = PAINLESS_LOOKUP.getStructs().stream().sorted(comparing(Class::getCanonicalName)).collect(toList()); + List> classes = PAINLESS_LOOKUP.getClasses().stream().sorted( + Comparator.comparing(Class::getCanonicalName)).collect(Collectors.toList()); for (Class clazz : classes) { - PainlessClass struct = PAINLESS_LOOKUP.getPainlessStructFromJavaClass(clazz); + PainlessClass struct = PAINLESS_LOOKUP.lookupPainlessClass(clazz); String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(clazz); if (clazz.isPrimitive()) { diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java index 911a50468cc..8143c39ce6f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java @@ -252,7 +252,7 @@ public class RegexTests extends ScriptTestCase { IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> { exec("Pattern.compile('aa')"); }); - assertEquals("Unknown call [compile] with [1] arguments on type [java.util.regex.Pattern].", e.getMessage()); + assertTrue(e.getMessage().contains("[java.util.regex.Pattern, compile/1]")); } public void testBadRegexPattern() { diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java index c64014d81a5..f6ad38f997e 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java @@ -404,7 +404,7 @@ public class NodeToStringTests extends ESTestCase { public void testPSubCallInvoke() { Location l = new Location(getTestName(), 0); - PainlessClass c = painlessLookup.getPainlessStructFromJavaClass(Integer.class); + PainlessClass c = painlessLookup.lookupPainlessClass(Integer.class); PainlessMethod m = c.methods.get(PainlessLookupUtility.buildPainlessMethodKey("toString", 0)); PSubCallInvoke node = new PSubCallInvoke(l, m, null, emptyList()); node.prefix = new EVariable(l, "a"); @@ -459,7 +459,7 @@ public class NodeToStringTests extends ESTestCase { public void testPSubField() { Location l = new Location(getTestName(), 0); - PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(Boolean.class); + PainlessClass s = painlessLookup.lookupPainlessClass(Boolean.class); PainlessField f = s.staticFields.get("TRUE"); PSubField node = new PSubField(l, f); node.prefix = new EStatic(l, "Boolean"); @@ -497,7 +497,7 @@ public class NodeToStringTests extends ESTestCase { public void testPSubShortcut() { Location l = new Location(getTestName(), 0); - PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(FeatureTest.class); + PainlessClass s = painlessLookup.lookupPainlessClass(FeatureTest.class); PainlessMethod getter = s.methods.get(PainlessLookupUtility.buildPainlessMethodKey("getX", 0)); PainlessMethod setter = s.methods.get(PainlessLookupUtility.buildPainlessMethodKey("setX", 1)); PSubShortcut node = new PSubShortcut(l, "x", FeatureTest.class.getName(), getter, setter); From 9e1e38ff51e3ace950f6e1c3450554bbf20d6484 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 2 Aug 2018 23:46:08 +0300 Subject: [PATCH 02/11] Minor fix for javadoc (applicable for java 11). (#32573) --- .../expression/function/scalar/string/StringFunctionUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java index 585dd3f2f4c..1f38456cba1 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java @@ -33,7 +33,7 @@ abstract class StringFunctionUtils { } /** - * Trims the trailing whitespace characters from the given String. Uses java.lang.Character.isWhitespace(char) + * Trims the trailing whitespace characters from the given String. Uses {@link Character#isWhitespace(char)} * to determine if a character is whitespace or not. * * @param s the original String From 080b9f58ea757a194053d09f49e3d0c9898cbe18 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Thu, 2 Aug 2018 17:16:34 -0400 Subject: [PATCH 03/11] [TEST] Test for shard failures, add debug to testProfileMatchesRegular Unmuting the test and adding some more debug output. Was not able to reproduce the prior failure, but it seems possible that the failure (mismatched counts) could be caused by partial search results during the test. The assertions check for shard failures first, because if one of the two searches is partial the rest of the test will fail. Next, instead of just checking respective hit counts, we emit the difference in hits to help identify what went wrong. Closes #32492 --- .../search/profile/query/QueryProfilerIT.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java index 4f725ed2f11..da7f5f8a4cf 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java +++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java @@ -35,8 +35,10 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.search.profile.query.RandomQueryGenerator.randomQueryBuilder; import static org.hamcrest.Matchers.equalTo; @@ -105,7 +107,6 @@ public class QueryProfilerIT extends ESIntegTestCase { * search for each query. It then does some basic sanity checking of score and hits * to make sure the profiling doesn't interfere with the hits being returned */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32492") public void testProfileMatchesRegular() throws Exception { createIndex("test"); ensureGreen(); @@ -150,6 +151,10 @@ public class QueryProfilerIT extends ESIntegTestCase { SearchResponse vanillaResponse = responses[0].getResponse(); SearchResponse profileResponse = responses[1].getResponse(); + assertThat(vanillaResponse.getFailedShards(), equalTo(0)); + assertThat(profileResponse.getFailedShards(), equalTo(0)); + assertThat(vanillaResponse.getSuccessfulShards(), equalTo(profileResponse.getSuccessfulShards())); + float vanillaMaxScore = vanillaResponse.getHits().getMaxScore(); float profileMaxScore = profileResponse.getHits().getMaxScore(); if (Float.isNaN(vanillaMaxScore)) { @@ -160,10 +165,19 @@ public class QueryProfilerIT extends ESIntegTestCase { vanillaMaxScore, profileMaxScore, 0.001); } - assertThat( - "Profile totalHits of [" + profileResponse.getHits().getTotalHits() + "] is not close to Vanilla totalHits [" - + vanillaResponse.getHits().getTotalHits() + "]", - vanillaResponse.getHits().getTotalHits(), equalTo(profileResponse.getHits().getTotalHits())); + if (vanillaResponse.getHits().totalHits != profileResponse.getHits().totalHits) { + Set vanillaSet = new HashSet<>(Arrays.asList(vanillaResponse.getHits().getHits())); + Set profileSet = new HashSet<>(Arrays.asList(profileResponse.getHits().getHits())); + if (vanillaResponse.getHits().totalHits > profileResponse.getHits().totalHits) { + vanillaSet.removeAll(profileSet); + fail("Vanilla hits were larger than profile hits. Non-overlapping elements were: " + + vanillaSet.toString()); + } else { + profileSet.removeAll(vanillaSet); + fail("Profile hits were larger than vanilla hits. Non-overlapping elements were: " + + profileSet.toString()); + } + } SearchHit[] vanillaHits = vanillaResponse.getHits().getHits(); SearchHit[] profileHits = profileResponse.getHits().getHits(); From eb3accb72118f2c85eacd6f598754fcd2e9a1f07 Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Thu, 2 Aug 2018 15:21:34 -0600 Subject: [PATCH 04/11] Security: move User to protocol project (#32367) The User class has been moved to the protocol project for upcoming work to add more security APIs to the high level rest client. As part of this change, the toString method no longer uses a custom output method from MetadataUtils and instead just relies on Java's toString implementation. --- .../xpack/core/security/SecurityContext.java | 2 +- .../xpack/core/security/UserSettings.java | 2 +- .../action/user/AuthenticateResponse.java | 2 +- .../user/ChangePasswordRequestBuilder.java | 2 +- .../action/user/GetUsersResponse.java | 2 +- .../action/user/PutUserRequestBuilder.java | 2 +- .../core/security/authc/Authentication.java | 2 +- .../security/authc/AuthenticationResult.java | 2 +- .../xpack/core/security/authc/Realm.java | 2 +- .../core/security/authz/RoleDescriptor.java | 3 +- .../SecurityIndexSearcherWrapper.java | 2 +- .../core/security/support/MetadataUtils.java | 46 ------------------- .../core/security/user/AnonymousUser.java | 1 + .../core/security/user/BeatsSystemUser.java | 1 + .../xpack/core/security/user/ElasticUser.java | 1 + .../user/InternalUserSerializationHelper.java | 1 + .../xpack/core/security/user/KibanaUser.java | 1 + .../security/user/LogstashSystemUser.java | 1 + .../xpack/core/security/user/SystemUser.java | 1 + .../core/security/user/XPackSecurityUser.java | 2 + .../xpack/core/security/user/XPackUser.java | 1 + ...SecurityIndexSearcherWrapperUnitTests.java | 4 +- .../saml/TransportSamlLogoutAction.java | 2 +- .../user/TransportAuthenticateAction.java | 2 +- .../action/user/TransportGetUsersAction.java | 2 +- .../user/TransportHasPrivilegesAction.java | 2 +- .../xpack/security/audit/AuditTrail.java | 2 +- .../security/audit/AuditTrailService.java | 2 +- .../security/audit/index/IndexAuditTrail.java | 2 +- .../audit/logfile/LoggingAuditTrail.java | 2 +- .../security/authc/AuthenticationService.java | 2 +- .../security/authc/esnative/NativeRealm.java | 2 +- .../authc/esnative/NativeUsersStore.java | 4 +- .../authc/esnative/ReservedRealm.java | 2 +- .../authc/esnative/UserAndPassword.java | 2 +- .../xpack/security/authc/file/FileRealm.java | 2 +- .../authc/file/FileUserPasswdStore.java | 2 +- .../authc/kerberos/KerberosRealm.java | 4 +- .../xpack/security/authc/ldap/LdapRealm.java | 2 +- .../xpack/security/authc/pki/PkiRealm.java | 2 +- .../xpack/security/authc/saml/SamlRealm.java | 2 +- .../support/CachingUsernamePasswordRealm.java | 2 +- .../security/authz/AuthorizationService.java | 2 +- .../security/authz/AuthorizedIndices.java | 2 +- .../ingest/SetSecurityUserProcessor.java | 2 +- .../rest/action/RestAuthenticateAction.java | 2 +- .../action/user/RestChangePasswordAction.java | 2 +- .../rest/action/user/RestGetUsersAction.java | 2 +- .../transport/ServerTransportFilter.java | 2 +- .../integration/ClearRealmsCacheTests.java | 2 +- .../xpack/security/SecurityContextTests.java | 2 +- .../filter/SecurityActionFilterTests.java | 2 +- ...IndicesAliasesRequestInterceptorTests.java | 2 +- .../ResizeRequestInterceptorTests.java | 2 +- ...sportSamlInvalidateSessionActionTests.java | 2 +- .../saml/TransportSamlLogoutActionTests.java | 2 +- .../TransportAuthenticateActionTests.java | 2 +- .../TransportChangePasswordActionTests.java | 2 +- .../user/TransportDeleteUserActionTests.java | 2 +- .../user/TransportGetUsersActionTests.java | 2 +- .../TransportHasPrivilegesActionTests.java | 2 +- .../user/TransportPutUserActionTests.java | 2 +- .../user/TransportSetEnabledActionTests.java | 2 +- .../audit/AuditTrailServiceTests.java | 2 +- .../index/IndexAuditTrailMutedTests.java | 2 +- .../audit/index/IndexAuditTrailTests.java | 2 +- .../logfile/LoggingAuditTrailFilterTests.java | 2 +- .../audit/logfile/LoggingAuditTrailTests.java | 2 +- .../authc/AuthenticationServiceTests.java | 2 +- .../xpack/security/authc/RealmsTests.java | 2 +- .../security/authc/TokenServiceTests.java | 2 +- .../xpack/security/authc/UserTokenTests.java | 2 +- .../authc/esnative/NativeRealmIntegTests.java | 2 +- .../authc/esnative/NativeUsersStoreTests.java | 2 +- .../authc/esnative/ReservedRealmTests.java | 2 +- .../security/authc/file/FileRealmTests.java | 2 +- .../authc/file/FileUserPasswdStoreTests.java | 2 +- .../KerberosRealmAuthenticateFailedTests.java | 2 +- .../kerberos/KerberosRealmCacheTests.java | 2 +- .../authc/kerberos/KerberosRealmTestCase.java | 2 +- .../authc/kerberos/KerberosRealmTests.java | 4 +- .../authc/ldap/ActiveDirectoryRealmTests.java | 2 +- .../ldap/CancellableLdapRunnableTests.java | 2 +- .../security/authc/ldap/LdapRealmTests.java | 2 +- .../security/authc/pki/PkiRealmTests.java | 2 +- .../CachingUsernamePasswordRealmTests.java | 2 +- .../mapper/NativeRoleMappingStoreTests.java | 2 +- .../authz/AuthorizationServiceTests.java | 2 +- .../authz/AuthorizationUtilsTests.java | 2 +- .../authz/AuthorizedIndicesTests.java | 2 +- .../authz/IndicesAndAliasesResolverTests.java | 2 +- .../SecuritySearchOperationListenerTests.java | 2 +- .../ingest/SetSecurityUserProcessorTests.java | 2 +- ...curityServerTransportInterceptorTests.java | 2 +- .../transport/ServerTransportFilterTests.java | 2 +- .../security/user/AnonymousUserTests.java | 2 +- ...Tests.java => UserSerializationTests.java} | 15 +----- .../execution/ExecutionServiceTests.java | 2 +- .../protocol/xpack/security}/User.java | 25 +++++++--- .../protocol/xpack/security/UserTests.java | 39 ++++++++++++++++ .../example/realm/CustomRealm.java | 2 +- .../example/realm/CustomRealmTests.java | 2 +- .../xpack/security/MigrateToolIT.java | 2 +- 103 files changed, 164 insertions(+), 160 deletions(-) rename x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/{UserTests.java => UserSerializationTests.java} (89%) rename x-pack/{plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user => protocol/src/main/java/org/elasticsearch/protocol/xpack/security}/User.java (91%) create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/security/UserTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java index 99788ac1de4..8d56221d78b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; import org.elasticsearch.node.Node; import org.elasticsearch.xpack.core.security.authc.Authentication; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.io.IOException; import java.util.Objects; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/UserSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/UserSettings.java index 7f22f90351e..536464cb337 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/UserSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/UserSettings.java @@ -10,7 +10,7 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.xpack.core.security.authc.Authentication; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateResponse.java index 0cf7ace1103..c3e126b92ce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateResponse.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.core.security.action.user; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java index d7538c2a556..05b3af41de2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java @@ -18,7 +18,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.support.Validation; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.xcontent.XContentUtils; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java index 666b79cfe5d..0da525b6ffc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.core.security.action.user; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.io.IOException; import java.util.Collection; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java index eea804d81fe..7dc958bbef9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java @@ -20,7 +20,7 @@ import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.support.Validation; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.xcontent.XContentUtils; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java index 161d9d44999..e72df007cf6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.xpack.core.security.user.InternalUserSerializationHelper; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.io.IOException; import java.util.Base64; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java index 0f073ef4ae3..08e01025e7e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java @@ -6,7 +6,7 @@ package org.elasticsearch.xpack.core.security.authc; import org.elasticsearch.common.Nullable; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.util.Objects; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java index 2c63ca95eb9..ae04d474f41 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java @@ -9,7 +9,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.xpack.core.XPackField; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.util.Collections; import java.util.HashMap; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java index 42bd771103f..54fd8cc7974 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; import org.elasticsearch.xpack.core.security.support.Validation; import org.elasticsearch.xpack.core.security.xcontent.XContentUtils; @@ -163,7 +162,7 @@ public class RoleDescriptor implements ToXContentObject { } sb.append("], runAs=[").append(Strings.arrayToCommaDelimitedString(runAs)); sb.append("], metadata=["); - MetadataUtils.writeValue(sb, metadata); + sb.append(metadata); sb.append("]]"); return sb.toString(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java index f945b4e24c7..70b552b123e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java @@ -62,7 +62,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader; import org.elasticsearch.xpack.core.security.support.Exceptions; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.io.IOException; import java.util.ArrayList; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java index ff457cb8d06..a48f562c7af 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java @@ -5,8 +5,6 @@ */ package org.elasticsearch.xpack.core.security.support; -import java.lang.reflect.Array; -import java.util.Collection; import java.util.Collections; import java.util.Map; @@ -19,50 +17,6 @@ public class MetadataUtils { private MetadataUtils() { } - public static void writeValue(StringBuilder sb, Object object) { - if (object == null) { - sb.append(object); - } else if (object instanceof Map) { - sb.append("{"); - for (Map.Entry entry : ((Map) object).entrySet()) { - sb.append(entry.getKey()).append("="); - writeValue(sb, entry.getValue()); - } - sb.append("}"); - - } else if (object instanceof Collection) { - sb.append("["); - boolean first = true; - for (Object item : (Collection) object) { - if (!first) { - sb.append(","); - } - writeValue(sb, item); - first = false; - } - sb.append("]"); - } else if (object.getClass().isArray()) { - sb.append("["); - for (int i = 0; i < Array.getLength(object); i++) { - if (i != 0) { - sb.append(","); - } - writeValue(sb, Array.get(object, i)); - } - sb.append("]"); - } else { - sb.append(object); - } - } - - public static void verifyNoReservedMetadata(Map metadata) { - for (String key : metadata.keySet()) { - if (key.startsWith(RESERVED_PREFIX)) { - throw new IllegalArgumentException("invalid user metadata. [" + key + "] is a reserved for internal use"); - } - } - } - public static boolean containsReservedMetadata(Map metadata) { for (String key : metadata.keySet()) { if (key.startsWith(RESERVED_PREFIX)) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java index 36354ff58b3..ae6f41c3a1b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.support.MetadataUtils; import java.util.Collections; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java index dfa437fa8d2..9db64da97a5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.security.user; import org.elasticsearch.Version; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.support.MetadataUtils; /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java index ec618a4f482..c58f86ea422 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.security.user; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.support.MetadataUtils; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUserSerializationHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUserSerializationHelper.java index fa41828a7bb..c0b45aea57c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUserSerializationHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUserSerializationHelper.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.security.user; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.protocol.xpack.security.User; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java index 8dfa149987d..3e816aa54bc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.security.user; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.support.MetadataUtils; /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java index ce37d742a19..047758177fb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.security.user; import org.elasticsearch.Version; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.support.MetadataUtils; /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java index 4569c2a68a0..1c7ac129d17 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.security.user; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.authz.privilege.SystemPrivilege; import java.util.function.Predicate; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java index 906d3548377..e98df7fb50a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.core.security.user; +import org.elasticsearch.protocol.xpack.security.User; + /** * internal user that manages xpack security. Has all cluster/indices permissions. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java index 38c9fe84aa9..fe50b1b9c88 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.security.user; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.index.IndexAuditTrailField; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java index d02d7e5fe97..c26968ce54a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java @@ -76,7 +76,7 @@ import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.junit.After; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -442,7 +442,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase { return "rendered_text"; } }; - + when(scriptService.compile(any(Script.class), eq(TemplateScript.CONTEXT))).thenReturn(compiledTemplate); XContentBuilder builder = jsonBuilder(); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java index 63931d119e0..3e489c69d1d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.security.action.saml.SamlLogoutRequest; import org.elasticsearch.xpack.core.security.action.saml.SamlLogoutResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Realm; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.TokenService; import org.elasticsearch.xpack.security.authc.saml.SamlNameId; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java index 57510ce116f..af56ab8d4eb 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java @@ -18,7 +18,7 @@ import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction; import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest; import org.elasticsearch.xpack.core.security.action.user.AuthenticateResponse; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import java.util.function.Supplier; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java index 7e17cda75f0..f89745d23e3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java @@ -18,7 +18,7 @@ import org.elasticsearch.xpack.core.security.action.user.GetUsersRequest; import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse; import org.elasticsearch.xpack.core.security.authc.esnative.ClientReservedRealm; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java index b49984b28da..eefaaa72b1e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java @@ -30,7 +30,7 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.Privilege; import org.elasticsearch.xpack.core.security.support.Automatons; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authz.AuthorizationService; import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java index 3f19d281925..8dcaf1a61ff 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java @@ -9,7 +9,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.transport.TransportMessage; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; import java.net.InetAddress; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java index 3cd12b1a7ce..a9245c7653e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java @@ -12,7 +12,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.transport.TransportMessage; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; import java.net.InetAddress; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java index d8b4b4e4bc1..6d12455fdea 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java @@ -55,7 +55,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.index.IndexAuditTrailField; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.core.template.TemplateUtils; import org.elasticsearch.xpack.security.audit.AuditLevel; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 5da6a9eb77c..3a6cfa50193 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -30,7 +30,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.support.Automatons; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.audit.AuditLevel; import org.elasticsearch.xpack.security.audit.AuditTrail; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java index 85084da8464..416acfca3ab 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java @@ -31,7 +31,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.support.Exceptions; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.audit.AuditTrailService; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java index a84b76beab8..ffd9c3f73bc 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java @@ -11,7 +11,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; import org.elasticsearch.xpack.security.support.SecurityIndexManager; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index d923a029804..507ed4684a1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -48,8 +48,8 @@ import org.elasticsearch.xpack.core.security.authc.esnative.ClientReservedRealm; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.core.security.user.User.Fields; +import org.elasticsearch.protocol.xpack.security.User; +import org.elasticsearch.protocol.xpack.security.User.Fields; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.support.SecurityIndexManager; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java index 31de03ff433..99c138bbb12 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java @@ -29,7 +29,7 @@ import org.elasticsearch.xpack.core.security.user.BeatsSystemUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore.ReservedUserInfo; import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; import org.elasticsearch.xpack.security.support.SecurityIndexManager; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java index 3f636312f0f..d9971ab2388 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.security.authc.esnative; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.xpack.core.security.authc.support.Hasher; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; /** * Like User, but includes the hashed password diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java index e2586ea836d..8d529897534 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java @@ -12,7 +12,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; import java.util.Map; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java index 15a6c2c41da..220108b5637 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.support.NoOpLogger; import org.elasticsearch.xpack.core.security.support.Validation; import org.elasticsearch.xpack.core.security.support.Validation.Users; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.support.SecurityFiles; import java.io.IOException; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java index 71eeb8b2398..dc38f1f78c0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.authc.Realm; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.support.CachingRealm; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; @@ -222,4 +222,4 @@ public final class KerberosRealm extends Realm implements CachingRealm { public void lookupUser(final String username, final ActionListener listener) { listener.onResponse(null); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java index 87749850141..f689bc28789 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java @@ -25,7 +25,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings; import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings; import org.elasticsearch.xpack.core.security.authc.ldap.LdapSessionFactorySettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.ldap.support.LdapLoadBalancing; import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java index 7b9eabfd706..58e10a54755 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java @@ -26,7 +26,7 @@ import org.elasticsearch.xpack.core.security.authc.Realm; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.RealmSettings; import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.ssl.CertParsingUtils; import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings; import org.elasticsearch.xpack.security.authc.BytesKey; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java index cc160c8f78b..a8f50d975e8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java @@ -43,7 +43,7 @@ import org.elasticsearch.xpack.core.security.authc.Realm; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.RealmSettings; import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.ssl.SSLConfiguration; import org.elasticsearch.xpack.core.ssl.CertParsingUtils; import org.elasticsearch.xpack.core.ssl.SSLService; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java index 68338e9bcad..bcdbc1e1dd3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java @@ -20,7 +20,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.util.Collections; import java.util.Map; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 642bc167f7d..2c9bb8ce3c0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -57,7 +57,7 @@ import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.core.security.support.Automatons; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.audit.AuditTrailService; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java index 3068a3993d3..07845a131b7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.security.authz; import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.xpack.core.security.authz.permission.Role; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.ArrayList; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java index 15ac88b4d94..6db19d8edeb 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java @@ -10,7 +10,7 @@ import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; import org.elasticsearch.xpack.core.security.authc.Authentication; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.util.Arrays; import java.util.EnumSet; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java index b280b3a89a2..10b8e65c168 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java @@ -20,7 +20,7 @@ import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction; import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest; import org.elasticsearch.xpack.core.security.action.user.AuthenticateResponse; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.io.IOException; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java index 1b64b3ce2ba..7e33844e99b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java @@ -21,7 +21,7 @@ import org.elasticsearch.xpack.core.security.action.user.ChangePasswordResponse; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.core.security.rest.RestRequestFilter; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler; import java.io.IOException; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java index 1ab80954e9b..3e8cf26ad7d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java @@ -18,7 +18,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestBuilderListener; import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler; import java.io.IOException; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java index 2f0c40c1fdd..761af81b08e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java @@ -27,7 +27,7 @@ import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.action.SecurityActionMapper; import org.elasticsearch.xpack.security.authc.AuthenticationService; import org.elasticsearch.xpack.security.authz.AuthorizationService; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java index fc02a5c4d62..0b6321e5960 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.Realm; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.Realms; import org.junit.BeforeClass; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java index e3b1cd31246..2b9e540f3bf 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java @@ -15,7 +15,7 @@ import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java index 019901afa28..1ac5490dc0c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java @@ -33,7 +33,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.AuthenticationService; import org.elasticsearch.xpack.security.authz.AuthorizationService; import org.junit.Before; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java index 7c951c0014e..0809276932d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition; import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.audit.AuditTrailService; import java.util.Collections; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java index f1363214b07..f939b175e48 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition; import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.audit.AuditTrailService; import java.util.Collections; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index 158a74308d9..3371b901647 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -56,7 +56,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings; import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.TokenService; import org.elasticsearch.xpack.security.authc.UserToken; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java index 37cbb5ef279..1ce8b1aff13 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java @@ -46,7 +46,7 @@ import org.elasticsearch.xpack.core.security.action.saml.SamlLogoutResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.TokenService; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java index a8e24648058..7862097d000 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.security.action.user.AuthenticateResponse; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import java.util.Collections; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java index aabaa40381f..410c164ffe7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.mockito.invocation.InvocationOnMock; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserActionTests.java index 4e6e0b3551b..0c1ddbd9ba7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserActionTests.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.mockito.invocation.InvocationOnMock; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java index 1c5f93187c0..ebdb1455591 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.xpack.core.security.action.user.GetUsersRequest; import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java index a2e283e1b36..d7795d3ab91 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java @@ -34,7 +34,7 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivileg import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authz.AuthorizationService; import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore; import org.hamcrest.Matchers; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java index fff2479aa5d..b7eeb78fad3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java index d811b6359b1..4ca7ab97f73 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.mockito.invocation.InvocationOnMock; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java index b346fc6857e..ba9a67eb48f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.transport.TransportMessage; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; import org.junit.Before; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java index 9bc5c989d1f..923c918f011 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail.State; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; import org.junit.After; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java index cb1b69708bd..dcb8d8b7569 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java @@ -55,7 +55,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.index.IndexAuditTrailField; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.LocalStateSecurity; import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail.Message; import org.elasticsearch.xpack.security.support.SecurityIndexManager; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java index 4c9df8fd9d3..a3a9d05704f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java @@ -28,7 +28,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.AuditEventMetaInfo; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrailTests.MockMessage; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrailTests.RestContent; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 1059e22abd6..c8e14e668c9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -35,7 +35,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.rest.RemoteHostHeader; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index fda87d0340b..4a40e0d543b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -64,7 +64,7 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.elasticsearch.xpack.security.authc.AuthenticationService.Authenticator; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java index 9d795826298..1da7d68c91c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings; import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings; import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; import org.junit.Before; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index d5e67f3996a..c529ea8747b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -48,7 +48,7 @@ import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.TokenMetaData; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.watcher.watch.ClockMock; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.AfterClass; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/UserTokenTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/UserTokenTests.java index 1a8f8dc3b5d..c79d77718ca 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/UserTokenTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/UserTokenTests.java @@ -10,7 +10,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import java.io.IOException; import java.time.Clock; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java index b7cd23745b9..7f9e25e77e6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java @@ -45,7 +45,7 @@ import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; import org.junit.Before; import org.junit.BeforeClass; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java index d8af70cb6be..c7a7c4f07bb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java @@ -30,7 +30,7 @@ import org.elasticsearch.xpack.core.security.user.BeatsSystemUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.Before; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java index fad3d43e6d5..a03f30abc33 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.xpack.core.security.user.BeatsSystemUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.UsernamesField; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore.ReservedUserInfo; import org.elasticsearch.xpack.security.support.SecurityIndexManager; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java index f5dad8b7c68..aec2a4eb820 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java @@ -17,7 +17,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.junit.Before; import org.mockito.stubbing.Answer; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java index 739952af63e..ee89c2efe4f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.security.audit.logfile.CapturingLogger; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.support.Hasher; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.junit.After; import org.junit.Before; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java index 5bc239241cf..11aefb758fb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.ietf.jgss.GSSException; import java.nio.file.Path; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java index 69ebe15c5d7..7ce8ee39c03 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java @@ -11,7 +11,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper.UserData; import org.ietf.jgss.GSSException; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java index 9c2c6484c82..69b246cd7ca 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java @@ -21,7 +21,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings; import org.elasticsearch.xpack.core.security.support.Exceptions; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import org.elasticsearch.xpack.security.support.SecurityIndexManager; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java index 0f972498ab3..0f44544c9c2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java @@ -18,7 +18,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper.UserData; import org.ietf.jgss.GSSException; @@ -148,4 +148,4 @@ public class KerberosRealmTests extends KerberosRealmTestCase { () -> new KerberosRealm(config, mockNativeRoleMappingStore, mockKerberosTicketValidator, threadPool, null)); assertThat(iae.getMessage(), is(equalTo(expectedErrorMessage))); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java index 2c6756aada7..a8f555bc3a3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java @@ -35,7 +35,7 @@ import org.elasticsearch.xpack.core.security.authc.ldap.support.SessionFactorySe import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.DnRoleMapperSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.core.ssl.VerificationMode; import org.elasticsearch.xpack.security.authc.ldap.ActiveDirectorySessionFactory.DownLevelADAuthenticator; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/CancellableLdapRunnableTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/CancellableLdapRunnableTests.java index 18b84df6d61..2807c501a5d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/CancellableLdapRunnableTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/CancellableLdapRunnableTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.security.authc.ldap; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.ldap.LdapRealm.CancellableLdapRunnable; import java.util.concurrent.CountDownLatch; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java index 4aff821217d..5c98e2347cf 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java @@ -27,7 +27,7 @@ import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope; import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.DnRoleMapperSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.core.ssl.VerificationMode; import org.elasticsearch.xpack.security.authc.ldap.support.LdapTestCase; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java index 44d5859d12b..2410d8c4649 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java @@ -21,7 +21,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings; import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.support.NoOpLogger; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; import org.junit.Before; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java index 052758d8371..e6830be18c5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.junit.After; import org.junit.Before; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 052ba385510..2bee8fa09e3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -25,7 +25,7 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression.FieldValue; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.security.support.SecurityIndexManager; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index 7722a9d2166..65c558d3d81 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -122,7 +122,7 @@ import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java index 9c9f2b1b1a4..a581d1abbb5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java @@ -15,7 +15,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.junit.Before; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java index c48ac456898..d31f9f37b91 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCa import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; import org.elasticsearch.xpack.security.support.SecurityIndexManager; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index eed3297661a..cf9c09759ea 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -60,7 +60,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCa import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.core.security.user.AnonymousUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.audit.AuditTrailService; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java index fac88e8af09..087749da240 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.transport.TransportRequest.Empty; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.audit.AuditTrailService; import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java index 26c59a1ef54..05c2882f3ac 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java @@ -11,7 +11,7 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor.Property; import java.util.Collections; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java index dd7dda48ae8..09072f99fc2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java @@ -33,7 +33,7 @@ import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.AuthenticationService; import org.elasticsearch.xpack.security.authz.AuthorizationService; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java index 38b9a029d9b..08a991eb3ec 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java @@ -29,7 +29,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.authc.AuthenticationService; import org.elasticsearch.xpack.security.authz.AuthorizationService; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserTests.java index 4c72afeb5ce..32816e40e08 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserTests.java @@ -9,7 +9,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.user.AnonymousUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.equalTo; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserSerializationTests.java similarity index 89% rename from x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserTests.java rename to x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserSerializationTests.java index 5be4b1c0eca..6bea620982f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/UserSerializationTests.java @@ -13,18 +13,17 @@ import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.InternalUserSerializationHelper; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.security.user.XPackUser; import java.util.Arrays; -import java.util.Collections; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; -public class UserTests extends ESTestCase { +public class UserSerializationTests extends ESTestCase { public void testWriteToAndReadFrom() throws Exception { User user = new User(randomAlphaOfLengthBetween(4, 30), @@ -142,16 +141,6 @@ public class UserTests extends ESTestCase { } } - public void testUserToString() throws Exception { - User user = new User("u1", "r1"); - assertThat(user.toString(), is("User[username=u1,roles=[r1],fullName=null,email=null,metadata={}]")); - user = new User("u1", new String[] { "r1", "r2" }, "user1", "user1@domain.com", Collections.singletonMap("key", "val"), true); - assertThat(user.toString(), is("User[username=u1,roles=[r1,r2],fullName=user1,email=user1@domain.com,metadata={key=val}]")); - user = new User("u1", new String[] {"r1"}, new User("u2", "r2", "r3")); - assertThat(user.toString(), is("User[username=u1,roles=[r1],fullName=null,email=null,metadata={}," + - "authenticatedUser=[User[username=u2,roles=[r2,r3],fullName=null,email=null,metadata={}]]]")); - } - public void testReservedUserSerialization() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); final ElasticUser elasticUser = new ElasticUser(true); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java index d3f46d3d452..bb593bcb67a 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java @@ -33,7 +33,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.core.watcher.actions.Action; import org.elasticsearch.xpack.core.watcher.actions.ActionStatus; import org.elasticsearch.xpack.core.watcher.actions.ActionWrapper; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/User.java similarity index 91% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/User.java index e8161b9a7e2..42e957ecf2d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/User.java @@ -1,9 +1,23 @@ /* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.elasticsearch.xpack.core.security.user; + +package org.elasticsearch.protocol.xpack.security; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; @@ -13,7 +27,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; import java.io.IOException; import java.util.Arrays; @@ -128,7 +141,7 @@ public class User implements ToXContentObject { sb.append(",fullName=").append(fullName); sb.append(",email=").append(email); sb.append(",metadata="); - MetadataUtils.writeValue(sb, metadata); + sb.append(metadata); if (authenticatedUser != null) { sb.append(",authenticatedUser=[").append(authenticatedUser.toString()).append("]"); } diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/security/UserTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/security/UserTests.java new file mode 100644 index 00000000000..2e3c67131df --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/security/UserTests.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.protocol.xpack.security; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; + +import static org.hamcrest.Matchers.is; + +public class UserTests extends ESTestCase { + + public void testUserToString() { + User user = new User("u1", "r1"); + assertThat(user.toString(), is("User[username=u1,roles=[r1],fullName=null,email=null,metadata={}]")); + user = new User("u1", new String[] { "r1", "r2" }, "user1", "user1@domain.com", Collections.singletonMap("key", "val"), true); + assertThat(user.toString(), is("User[username=u1,roles=[r1,r2],fullName=user1,email=user1@domain.com,metadata={key=val}]")); + user = new User("u1", new String[] {"r1"}, new User("u2", "r2", "r3")); + assertThat(user.toString(), is("User[username=u1,roles=[r1],fullName=null,email=null,metadata={}," + + "authenticatedUser=[User[username=u2,roles=[r2,r3],fullName=null,email=null,metadata={}]]]")); + } +} diff --git a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java index 19ef9d2eb0d..af3fb160e13 100644 --- a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java +++ b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRealm.java @@ -14,7 +14,7 @@ import org.elasticsearch.xpack.core.security.authc.Realm; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.support.CharArrays; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; public class CustomRealm extends Realm { diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java index d1435ebaa3c..e206de6e392 100644 --- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java +++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; diff --git a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java b/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java index 4ac927c6646..e810e638f68 100644 --- a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java +++ b/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java @@ -25,7 +25,7 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.protocol.xpack.security.User; import org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool; import org.junit.Before; From 0a83968650d93144cabec755baf5ec1d93754aba Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 2 Aug 2018 17:14:19 -0700 Subject: [PATCH 05/11] Add cluster UUID to Cluster Stats API response (#32206) * Make cluster stats response contain cluster UUID * Updating constructor usage in Monitoring tests * Adding cluster_uuid field to Cluster Stats API reference doc * Adding rest api spec test for expecting cluster_uuid in cluster stats response * Adding missing newline * Indenting do section properly * Missed a spot! * Fixing the test cluster ID --- docs/reference/cluster/stats.asciidoc | 1 + .../test/cluster.stats/10_basic.yml | 37 +++++++++++++++++++ .../cluster/stats/ClusterStatsResponse.java | 8 ++++ .../stats/TransportClusterStatsAction.java | 1 + .../ClusterStatsMonitoringDocTests.java | 2 + 5 files changed, 49 insertions(+) diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 191da2660d6..3de85041871 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -22,6 +22,7 @@ Will return, for example: "successful" : 1, "failed" : 0 }, + "cluster_uuid": "YjAvIhsCQ9CbjWZb2qJw3Q", "cluster_name": "elasticsearch", "timestamp": 1459427693515, "status": "green", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml index 29f048068b4..0ff5708b93e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml @@ -29,3 +29,40 @@ - is_true: nodes.fs - is_true: nodes.plugins - is_true: nodes.network_types + +--- +"get cluster stats returns cluster_uuid at the top level": + - skip: + version: " - 6.99.99" + reason: "cluster stats including cluster_uuid at the top level is new in v6.5.0 and higher" + + - do: + cluster.stats: {} + + - is_true: cluster_uuid + - is_true: timestamp + - is_true: cluster_name + - match: {status: green} + - gte: { indices.count: 0} + - is_true: indices.docs + - is_true: indices.store + - is_true: indices.fielddata + - is_true: indices.query_cache + - is_true: indices.completion + - is_true: indices.segments + - gte: { nodes.count.total: 1} + - gte: { nodes.count.master: 1} + - gte: { nodes.count.data: 1} + - gte: { nodes.count.ingest: 0} + - gte: { nodes.count.coordinating_only: 0} + - is_true: nodes.os + - is_true: nodes.os.mem.total_in_bytes + - is_true: nodes.os.mem.free_in_bytes + - is_true: nodes.os.mem.used_in_bytes + - gte: { nodes.os.mem.free_percent: 0 } + - gte: { nodes.os.mem.used_percent: 0 } + - is_true: nodes.process + - is_true: nodes.jvm + - is_true: nodes.fs + - is_true: nodes.plugins + - is_true: nodes.network_types diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java index 469106c9a61..372704afaf6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -40,15 +40,18 @@ public class ClusterStatsResponse extends BaseNodesResponse nodes, List failures) { super(clusterName, nodes, failures); + this.clusterUUID = clusterUUID; this.timestamp = timestamp; nodesStats = new ClusterStatsNodes(nodes); indicesStats = new ClusterStatsIndices(nodes); @@ -61,6 +64,10 @@ public class ClusterStatsResponse extends BaseNodesResponse responses, List failures) { return new ClusterStatsResponse( System.currentTimeMillis(), + clusterService.state().metaData().clusterUUID(), clusterService.getClusterName(), responses, failures); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index 098f4190b0e..c7ddb3c4d24 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -302,6 +302,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase Date: Thu, 2 Aug 2018 19:38:49 -0500 Subject: [PATCH 06/11] HLRC: Move commercial clients from XPackClient (#32596) The commercial clients were improperly placed into XPackClient, which is a wrapper for the miscellaneous usage and info APIs. This commit moves them into the HLRC. --- .../client/RestHighLevelClient.java | 34 +++++++++++++++---- .../org/elasticsearch/client/XPackClient.java | 19 ----------- .../client/RestHighLevelClientTests.java | 4 ++- .../org/elasticsearch/client/WatcherIT.java | 6 ++-- .../LicensingDocumentationIT.java | 4 +-- .../documentation/WatcherDocumentationIT.java | 8 ++--- 6 files changed, 40 insertions(+), 35 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index c71bebf6903..816631ff94f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -205,6 +205,8 @@ public class RestHighLevelClient implements Closeable { private final SnapshotClient snapshotClient = new SnapshotClient(this); private final TasksClient tasksClient = new TasksClient(this); private final XPackClient xPackClient = new XPackClient(this); + private final WatcherClient watcherClient = new WatcherClient(this); + private final LicenseClient licenseClient = new LicenseClient(this); /** * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the @@ -296,18 +298,38 @@ public class RestHighLevelClient implements Closeable { } /** - * A wrapper for the {@link RestHighLevelClient} that provides methods for - * accessing the Elastic Licensed X-Pack APIs that are shipped with the - * default distribution of Elasticsearch. All of these APIs will 404 if run - * against the OSS distribution of Elasticsearch. + * Provides methods for accessing the Elastic Licensed X-Pack Info + * and Usage APIs that are shipped with the default distribution of + * Elasticsearch. All of these APIs will 404 if run against the OSS + * distribution of Elasticsearch. *

- * See the - * X-Pack APIs on elastic.co for more information. + * See the + * Info APIs on elastic.co for more information. */ public final XPackClient xpack() { return xPackClient; } + /** + * Provides methods for accessing the Elastic Licensed Watcher APIs that + * are shipped with the default distribution of Elasticsearch. All of + * these APIs will 404 if run against the OSS distribution of Elasticsearch. + *

+ * See the + * Watcher APIs on elastic.co for more information. + */ + public WatcherClient watcher() { return watcherClient; } + + /** + * Provides methods for accessing the Elastic Licensed Licensing APIs that + * are shipped with the default distribution of Elasticsearch. All of + * these APIs will 404 if run against the OSS distribution of Elasticsearch. + *

+ * See the + * Licensing APIs on elastic.co for more information. + */ + public LicenseClient license() { return licenseClient; } + /** * Executes a bulk request using the Bulk API. * See Bulk API on elastic.co diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java index 1401376527d..2af49ba1a1b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java @@ -41,17 +41,9 @@ import static java.util.Collections.emptySet; public final class XPackClient { private final RestHighLevelClient restHighLevelClient; - private final WatcherClient watcherClient; - private final LicenseClient licenseClient; XPackClient(RestHighLevelClient restHighLevelClient) { this.restHighLevelClient = restHighLevelClient; - this.watcherClient = new WatcherClient(restHighLevelClient); - this.licenseClient = new LicenseClient(restHighLevelClient); - } - - public WatcherClient watcher() { - return watcherClient; } /** @@ -102,15 +94,4 @@ public final class XPackClient { restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xpackUsage, options, XPackUsageResponse::fromXContent, listener, emptySet()); } - - /** - * A wrapper for the {@link RestHighLevelClient} that provides methods for - * accessing the Elastic Licensing APIs. - *

- * See the - * X-Pack APIs on elastic.co for more information. - */ - public LicenseClient license() { - return licenseClient; - } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 5cf3b352756..4dd58f0f0be 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -755,7 +755,9 @@ public class RestHighLevelClientTests extends ESTestCase { method.isAnnotationPresent(Deprecated.class)); } else { //TODO xpack api are currently ignored, we need to load xpack yaml spec too - if (apiName.startsWith("xpack.") == false) { + if (apiName.startsWith("xpack.") == false && + apiName.startsWith("license.") == false && + apiName.startsWith("watcher.") == false) { apiNotFound.add(apiName); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java index 67d1def323a..491992735af 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java @@ -46,7 +46,7 @@ public class WatcherIT extends ESRestHighLevelClientTestCase { "}"; BytesReference bytesReference = new BytesArray(json); PutWatchRequest putWatchRequest = new PutWatchRequest(watchId, bytesReference, XContentType.JSON); - return highLevelClient().xpack().watcher().putWatch(putWatchRequest, RequestOptions.DEFAULT); + return highLevelClient().watcher().putWatch(putWatchRequest, RequestOptions.DEFAULT); } public void testDeleteWatch() throws Exception { @@ -54,7 +54,7 @@ public class WatcherIT extends ESRestHighLevelClientTestCase { { String watchId = randomAlphaOfLength(10); createWatch(watchId); - DeleteWatchResponse deleteWatchResponse = highLevelClient().xpack().watcher().deleteWatch(new DeleteWatchRequest(watchId), + DeleteWatchResponse deleteWatchResponse = highLevelClient().watcher().deleteWatch(new DeleteWatchRequest(watchId), RequestOptions.DEFAULT); assertThat(deleteWatchResponse.getId(), is(watchId)); assertThat(deleteWatchResponse.getVersion(), is(2L)); @@ -64,7 +64,7 @@ public class WatcherIT extends ESRestHighLevelClientTestCase { // delete watch that does not exist { String watchId = randomAlphaOfLength(10); - DeleteWatchResponse deleteWatchResponse = highLevelClient().xpack().watcher().deleteWatch(new DeleteWatchRequest(watchId), + DeleteWatchResponse deleteWatchResponse = highLevelClient().watcher().deleteWatch(new DeleteWatchRequest(watchId), RequestOptions.DEFAULT); assertThat(deleteWatchResponse.getId(), is(watchId)); assertThat(deleteWatchResponse.getVersion(), is(1L)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java index 776c990610d..0f516d5d37d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java @@ -62,7 +62,7 @@ public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase { request.setLicenseDefinition(license); // <1> request.setAcknowledge(false); // <2> - PutLicenseResponse response = client.xpack().license().putLicense(request, RequestOptions.DEFAULT); + PutLicenseResponse response = client.license().putLicense(request, RequestOptions.DEFAULT); //end::put-license-execute //tag::put-license-response @@ -98,7 +98,7 @@ public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase { listener = new LatchedActionListener<>(listener, latch); // tag::put-license-execute-async - client.xpack().license().putLicenseAsync( + client.license().putLicenseAsync( request, RequestOptions.DEFAULT, listener); // <1> // end::put-license-execute-async diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java index 47f8510b746..707997d1f31 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java @@ -49,7 +49,7 @@ public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase { "}"); PutWatchRequest request = new PutWatchRequest("my_watch_id", watch, XContentType.JSON); request.setActive(false); // <1> - PutWatchResponse response = client.xpack().watcher().putWatch(request, RequestOptions.DEFAULT); + PutWatchResponse response = client.watcher().putWatch(request, RequestOptions.DEFAULT); //end::x-pack-put-watch-execute //tag::x-pack-put-watch-response @@ -85,7 +85,7 @@ public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase { listener = new LatchedActionListener<>(listener, latch); // tag::x-pack-put-watch-execute-async - client.xpack().watcher().putWatchAsync(request, RequestOptions.DEFAULT, listener); // <1> + client.watcher().putWatchAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::x-pack-put-watch-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -94,7 +94,7 @@ public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase { { //tag::x-pack-delete-watch-execute DeleteWatchRequest request = new DeleteWatchRequest("my_watch_id"); - DeleteWatchResponse response = client.xpack().watcher().deleteWatch(request, RequestOptions.DEFAULT); + DeleteWatchResponse response = client.watcher().deleteWatch(request, RequestOptions.DEFAULT); //end::x-pack-delete-watch-execute //tag::x-pack-delete-watch-response @@ -125,7 +125,7 @@ public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase { listener = new LatchedActionListener<>(listener, latch); // tag::x-pack-delete-watch-execute-async - client.xpack().watcher().deleteWatchAsync(request, RequestOptions.DEFAULT, listener); // <1> + client.watcher().deleteWatchAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::x-pack-delete-watch-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); From 0d60e8a029a8e38cbf69c2d923f0b97dc144c8af Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Fri, 3 Aug 2018 09:33:08 +0200 Subject: [PATCH 07/11] Fix race between replica reset and primary promotion (#32442) We've recently seen a number of test failures that tripped an assertion in IndexShard (see issues linked below), leading to the discovery of a race between resetting a replica when it learns about a higher term and when the same replica is promoted to primary. This commit fixes the race by distinguishing between a cluster state primary term (called pendingPrimaryTerm) and a shard-level operation term. The former is set during the cluster state update or when a replica learns about a new primary. The latter is only incremented under the operation block, which can happen in a delayed fashion. It also solves the issue where a replica that's still adjusting to the new term receives a cluster state update that promotes it to primary, which can happen in the situation of multiple nodes being shut down in short succession. In that case, the cluster state update thread would call `asyncBlockOperations` in `updateShardState`, which in turn would throw an exception as blocking permits is not allowed while an ongoing block is in place, subsequently failing the shard. This commit therefore extends the IndexShardOperationPermits to allow it to queue multiple blocks (which will all take precedence over operations acquiring permits). Finally, it also moves the primary activation of the replication tracker under the operation block, so that the actual transition to primary only happens under the operation block. Relates to #32431, #32304 and #32118 --- .../action/bulk/TransportShardBulkAction.java | 14 +- .../TransportReplicationAction.java | 4 +- .../elasticsearch/index/engine/Engine.java | 44 +++-- .../index/engine/InternalEngine.java | 32 +-- .../index/seqno/ReplicationTracker.java | 18 ++ .../elasticsearch/index/shard/IndexShard.java | 183 ++++++++++-------- .../shard/IndexShardOperationPermits.java | 40 ++-- .../index/shard/PrimaryReplicaSyncer.java | 2 +- .../index/shard/StoreRecovery.java | 11 +- .../index/translog/Translog.java | 5 +- .../recovery/RecoverySourceHandler.java | 2 +- .../indices/recovery/RecoveryTarget.java | 3 +- .../bulk/TransportShardBulkActionTests.java | 20 +- .../TransportReplicationActionTests.java | 12 +- .../TransportWriteActionTests.java | 6 +- .../routing/allocation/ShardStateIT.java | 2 +- .../IndexLevelReplicationTests.java | 106 +++++++++- .../RecoveryDuringReplicationTests.java | 5 +- .../index/seqno/ReplicationTrackerTests.java | 2 + .../IndexShardOperationPermitsTests.java | 40 +++- .../index/shard/IndexShardTests.java | 86 ++++---- .../shard/IndexingOperationListenerTests.java | 4 +- .../shard/PrimaryReplicaSyncerTests.java | 4 +- .../index/translog/TranslogTests.java | 4 +- .../recovery/RecoverySourceHandlerTests.java | 8 +- .../indices/recovery/RecoveryTests.java | 2 +- .../ESIndexLevelReplicationTestCase.java | 54 ++++-- .../index/shard/IndexShardTestCase.java | 14 +- 28 files changed, 471 insertions(+), 256 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 15a98077eac..ed99c739afb 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -144,7 +144,7 @@ public class TransportShardBulkAction extends TransportWriteAction primary.applyIndexOperationOnPrimary(request.version(), request.versionType(), sourceToParse, request.getAutoGeneratedTimestamp(), request.isRetry()), - e -> new Engine.IndexResult(e, request.version()), + e -> primary.getFailedIndexResult(e, request.version()), mappingUpdater); } @@ -567,7 +567,7 @@ public class TransportShardBulkAction extends TransportWriteAction primary.applyDeleteOperationOnPrimary(request.version(), request.type(), request.id(), request.versionType()), - e -> new Engine.DeleteResult(e, request.version()), + e -> primary.getFailedDeleteResult(e, request.version()), mappingUpdater); } diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 53d9752f4ed..dbdd5acae1f 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -929,7 +929,7 @@ public abstract class TransportReplicationAction< if (actualAllocationId.equals(allocationId) == false) { throw new ShardNotFoundException(shardId, "expected aID [{}] but found [{}]", allocationId, actualAllocationId); } - final long actualTerm = indexShard.getPrimaryTerm(); + final long actualTerm = indexShard.getPendingPrimaryTerm(); if (actualTerm != primaryTerm) { throw new ShardNotFoundException(shardId, "expected aID [{}] with term [{}] but found [{}]", allocationId, primaryTerm, actualTerm); @@ -983,7 +983,7 @@ public abstract class TransportReplicationAction< } public boolean isRelocated() { - return indexShard.isPrimaryMode() == false; + return indexShard.isRelocatedPrimary(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index b7c938b469f..31da7afc51a 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -304,6 +304,7 @@ public abstract class Engine implements Closeable { private final Operation.TYPE operationType; private final Result.Type resultType; private final long version; + private final long term; private final long seqNo; private final Exception failure; private final SetOnce freeze = new SetOnce<>(); @@ -311,19 +312,21 @@ public abstract class Engine implements Closeable { private Translog.Location translogLocation; private long took; - protected Result(Operation.TYPE operationType, Exception failure, long version, long seqNo) { + protected Result(Operation.TYPE operationType, Exception failure, long version, long term, long seqNo) { this.operationType = operationType; this.failure = Objects.requireNonNull(failure); this.version = version; + this.term = term; this.seqNo = seqNo; this.requiredMappingUpdate = null; this.resultType = Type.FAILURE; } - protected Result(Operation.TYPE operationType, long version, long seqNo) { + protected Result(Operation.TYPE operationType, long version, long term, long seqNo) { this.operationType = operationType; this.version = version; this.seqNo = seqNo; + this.term = term; this.failure = null; this.requiredMappingUpdate = null; this.resultType = Type.SUCCESS; @@ -333,6 +336,7 @@ public abstract class Engine implements Closeable { this.operationType = operationType; this.version = Versions.NOT_FOUND; this.seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + this.term = 0L; this.failure = null; this.requiredMappingUpdate = requiredMappingUpdate; this.resultType = Type.MAPPING_UPDATE_REQUIRED; @@ -357,6 +361,10 @@ public abstract class Engine implements Closeable { return seqNo; } + public long getTerm() { + return term; + } + /** * If the operation was aborted due to missing mappings, this method will return the mappings * that are required to complete the operation. @@ -415,20 +423,20 @@ public abstract class Engine implements Closeable { private final boolean created; - public IndexResult(long version, long seqNo, boolean created) { - super(Operation.TYPE.INDEX, version, seqNo); + public IndexResult(long version, long term, long seqNo, boolean created) { + super(Operation.TYPE.INDEX, version, term, seqNo); this.created = created; } /** * use in case of the index operation failed before getting to internal engine **/ - public IndexResult(Exception failure, long version) { - this(failure, version, SequenceNumbers.UNASSIGNED_SEQ_NO); + public IndexResult(Exception failure, long version, long term) { + this(failure, version, term, SequenceNumbers.UNASSIGNED_SEQ_NO); } - public IndexResult(Exception failure, long version, long seqNo) { - super(Operation.TYPE.INDEX, failure, version, seqNo); + public IndexResult(Exception failure, long version, long term, long seqNo) { + super(Operation.TYPE.INDEX, failure, version, term, seqNo); this.created = false; } @@ -447,20 +455,20 @@ public abstract class Engine implements Closeable { private final boolean found; - public DeleteResult(long version, long seqNo, boolean found) { - super(Operation.TYPE.DELETE, version, seqNo); + public DeleteResult(long version, long term, long seqNo, boolean found) { + super(Operation.TYPE.DELETE, version, term, seqNo); this.found = found; } /** * use in case of the delete operation failed before getting to internal engine **/ - public DeleteResult(Exception failure, long version) { - this(failure, version, SequenceNumbers.UNASSIGNED_SEQ_NO, false); + public DeleteResult(Exception failure, long version, long term) { + this(failure, version, term, SequenceNumbers.UNASSIGNED_SEQ_NO, false); } - public DeleteResult(Exception failure, long version, long seqNo, boolean found) { - super(Operation.TYPE.DELETE, failure, version, seqNo); + public DeleteResult(Exception failure, long version, long term, long seqNo, boolean found) { + super(Operation.TYPE.DELETE, failure, version, term, seqNo); this.found = found; } @@ -477,12 +485,12 @@ public abstract class Engine implements Closeable { public static class NoOpResult extends Result { - NoOpResult(long seqNo) { - super(Operation.TYPE.NO_OP, 0, seqNo); + NoOpResult(long term, long seqNo) { + super(Operation.TYPE.NO_OP, term, 0, seqNo); } - NoOpResult(long seqNo, Exception failure) { - super(Operation.TYPE.NO_OP, failure, 0, seqNo); + NoOpResult(long term, long seqNo, Exception failure) { + super(Operation.TYPE.NO_OP, failure, term, 0, seqNo); } } diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index bdcfb2fc731..a30127a24ae 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -736,6 +736,10 @@ public class InternalEngine extends Engine { return localCheckpointTracker.generateSeqNo(); } + private long getPrimaryTerm() { + return engineConfig.getPrimaryTermSupplier().getAsLong(); + } + @Override public IndexResult index(Index index) throws IOException { assert Objects.equals(index.uid().field(), IdFieldMapper.NAME) : index.uid().field(); @@ -788,7 +792,7 @@ public class InternalEngine extends Engine { indexResult = indexIntoLucene(index, plan); } else { indexResult = new IndexResult( - plan.versionForIndexing, plan.seqNoForIndexing, plan.currentNotFoundOrDeleted); + plan.versionForIndexing, getPrimaryTerm(), plan.seqNoForIndexing, plan.currentNotFoundOrDeleted); } if (index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { final Translog.Location location; @@ -900,7 +904,7 @@ public class InternalEngine extends Engine { currentVersion, index.version(), currentNotFoundOrDeleted)) { final VersionConflictEngineException e = new VersionConflictEngineException(shardId, index, currentVersion, currentNotFoundOrDeleted); - plan = IndexingStrategy.skipDueToVersionConflict(e, currentNotFoundOrDeleted, currentVersion); + plan = IndexingStrategy.skipDueToVersionConflict(e, currentNotFoundOrDeleted, currentVersion, getPrimaryTerm()); } else { plan = IndexingStrategy.processNormally(currentNotFoundOrDeleted, generateSeqNoForOperation(index), @@ -930,7 +934,7 @@ public class InternalEngine extends Engine { assert assertDocDoesNotExist(index, canOptimizeAddDocument(index) == false); addDocs(index.docs(), indexWriter); } - return new IndexResult(plan.versionForIndexing, plan.seqNoForIndexing, plan.currentNotFoundOrDeleted); + return new IndexResult(plan.versionForIndexing, getPrimaryTerm(), plan.seqNoForIndexing, plan.currentNotFoundOrDeleted); } catch (Exception ex) { if (indexWriter.getTragicException() == null) { /* There is no tragic event recorded so this must be a document failure. @@ -946,7 +950,7 @@ public class InternalEngine extends Engine { * we return a `MATCH_ANY` version to indicate no document was index. The value is * not used anyway */ - return new IndexResult(ex, Versions.MATCH_ANY, plan.seqNoForIndexing); + return new IndexResult(ex, Versions.MATCH_ANY, getPrimaryTerm(), plan.seqNoForIndexing); } else { throw ex; } @@ -1019,8 +1023,8 @@ public class InternalEngine extends Engine { } static IndexingStrategy skipDueToVersionConflict( - VersionConflictEngineException e, boolean currentNotFoundOrDeleted, long currentVersion) { - final IndexResult result = new IndexResult(e, currentVersion); + VersionConflictEngineException e, boolean currentNotFoundOrDeleted, long currentVersion, long term) { + final IndexResult result = new IndexResult(e, currentVersion, term); return new IndexingStrategy( currentNotFoundOrDeleted, false, false, SequenceNumbers.UNASSIGNED_SEQ_NO, Versions.NOT_FOUND, result); } @@ -1097,7 +1101,7 @@ public class InternalEngine extends Engine { deleteResult = deleteInLucene(delete, plan); } else { deleteResult = new DeleteResult( - plan.versionOfDeletion, plan.seqNoOfDeletion, plan.currentlyDeleted == false); + plan.versionOfDeletion, getPrimaryTerm(), plan.seqNoOfDeletion, plan.currentlyDeleted == false); } if (delete.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { final Translog.Location location; @@ -1178,7 +1182,7 @@ public class InternalEngine extends Engine { final DeletionStrategy plan; if (delete.versionType().isVersionConflictForWrites(currentVersion, delete.version(), currentlyDeleted)) { final VersionConflictEngineException e = new VersionConflictEngineException(shardId, delete, currentVersion, currentlyDeleted); - plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, currentlyDeleted); + plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, getPrimaryTerm(), currentlyDeleted); } else { plan = DeletionStrategy.processNormally( currentlyDeleted, @@ -1201,12 +1205,12 @@ public class InternalEngine extends Engine { new DeleteVersionValue(plan.versionOfDeletion, plan.seqNoOfDeletion, delete.primaryTerm(), engineConfig.getThreadPool().relativeTimeInMillis())); return new DeleteResult( - plan.versionOfDeletion, plan.seqNoOfDeletion, plan.currentlyDeleted == false); + plan.versionOfDeletion, getPrimaryTerm(), plan.seqNoOfDeletion, plan.currentlyDeleted == false); } catch (Exception ex) { if (indexWriter.getTragicException() == null) { // there is no tragic event and such it must be a document level failure return new DeleteResult( - ex, plan.versionOfDeletion, plan.seqNoOfDeletion, plan.currentlyDeleted == false); + ex, plan.versionOfDeletion, getPrimaryTerm(), plan.seqNoOfDeletion, plan.currentlyDeleted == false); } else { throw ex; } @@ -1237,9 +1241,9 @@ public class InternalEngine extends Engine { } static DeletionStrategy skipDueToVersionConflict( - VersionConflictEngineException e, long currentVersion, boolean currentlyDeleted) { + VersionConflictEngineException e, long currentVersion, long term, boolean currentlyDeleted) { final long unassignedSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; - final DeleteResult deleteResult = new DeleteResult(e, currentVersion, unassignedSeqNo, currentlyDeleted == false); + final DeleteResult deleteResult = new DeleteResult(e, currentVersion, term, unassignedSeqNo, currentlyDeleted == false); return new DeletionStrategy(false, currentlyDeleted, unassignedSeqNo, Versions.NOT_FOUND, deleteResult); } @@ -1268,7 +1272,7 @@ public class InternalEngine extends Engine { try (ReleasableLock ignored = readLock.acquire()) { noOpResult = innerNoOp(noOp); } catch (final Exception e) { - noOpResult = new NoOpResult(noOp.seqNo(), e); + noOpResult = new NoOpResult(getPrimaryTerm(), noOp.seqNo(), e); } return noOpResult; } @@ -1278,7 +1282,7 @@ public class InternalEngine extends Engine { assert noOp.seqNo() > SequenceNumbers.NO_OPS_PERFORMED; final long seqNo = noOp.seqNo(); try { - final NoOpResult noOpResult = new NoOpResult(noOp.seqNo()); + final NoOpResult noOpResult = new NoOpResult(getPrimaryTerm(), noOp.seqNo()); if (noOp.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { final Translog.Location location = translog.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason())); noOpResult.setTranslogLocation(location); diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java index 6548aad7670..e868da5e82a 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java @@ -85,6 +85,7 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L * computation from that point on. */ volatile boolean primaryMode; + /** * Boolean flag that indicates if a relocation handoff is in progress. A handoff is started by calling {@link #startRelocationHandoff} * and is finished by either calling {@link #completeRelocationHandoff} or {@link #abortRelocationHandoff}, depending on whether the @@ -102,6 +103,11 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L */ boolean handoffInProgress; + /** + * Boolean flag that indicates whether a relocation handoff completed (see {@link #completeRelocationHandoff}). + */ + volatile boolean relocated; + /** * The global checkpoint tracker relies on the property that cluster state updates are applied in-order. After transferring a primary * context from the primary relocation source to the target and initializing the target, it is possible for the target to apply a @@ -260,6 +266,13 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L return primaryMode; } + /** + * Returns whether the replication tracker has relocated away to another shard copy. + */ + public boolean isRelocated() { + return relocated; + } + /** * Class invariant that should hold before and after every invocation of public methods on this class. As Java lacks implication * as a logical operator, many of the invariants are written under the form (!A || B), they should be read as (A implies B) however. @@ -287,6 +300,9 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L // relocation handoff can only occur in primary mode assert !handoffInProgress || primaryMode; + // a relocated copy is not in primary mode + assert !relocated || !primaryMode; + // the current shard is marked as in-sync when the global checkpoint tracker operates in primary mode assert !primaryMode || checkpoints.get(shardAllocationId).inSync; @@ -766,8 +782,10 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L assert invariant(); assert primaryMode; assert handoffInProgress; + assert relocated == false; primaryMode = false; handoffInProgress = false; + relocated = true; // forget all checkpoint information except for global checkpoint of current shard checkpoints.entrySet().stream().forEach(e -> { final CheckpointState cps = e.getValue(); diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index d4a1d0502d0..f29f17a46dc 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -52,6 +52,7 @@ import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -192,7 +193,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl protected volatile ShardRouting shardRouting; protected volatile IndexShardState state; - protected volatile long primaryTerm; + protected volatile long pendingPrimaryTerm; // see JavaDocs for getPendingPrimaryTerm + protected volatile long operationPrimaryTerm; protected final AtomicReference currentEngineReference = new AtomicReference<>(); final EngineFactory engineFactory; @@ -315,7 +317,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } indexShardOperationPermits = new IndexShardOperationPermits(shardId, threadPool); searcherWrapper = indexSearcherWrapper; - primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id()); + pendingPrimaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id()); + operationPrimaryTerm = pendingPrimaryTerm; refreshListeners = buildRefreshListeners(); lastSearcherAccess.set(threadPool.relativeTimeInMillis()); persistMetadata(path, indexSettings, shardRouting, null, logger); @@ -365,10 +368,14 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } /** - * Returns the primary term the index shard is on. See {@link org.elasticsearch.cluster.metadata.IndexMetaData#primaryTerm(int)} + * USE THIS METHOD WITH CARE! + * Returns the primary term the index shard is supposed to be on. In case of primary promotion or when a replica learns about + * a new term due to a new primary, the term that's exposed here will not be the term that the shard internally uses to assign + * to operations. The shard will auto-correct its internal operation term, but this might take time. + * See {@link org.elasticsearch.cluster.metadata.IndexMetaData#primaryTerm(int)} */ - public long getPrimaryTerm() { - return this.primaryTerm; + public long getPendingPrimaryTerm() { + return this.pendingPrimaryTerm; } /** @@ -418,7 +425,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl "a primary relocation is completed by the master, but primary mode is not active " + currentRouting; changeState(IndexShardState.STARTED, "global state is [" + newRouting.state() + "]"); - } else if (currentRouting.primary() && currentRouting.relocating() && replicationTracker.isPrimaryMode() == false && + } else if (currentRouting.primary() && currentRouting.relocating() && replicationTracker.isRelocated() && (newRouting.relocating() == false || newRouting.equalsIgnoringMetaData(currentRouting) == false)) { // if the shard is not in primary mode anymore (after primary relocation) we have to fail when any changes in shard routing occur (e.g. due to recovery // failure / cancellation). The reason is that at the moment we cannot safely reactivate primary mode without risking two @@ -431,7 +438,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl final CountDownLatch shardStateUpdated = new CountDownLatch(1); if (newRouting.primary()) { - if (newPrimaryTerm == primaryTerm) { + if (newPrimaryTerm == pendingPrimaryTerm) { if (currentRouting.initializing() && currentRouting.isRelocationTarget() == false && newRouting.active()) { // the master started a recovering primary, activate primary mode. replicationTracker.activatePrimaryMode(getLocalCheckpoint()); @@ -454,10 +461,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl assert newRouting.initializing() == false : "a started primary shard should never update its term; " + "shard " + newRouting + ", " - + "current term [" + primaryTerm + "], " + + "current term [" + pendingPrimaryTerm + "], " + "new term [" + newPrimaryTerm + "]"; - assert newPrimaryTerm > primaryTerm : - "primary terms can only go up; current term [" + primaryTerm + "], new term [" + newPrimaryTerm + "]"; + assert newPrimaryTerm > pendingPrimaryTerm : + "primary terms can only go up; current term [" + pendingPrimaryTerm + "], new term [" + newPrimaryTerm + "]"; /* * Before this call returns, we are guaranteed that all future operations are delayed and so this happens before we * increment the primary term. The latch is needed to ensure that we do not unblock operations before the primary term is @@ -468,12 +475,15 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl if (resyncStarted == false) { throw new IllegalStateException("cannot start resync while it's already in progress"); } - indexShardOperationPermits.asyncBlockOperations( - 30, - TimeUnit.MINUTES, + bumpPrimaryTerm(newPrimaryTerm, () -> { shardStateUpdated.await(); + assert pendingPrimaryTerm == newPrimaryTerm : + "shard term changed on primary. expected [" + newPrimaryTerm + "] but was [" + pendingPrimaryTerm + "]" + + ", current routing: " + currentRouting + ", new routing: " + newRouting; + assert operationPrimaryTerm == newPrimaryTerm; try { + replicationTracker.activatePrimaryMode(getLocalCheckpoint()); /* * If this shard was serving as a replica shard when another shard was promoted to primary then the state of * its local checkpoint tracker was reset during the primary term transition. In particular, the local @@ -517,10 +527,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } catch (final AlreadyClosedException e) { // okay, the index was deleted } - }, - e -> failShard("exception during primary term transition", e)); - replicationTracker.activatePrimaryMode(getLocalCheckpoint()); - primaryTerm = newPrimaryTerm; + }); } } // set this last, once we finished updating all internal state. @@ -528,8 +535,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl assert this.shardRouting.primary() == false || this.shardRouting.started() == false || // note that we use started and not active to avoid relocating shards + this.indexShardOperationPermits.isBlocked() || // if permits are blocked, we are still transitioning this.replicationTracker.isPrimaryMode() - : "an started primary must be in primary mode " + this.shardRouting; + : "a started primary with non-pending operation term must be in primary mode " + this.shardRouting; shardStateUpdated.countDown(); } if (currentRouting != null && currentRouting.active() == false && newRouting.active()) { @@ -590,7 +598,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl consumer.accept(primaryContext); synchronized (mutex) { verifyRelocatingState(); - replicationTracker.completeRelocationHandoff(); // make changes to primaryMode flag only under mutex + replicationTracker.completeRelocationHandoff(); // make changes to primaryMode and relocated flag only under mutex } } catch (final Exception e) { try { @@ -655,21 +663,22 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl public Engine.IndexResult applyIndexOperationOnPrimary(long version, VersionType versionType, SourceToParse sourceToParse, long autoGeneratedTimestamp, boolean isRetry) throws IOException { assert versionType.validateVersionForWrites(version); - return applyIndexOperation(SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm, version, versionType, autoGeneratedTimestamp, + return applyIndexOperation(SequenceNumbers.UNASSIGNED_SEQ_NO, operationPrimaryTerm, version, versionType, autoGeneratedTimestamp, isRetry, Engine.Operation.Origin.PRIMARY, sourceToParse); } public Engine.IndexResult applyIndexOperationOnReplica(long seqNo, long version, long autoGeneratedTimeStamp, boolean isRetry, SourceToParse sourceToParse) throws IOException { - return applyIndexOperation(seqNo, primaryTerm, version, null, autoGeneratedTimeStamp, isRetry, + return applyIndexOperation(seqNo, operationPrimaryTerm, version, null, autoGeneratedTimeStamp, isRetry, Engine.Operation.Origin.REPLICA, sourceToParse); } private Engine.IndexResult applyIndexOperation(long seqNo, long opPrimaryTerm, long version, @Nullable VersionType versionType, long autoGeneratedTimeStamp, boolean isRetry, Engine.Operation.Origin origin, SourceToParse sourceToParse) throws IOException { - assert opPrimaryTerm <= this.primaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.primaryTerm + "]"; + assert opPrimaryTerm <= this.operationPrimaryTerm: "op term [ " + opPrimaryTerm + " ] > shard term [" + this.operationPrimaryTerm + + "]"; ensureWriteAllowed(origin); Engine.Index operation; try { @@ -686,7 +695,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl // can not raise an exception that may block any replication of previous operations to the // replicas verifyNotClosed(e); - return new Engine.IndexResult(e, version, seqNo); + return new Engine.IndexResult(e, version, opPrimaryTerm, seqNo); } return index(getEngine(), operation); @@ -723,12 +732,13 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } public Engine.NoOpResult markSeqNoAsNoop(long seqNo, String reason) throws IOException { - return markSeqNoAsNoop(seqNo, primaryTerm, reason, Engine.Operation.Origin.REPLICA); + return markSeqNoAsNoop(seqNo, operationPrimaryTerm, reason, Engine.Operation.Origin.REPLICA); } private Engine.NoOpResult markSeqNoAsNoop(long seqNo, long opPrimaryTerm, String reason, Engine.Operation.Origin origin) throws IOException { - assert opPrimaryTerm <= this.primaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.primaryTerm + "]"; + assert opPrimaryTerm <= this.operationPrimaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.operationPrimaryTerm + + "]"; long startTime = System.nanoTime(); ensureWriteAllowed(origin); final Engine.NoOp noOp = new Engine.NoOp(seqNo, opPrimaryTerm, origin, startTime, reason); @@ -743,20 +753,29 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl return engine.noOp(noOp); } + public Engine.IndexResult getFailedIndexResult(Exception e, long version) { + return new Engine.IndexResult(e, version, operationPrimaryTerm); + } + + public Engine.DeleteResult getFailedDeleteResult(Exception e, long version) { + return new Engine.DeleteResult(e, version, operationPrimaryTerm); + } + public Engine.DeleteResult applyDeleteOperationOnPrimary(long version, String type, String id, VersionType versionType) throws IOException { assert versionType.validateVersionForWrites(version); - return applyDeleteOperation(SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm, version, type, id, versionType, + return applyDeleteOperation(SequenceNumbers.UNASSIGNED_SEQ_NO, operationPrimaryTerm, version, type, id, versionType, Engine.Operation.Origin.PRIMARY); } public Engine.DeleteResult applyDeleteOperationOnReplica(long seqNo, long version, String type, String id) throws IOException { - return applyDeleteOperation(seqNo, primaryTerm, version, type, id, null, Engine.Operation.Origin.REPLICA); + return applyDeleteOperation(seqNo, operationPrimaryTerm, version, type, id, null, Engine.Operation.Origin.REPLICA); } private Engine.DeleteResult applyDeleteOperation(long seqNo, long opPrimaryTerm, long version, String type, String id, @Nullable VersionType versionType, Engine.Operation.Origin origin) throws IOException { - assert opPrimaryTerm <= this.primaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.primaryTerm + "]"; + assert opPrimaryTerm <= this.operationPrimaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.operationPrimaryTerm + + "]"; ensureWriteAllowed(origin); // When there is a single type, the unique identifier is only composed of the _id, // so there is no way to differenciate foo#1 from bar#1. This is especially an issue @@ -772,7 +791,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl return new Engine.DeleteResult(update); } } catch (MapperParsingException | IllegalArgumentException | TypeMissingException e) { - return new Engine.DeleteResult(e, version, seqNo, false); + return new Engine.DeleteResult(e, version, operationPrimaryTerm, seqNo, false); } final Term uid = extractUidForDelete(type, id); final Engine.Delete delete = prepareDelete(type, id, uid, seqNo, opPrimaryTerm, version, @@ -1209,7 +1228,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } public void trimOperationOfPreviousPrimaryTerms(long aboveSeqNo) { - getEngine().trimOperationsFromTranslog(primaryTerm, aboveSeqNo); + getEngine().trimOperationsFromTranslog(operationPrimaryTerm, aboveSeqNo); } public Engine.Result applyTranslogOperation(Translog.Operation operation, Engine.Operation.Origin origin) throws IOException { @@ -2082,10 +2101,11 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } /** - * Returns whether the shard is in primary mode, i.e., in charge of replicating changes (see {@link ReplicationTracker}). + * Returns whether the shard is a relocated primary, i.e. not in charge anymore of replicating changes (see {@link ReplicationTracker}). */ - public boolean isPrimaryMode() { - return replicationTracker.isPrimaryMode(); + public boolean isRelocatedPrimary() { + assert shardRouting.primary() : "only call isRelocatedPrimary on primary shard"; + return replicationTracker.isRelocated(); } class ShardEventListener implements Engine.EventListener { @@ -2175,7 +2195,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()), Collections.singletonList(refreshListeners), Collections.singletonList(new RefreshMetricUpdater(refreshMetric)), - indexSort, this::runTranslogRecovery, circuitBreakerService, replicationTracker, this::getPrimaryTerm); + indexSort, this::runTranslogRecovery, circuitBreakerService, replicationTracker, () -> operationPrimaryTerm); } /** @@ -2194,7 +2214,25 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl indexShardOperationPermits.acquire(onPermitAcquired, executorOnDelay, false, debugInfo); } - private final Object primaryTermMutex = new Object(); + private void bumpPrimaryTerm(long newPrimaryTerm, final CheckedRunnable onBlocked) { + assert Thread.holdsLock(mutex); + assert newPrimaryTerm > pendingPrimaryTerm; + assert operationPrimaryTerm <= pendingPrimaryTerm; + final CountDownLatch termUpdated = new CountDownLatch(1); + indexShardOperationPermits.asyncBlockOperations(30, TimeUnit.MINUTES, () -> { + assert operationPrimaryTerm <= pendingPrimaryTerm; + termUpdated.await(); + // indexShardOperationPermits doesn't guarantee that async submissions are executed + // in the order submitted. We need to guard against another term bump + if (operationPrimaryTerm < newPrimaryTerm) { + operationPrimaryTerm = newPrimaryTerm; + onBlocked.run(); + } + }, + e -> failShard("exception during primary term transition", e)); + pendingPrimaryTerm = newPrimaryTerm; + termUpdated.countDown(); + } /** * Acquire a replica operation permit whenever the shard is ready for indexing (see @@ -2203,7 +2241,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * {@link IllegalStateException}. If permit acquisition is delayed, the listener will be invoked on the executor with the specified * name. * - * @param operationPrimaryTerm the operation primary term + * @param opPrimaryTerm the operation primary term * @param globalCheckpoint the global checkpoint associated with the request * @param onPermitAcquired the listener for permit acquisition * @param executorOnDelay the name of the executor to invoke the listener on if permit acquisition is delayed @@ -2211,15 +2249,14 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * the tracing will capture the supplied object's {@link Object#toString()} value. Otherwise the object * isn't used */ - public void acquireReplicaOperationPermit(final long operationPrimaryTerm, final long globalCheckpoint, + public void acquireReplicaOperationPermit(final long opPrimaryTerm, final long globalCheckpoint, final ActionListener onPermitAcquired, final String executorOnDelay, final Object debugInfo) { verifyNotClosed(); verifyReplicationTarget(); - final boolean globalCheckpointUpdated; - if (operationPrimaryTerm > primaryTerm) { - synchronized (primaryTermMutex) { - if (operationPrimaryTerm > primaryTerm) { + if (opPrimaryTerm > pendingPrimaryTerm) { + synchronized (mutex) { + if (opPrimaryTerm > pendingPrimaryTerm) { IndexShardState shardState = state(); // only roll translog and update primary term if shard has made it past recovery // Having a new primary term here means that the old primary failed and that there is a new primary, which again @@ -2229,64 +2266,52 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl shardState != IndexShardState.STARTED) { throw new IndexShardNotStartedException(shardId, shardState); } - try { - indexShardOperationPermits.blockOperations(30, TimeUnit.MINUTES, () -> { - assert operationPrimaryTerm > primaryTerm : - "shard term already update. op term [" + operationPrimaryTerm + "], shardTerm [" + primaryTerm + "]"; - primaryTerm = operationPrimaryTerm; - updateGlobalCheckpointOnReplica(globalCheckpoint, "primary term transition"); - final long currentGlobalCheckpoint = getGlobalCheckpoint(); - final long localCheckpoint; - if (currentGlobalCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO) { - localCheckpoint = SequenceNumbers.NO_OPS_PERFORMED; - } else { - localCheckpoint = currentGlobalCheckpoint; - } - logger.trace( + + if (opPrimaryTerm > pendingPrimaryTerm) { + bumpPrimaryTerm(opPrimaryTerm, () -> { + updateGlobalCheckpointOnReplica(globalCheckpoint, "primary term transition"); + final long currentGlobalCheckpoint = getGlobalCheckpoint(); + final long localCheckpoint; + if (currentGlobalCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO) { + localCheckpoint = SequenceNumbers.NO_OPS_PERFORMED; + } else { + localCheckpoint = currentGlobalCheckpoint; + } + logger.trace( "detected new primary with primary term [{}], resetting local checkpoint from [{}] to [{}]", - operationPrimaryTerm, + opPrimaryTerm, getLocalCheckpoint(), localCheckpoint); - getEngine().resetLocalCheckpoint(localCheckpoint); - getEngine().rollTranslogGeneration(); + getEngine().resetLocalCheckpoint(localCheckpoint); + getEngine().rollTranslogGeneration(); }); - globalCheckpointUpdated = true; - } catch (final Exception e) { - onPermitAcquired.onFailure(e); - return; } - } else { - globalCheckpointUpdated = false; } } - } else { - globalCheckpointUpdated = false; } - assert operationPrimaryTerm <= primaryTerm - : "operation primary term [" + operationPrimaryTerm + "] should be at most [" + primaryTerm + "]"; + assert opPrimaryTerm <= pendingPrimaryTerm + : "operation primary term [" + opPrimaryTerm + "] should be at most [" + pendingPrimaryTerm + "]"; indexShardOperationPermits.acquire( new ActionListener() { @Override public void onResponse(final Releasable releasable) { - if (operationPrimaryTerm < primaryTerm) { + if (opPrimaryTerm < operationPrimaryTerm) { releasable.close(); final String message = String.format( Locale.ROOT, "%s operation primary term [%d] is too old (current [%d])", shardId, - operationPrimaryTerm, - primaryTerm); + opPrimaryTerm, + operationPrimaryTerm); onPermitAcquired.onFailure(new IllegalStateException(message)); } else { - if (globalCheckpointUpdated == false) { - try { - updateGlobalCheckpointOnReplica(globalCheckpoint, "operation"); - } catch (Exception e) { - releasable.close(); - onPermitAcquired.onFailure(e); - return; - } + try { + updateGlobalCheckpointOnReplica(globalCheckpoint, "operation"); + } catch (Exception e) { + releasable.close(); + onPermitAcquired.onFailure(e); + return; } onPermitAcquired.onResponse(releasable); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java index a0d46c8eb23..d4c3833b13a 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.shard; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.Assertions; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; @@ -29,10 +28,12 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.threadpool.ThreadPool; import java.io.Closeable; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -59,7 +60,7 @@ final class IndexShardOperationPermits implements Closeable { final Semaphore semaphore = new Semaphore(TOTAL_PERMITS, true); // fair to ensure a blocking thread is not starved private final List delayedOperations = new ArrayList<>(); // operations that are delayed private volatile boolean closed; - private boolean delayed; // does not need to be volatile as all accesses are done under a lock on this + private int queuedBlockOperations; // does not need to be volatile as all accesses are done under a lock on this // only valid when assertions are enabled. Key is AtomicBoolean associated with each permit to ensure close once semantics. // Value is a tuple, with a some debug information supplied by the caller and a stack trace of the acquiring thread @@ -102,9 +103,6 @@ final class IndexShardOperationPermits implements Closeable { final long timeout, final TimeUnit timeUnit, final CheckedRunnable onBlocked) throws InterruptedException, TimeoutException, E { - if (closed) { - throw new IndexShardClosedException(shardId); - } delayOperations(); try { doBlockOperations(timeout, timeUnit, onBlocked); @@ -147,13 +145,12 @@ final class IndexShardOperationPermits implements Closeable { } private void delayOperations() { + if (closed) { + throw new IndexShardClosedException(shardId); + } synchronized (this) { - if (delayed) { - throw new IllegalStateException("operations are already delayed"); - } else { - assert delayedOperations.isEmpty(); - delayed = true; - } + assert queuedBlockOperations > 0 || delayedOperations.isEmpty(); + queuedBlockOperations++; } } @@ -164,7 +161,7 @@ final class IndexShardOperationPermits implements Closeable { if (Assertions.ENABLED) { // since delayed is not volatile, we have to synchronize even here for visibility synchronized (this) { - assert delayed; + assert queuedBlockOperations > 0; } } if (semaphore.tryAcquire(TOTAL_PERMITS, timeout, timeUnit)) { @@ -182,10 +179,14 @@ final class IndexShardOperationPermits implements Closeable { private void releaseDelayedOperations() { final List queuedActions; synchronized (this) { - assert delayed; - queuedActions = new ArrayList<>(delayedOperations); - delayedOperations.clear(); - delayed = false; + assert queuedBlockOperations > 0; + queuedBlockOperations--; + if (queuedBlockOperations == 0) { + queuedActions = new ArrayList<>(delayedOperations); + delayedOperations.clear(); + } else { + queuedActions = Collections.emptyList(); + } } if (!queuedActions.isEmpty()) { /* @@ -242,7 +243,7 @@ final class IndexShardOperationPermits implements Closeable { final Releasable releasable; try { synchronized (this) { - if (delayed) { + if (queuedBlockOperations > 0) { final Supplier contextSupplier = threadPool.getThreadContext().newRestorableContext(false); final ActionListener wrappedListener; if (executorOnDelay != null) { @@ -308,6 +309,11 @@ final class IndexShardOperationPermits implements Closeable { } } + + synchronized boolean isBlocked() { + return queuedBlockOperations > 0; + } + /** * @return a list of describing each permit that wasn't released yet. The description consist of the debugInfo supplied * when the permit was acquired plus a stack traces that was captured when the permit was request. diff --git a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java index e66d78f2e1a..1edc0eb5dca 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java +++ b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java @@ -136,7 +136,7 @@ public class PrimaryReplicaSyncer extends AbstractComponent { } }; - resync(shardId, indexShard.routingEntry().allocationId().getId(), indexShard.getPrimaryTerm(), wrappedSnapshot, + resync(shardId, indexShard.routingEntry().allocationId().getId(), indexShard.getPendingPrimaryTerm(), wrappedSnapshot, startingSeqNo, maxSeqNo, resyncListener); } catch (Exception e) { try { diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index 54718c545a4..e9acfe3d8b0 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -394,7 +394,7 @@ final class StoreRecovery { final SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo(); final long maxSeqNo = Long.parseLong(segmentInfos.userData.get(SequenceNumbers.MAX_SEQ_NO)); final String translogUUID = Translog.createEmptyTranslog( - indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPrimaryTerm()); + indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPendingPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); } else if (indexShouldExists) { // since we recover from local, just fill the files and size @@ -409,11 +409,12 @@ final class StoreRecovery { } else { store.createEmpty(); final String translogUUID = Translog.createEmptyTranslog( - indexShard.shardPath().resolveTranslog(), SequenceNumbers.NO_OPS_PERFORMED, shardId, indexShard.getPrimaryTerm()); + indexShard.shardPath().resolveTranslog(), SequenceNumbers.NO_OPS_PERFORMED, shardId, + indexShard.getPendingPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); } indexShard.openEngineAndRecoverFromTranslog(); - indexShard.getEngine().fillSeqNoGaps(indexShard.getPrimaryTerm()); + indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm()); indexShard.finalizeRecovery(); indexShard.postRecovery("post recovery from shard_store"); } catch (EngineException | IOException e) { @@ -458,11 +459,11 @@ final class StoreRecovery { final SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo(); final long maxSeqNo = Long.parseLong(segmentInfos.userData.get(SequenceNumbers.MAX_SEQ_NO)); final String translogUUID = Translog.createEmptyTranslog( - indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPrimaryTerm()); + indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPendingPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); assert indexShard.shardRouting.primary() : "only primary shards can recover from store"; indexShard.openEngineAndRecoverFromTranslog(); - indexShard.getEngine().fillSeqNoGaps(indexShard.getPrimaryTerm()); + indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm()); indexShard.finalizeRecovery(); indexShard.postRecovery("restore done"); } catch (Exception e) { diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index 31404b7874a..04744bc68c4 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -491,7 +491,10 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC try (ReleasableLock ignored = readLock.acquire()) { ensureOpen(); if (operation.primaryTerm() > current.getPrimaryTerm()) { - throw new IllegalArgumentException("Operation term is newer than the current term;" + assert false : + "Operation term is newer than the current term; " + + "current term[" + current.getPrimaryTerm() + "], operation term[" + operation + "]"; + throw new IllegalArgumentException("Operation term is newer than the current term; " + "current term[" + current.getPrimaryTerm() + "], operation term[" + operation + "]"); } return current.add(bytes, operation.seqNo()); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index 45500349865..352f07d5764 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -250,7 +250,7 @@ public class RecoverySourceHandler { try (Releasable ignored = FutureUtils.get(permit)) { // check that the IndexShard still has the primary authority. This needs to be checked under operation permit to prevent // races, as IndexShard will switch its authority only when it holds all operation permits, see IndexShard.relocated() - if (primary.isPrimaryMode() == false) { + if (primary.isRelocatedPrimary()) { throw new IndexShardRelocatedException(primary.shardId()); } runnable.run(); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index 57deb4666da..1a772f0c3f8 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -443,7 +443,8 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget } // TODO: Assign the global checkpoint to the max_seqno of the safe commit if the index version >= 6.2 final String translogUUID = Translog.createEmptyTranslog( - indexShard.shardPath().resolveTranslog(), SequenceNumbers.UNASSIGNED_SEQ_NO, shardId, indexShard.getPrimaryTerm()); + indexShard.shardPath().resolveTranslog(), SequenceNumbers.UNASSIGNED_SEQ_NO, shardId, + indexShard.getPendingPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) { // this is a fatal exception at this stage. diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java index 006d2d04fdd..bbe25ea02d6 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java @@ -441,7 +441,7 @@ public class TransportShardBulkActionTests extends IndexShardTestCase { BulkItemRequest replicaRequest = new BulkItemRequest(0, writeRequest); Exception err = new ElasticsearchException("I'm dead <(x.x)>"); - Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0); + Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0); BulkItemResultHolder failedResults = new BulkItemResultHolder(null, indexResult, replicaRequest); @@ -478,7 +478,7 @@ public class TransportShardBulkActionTests extends IndexShardTestCase { Exception err = new VersionConflictEngineException(shardId, "_doc", "id", "I'm conflicted <(;_;)>"); - Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0); + Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0); BulkItemResultHolder failedResults = new BulkItemResultHolder(null, indexResult, replicaRequest); @@ -516,7 +516,7 @@ public class TransportShardBulkActionTests extends IndexShardTestCase { boolean created = randomBoolean(); Translog.Location resultLocation = new Translog.Location(42, 42, 42); - Engine.IndexResult indexResult = new FakeResult(1, 1, created, resultLocation); + Engine.IndexResult indexResult = new FakeResult(1, 1, 1, created, resultLocation); DocWriteResponse indexResponse = new IndexResponse(shardId, "_doc", "id", 1, 17, 1, created); BulkItemResultHolder goodResults = new BulkItemResultHolder(indexResponse, indexResult, replicaRequest); @@ -559,7 +559,7 @@ public class TransportShardBulkActionTests extends IndexShardTestCase { Translog.Location newLocation = new Translog.Location(1, 1, 1); final long version = randomNonNegativeLong(); final long seqNo = randomNonNegativeLong(); - Engine.IndexResult indexResult = new IndexResultWithLocation(version, seqNo, created, newLocation); + Engine.IndexResult indexResult = new IndexResultWithLocation(version, 0L, seqNo, created, newLocation); results = new BulkItemResultHolder(indexResponse, indexResult, replicaRequest); assertThat(TransportShardBulkAction.calculateTranslogLocation(original, results), equalTo(newLocation)); @@ -629,8 +629,8 @@ public class TransportShardBulkActionTests extends IndexShardTestCase { public class IndexResultWithLocation extends Engine.IndexResult { private final Translog.Location location; - public IndexResultWithLocation(long version, long seqNo, boolean created, Translog.Location newLocation) { - super(version, seqNo, created); + public IndexResultWithLocation(long version, long term, long seqNo, boolean created, Translog.Location newLocation) { + super(version, term, seqNo, created); this.location = newLocation; } @@ -647,8 +647,8 @@ public class TransportShardBulkActionTests extends IndexShardTestCase { BulkItemRequest request = new BulkItemRequest(0, updateRequest); Exception err = new VersionConflictEngineException(shardId, "_doc", "id", "I'm conflicted <(;_;)>"); - Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0); - Engine.DeleteResult deleteResult = new Engine.DeleteResult(1, 1, true); + Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0, 0); + Engine.DeleteResult deleteResult = new Engine.DeleteResult(1, 1, 1, true); DocWriteResponse.Result docWriteResult = DocWriteResponse.Result.CREATED; DocWriteResponse.Result deleteWriteResult = DocWriteResponse.Result.DELETED; IndexRequest indexRequest = new IndexRequest("index", "_doc", "id"); @@ -830,8 +830,8 @@ public class TransportShardBulkActionTests extends IndexShardTestCase { private final Translog.Location location; - protected FakeResult(long version, long seqNo, boolean created, Translog.Location location) { - super(version, seqNo, created); + protected FakeResult(long version, long term, long seqNo, boolean created, Translog.Location location) { + super(version, term, seqNo, created); this.location = location; } diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 08301e99d6a..6756c00ea84 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -587,8 +587,6 @@ public class TransportReplicationActionTests extends ESTestCase { public void testPrimaryReference() throws Exception { final IndexShard shard = mock(IndexShard.class); - final long primaryTerm = 1 + randomInt(200); - when(shard.getPrimaryTerm()).thenReturn(primaryTerm); AtomicBoolean closed = new AtomicBoolean(); Releasable releasable = () -> { @@ -683,9 +681,9 @@ public class TransportReplicationActionTests extends ESTestCase { final IndexShard shard = mock(IndexShard.class); - when(shard.getPrimaryTerm()).thenReturn(primaryTerm); + when(shard.getPendingPrimaryTerm()).thenReturn(primaryTerm); when(shard.routingEntry()).thenReturn(routingEntry); - when(shard.isPrimaryMode()).thenReturn(true); + when(shard.isRelocatedPrimary()).thenReturn(false); IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().shardRoutingTable(shardId); Set inSyncIds = randomBoolean() ? Collections.singleton(routingEntry.allocationId().getId()) : clusterService.state().metaData().index(index).inSyncAllocationIds(0); @@ -1201,7 +1199,7 @@ public class TransportReplicationActionTests extends ESTestCase { doAnswer(invocation -> { long term = (Long)invocation.getArguments()[0]; ActionListener callback = (ActionListener) invocation.getArguments()[2]; - final long primaryTerm = indexShard.getPrimaryTerm(); + final long primaryTerm = indexShard.getPendingPrimaryTerm(); if (term < primaryTerm) { throw new IllegalArgumentException(String.format(Locale.ROOT, "%s operation term [%d] is too old (current [%d])", shardId, term, primaryTerm)); @@ -1219,9 +1217,9 @@ public class TransportReplicationActionTests extends ESTestCase { } return routing; }); - when(indexShard.isPrimaryMode()).thenAnswer(invocationOnMock -> isRelocated.get() == false); + when(indexShard.isRelocatedPrimary()).thenAnswer(invocationOnMock -> isRelocated.get()); doThrow(new AssertionError("failed shard is not supported")).when(indexShard).failShard(anyString(), any(Exception.class)); - when(indexShard.getPrimaryTerm()).thenAnswer(i -> + when(indexShard.getPendingPrimaryTerm()).thenAnswer(i -> clusterService.state().metaData().getIndexSafe(shardId.getIndex()).primaryTerm(shardId.id())); return indexShard; } diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index bfcc5938a86..d305630f83e 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -454,7 +454,7 @@ public class TransportWriteActionTests extends ESTestCase { doAnswer(invocation -> { long term = (Long)invocation.getArguments()[0]; ActionListener callback = (ActionListener) invocation.getArguments()[1]; - final long primaryTerm = indexShard.getPrimaryTerm(); + final long primaryTerm = indexShard.getPendingPrimaryTerm(); if (term < primaryTerm) { throw new IllegalArgumentException(String.format(Locale.ROOT, "%s operation term [%d] is too old (current [%d])", shardId, term, primaryTerm)); @@ -472,9 +472,9 @@ public class TransportWriteActionTests extends ESTestCase { } return routing; }); - when(indexShard.isPrimaryMode()).thenAnswer(invocationOnMock -> isRelocated.get() == false); + when(indexShard.isRelocatedPrimary()).thenAnswer(invocationOnMock -> isRelocated.get()); doThrow(new AssertionError("failed shard is not supported")).when(indexShard).failShard(anyString(), any(Exception.class)); - when(indexShard.getPrimaryTerm()).thenAnswer(i -> + when(indexShard.getPendingPrimaryTerm()).thenAnswer(i -> clusterService.state().metaData().getIndexSafe(shardId.getIndex()).primaryTerm(shardId.id())); return indexShard; } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java index aa77d7b4bf9..530e3dd3185 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java @@ -76,7 +76,7 @@ public class ShardStateIT extends ESIntegTestCase { if (indexService != null) { for (IndexShard shard : indexService) { assertThat("term mismatch for shard " + shard.shardId(), - shard.getPrimaryTerm(), equalTo(metaData.primaryTerm(shard.shardId().id()))); + shard.getPendingPrimaryTerm(), equalTo(metaData.primaryTerm(shard.shardId().id()))); } } } diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java index b05b1e5cc5c..1e2c00e5896 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java @@ -59,8 +59,10 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder; import static org.hamcrest.Matchers.anyOf; @@ -221,7 +223,7 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase } logger.info("--> promoting replica to primary " + replica1.routingEntry()); - shards.promoteReplicaToPrimary(replica1); + shards.promoteReplicaToPrimary(replica1).get(); indexRequest = new IndexRequest(index.getName(), "type", "1").source("{ \"f\": \"2\"}", XContentType.JSON); shards.index(indexRequest); shards.refresh("test"); @@ -234,6 +236,102 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase } } + public void testReplicaTermIncrementWithConcurrentPrimaryPromotion() throws Exception { + Map mappings = + Collections.singletonMap("type", "{ \"type\": { \"properties\": { \"f\": { \"type\": \"keyword\"} }}}"); + try (ReplicationGroup shards = new ReplicationGroup(buildIndexMetaData(2, mappings))) { + shards.startAll(); + long primaryPrimaryTerm = shards.getPrimary().getPendingPrimaryTerm(); + List replicas = shards.getReplicas(); + IndexShard replica1 = replicas.get(0); + IndexShard replica2 = replicas.get(1); + + shards.promoteReplicaToPrimary(replica1, (shard, listener) -> {}); + long newReplica1Term = replica1.getPendingPrimaryTerm(); + assertEquals(primaryPrimaryTerm + 1, newReplica1Term); + + assertEquals(primaryPrimaryTerm, replica2.getPendingPrimaryTerm()); + + IndexRequest indexRequest = new IndexRequest(index.getName(), "type", "1").source("{ \"f\": \"1\"}", XContentType.JSON); + BulkShardRequest replicationRequest = indexOnPrimary(indexRequest, replica1); + + CyclicBarrier barrier = new CyclicBarrier(2); + Thread t1 = new Thread(() -> { + try { + barrier.await(); + indexOnReplica(replicationRequest, shards, replica2, newReplica1Term); + } catch (IllegalStateException ise) { + assertThat(ise.getMessage(), containsString("is too old")); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Thread t2 = new Thread(() -> { + try { + barrier.await(); + shards.promoteReplicaToPrimary(replica2).get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + t2.start(); + t1.start(); + t1.join(); + t2.join(); + + assertEquals(newReplica1Term + 1, replica2.getPendingPrimaryTerm()); + } + } + + public void testReplicaOperationWithConcurrentPrimaryPromotion() throws Exception { + Map mappings = + Collections.singletonMap("type", "{ \"type\": { \"properties\": { \"f\": { \"type\": \"keyword\"} }}}"); + try (ReplicationGroup shards = new ReplicationGroup(buildIndexMetaData(1, mappings))) { + shards.startAll(); + long primaryPrimaryTerm = shards.getPrimary().getPendingPrimaryTerm(); + IndexRequest indexRequest = new IndexRequest(index.getName(), "type", "1").source("{ \"f\": \"1\"}", XContentType.JSON); + BulkShardRequest replicationRequest = indexOnPrimary(indexRequest, shards.getPrimary()); + + List replicas = shards.getReplicas(); + IndexShard replica = replicas.get(0); + + CyclicBarrier barrier = new CyclicBarrier(2); + AtomicBoolean successFullyIndexed = new AtomicBoolean(); + Thread t1 = new Thread(() -> { + try { + barrier.await(); + indexOnReplica(replicationRequest, shards, replica, primaryPrimaryTerm); + successFullyIndexed.set(true); + } catch (IllegalStateException ise) { + assertThat(ise.getMessage(), containsString("is too old")); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Thread t2 = new Thread(() -> { + try { + barrier.await(); + shards.promoteReplicaToPrimary(replica).get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + t2.start(); + t1.start(); + t1.join(); + t2.join(); + + assertEquals(primaryPrimaryTerm + 1, replica.getPendingPrimaryTerm()); + if (successFullyIndexed.get()) { + try(Translog.Snapshot snapshot = getTranslog(replica).newSnapshot()) { + assertThat(snapshot.totalOperations(), equalTo(1)); + Translog.Operation op = snapshot.next(); + assertThat(op.primaryTerm(), equalTo(primaryPrimaryTerm)); + } + } + } + } + /** * test document failures (failures after seq_no generation) are added as noop operation to the translog * for primary and replica shards @@ -255,7 +353,7 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase .source("{}", XContentType.JSON) ); assertTrue(response.isFailed()); - assertNoOpTranslogOperationForDocumentFailure(shards, 1, shards.getPrimary().getPrimaryTerm(), failureMessage); + assertNoOpTranslogOperationForDocumentFailure(shards, 1, shards.getPrimary().getPendingPrimaryTerm(), failureMessage); shards.assertAllEqual(0); // add some replicas @@ -269,7 +367,7 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase .source("{}", XContentType.JSON) ); assertTrue(response.isFailed()); - assertNoOpTranslogOperationForDocumentFailure(shards, 2, shards.getPrimary().getPrimaryTerm(), failureMessage); + assertNoOpTranslogOperationForDocumentFailure(shards, 2, shards.getPrimary().getPendingPrimaryTerm(), failureMessage); shards.assertAllEqual(0); } } @@ -361,7 +459,7 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase // Make sure that peer-recovery transfers all but non-overridden operations. IndexShard replica3 = shards.addReplica(); logger.info("--> Promote replica2 as the primary"); - shards.promoteReplicaToPrimary(replica2); + shards.promoteReplicaToPrimary(replica2).get(); logger.info("--> Recover replica3 from replica2"); recoverReplica(replica3, replica2, true); try (Translog.Snapshot snapshot = getTranslog(replica3).newSnapshot()) { diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java index f01d5e54a2e..2d198c32ba7 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java @@ -245,7 +245,7 @@ public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestC } } - shards.promoteReplicaToPrimary(newPrimary); + shards.promoteReplicaToPrimary(newPrimary).get(); // check that local checkpoint of new primary is properly tracked after primary promotion assertThat(newPrimary.getLocalCheckpoint(), equalTo(totalDocs - 1L)); @@ -432,7 +432,8 @@ public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestC while ((next = snapshot.next()) != null) { translogOperations++; assertThat("unexpected op: " + next, (int)next.seqNo(), lessThan(initialDocs + extraDocs)); - assertThat("unexpected primaryTerm: " + next.primaryTerm(), next.primaryTerm(), is(oldPrimary.getPrimaryTerm())); + assertThat("unexpected primaryTerm: " + next.primaryTerm(), next.primaryTerm(), + is(oldPrimary.getPendingPrimaryTerm())); final Translog.Source source = next.getSource(); assertThat(source.source.utf8ToString(), is("{ \"f\": \"normal\"}")); } diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java index 6fdce76912e..e001f82809b 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java @@ -770,8 +770,10 @@ public class ReplicationTrackerTests extends ESTestCase { assertThat(newPrimary.routingTable, equalTo(oldPrimary.routingTable)); assertThat(newPrimary.replicationGroup, equalTo(oldPrimary.replicationGroup)); + assertFalse(oldPrimary.relocated); oldPrimary.completeRelocationHandoff(); assertFalse(oldPrimary.primaryMode); + assertTrue(oldPrimary.relocated); } public void testIllegalStateExceptionIfUnknownAllocationId() { diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java index 27d08b76c03..23337def2ae 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java @@ -71,7 +71,7 @@ public class IndexShardOperationPermitsTests extends ESTestCase { public static void setupThreadPool() { int writeThreadPoolSize = randomIntBetween(1, 2); int writeThreadPoolQueueSize = randomIntBetween(1, 2); - threadPool = new TestThreadPool("IndexShardOperationsLockTests", + threadPool = new TestThreadPool("IndexShardOperationPermitsTests", Settings.builder() .put("thread_pool." + ThreadPool.Names.WRITE + ".size", writeThreadPoolSize) .put("thread_pool." + ThreadPool.Names.WRITE + ".queue_size", writeThreadPoolQueueSize) @@ -100,7 +100,7 @@ public class IndexShardOperationPermitsTests extends ESTestCase { assertThat(permits.getActiveOperationsCount(), equalTo(0)); } - public void testAllOperationsInvoked() throws InterruptedException, TimeoutException, ExecutionException { + public void testAllOperationsInvoked() throws InterruptedException, TimeoutException { int numThreads = 10; class DummyException extends RuntimeException {} @@ -187,7 +187,7 @@ public class IndexShardOperationPermitsTests extends ESTestCase { future.get().close(); } - public void testOperationsIfClosed() throws ExecutionException, InterruptedException { + public void testOperationsIfClosed() { PlainActionFuture future = new PlainActionFuture<>(); permits.close(); permits.acquire(future, ThreadPool.Names.GENERIC, true, ""); @@ -195,10 +195,12 @@ public class IndexShardOperationPermitsTests extends ESTestCase { assertThat(exception.getCause(), instanceOf(IndexShardClosedException.class)); } - public void testBlockIfClosed() throws ExecutionException, InterruptedException { + public void testBlockIfClosed() { permits.close(); expectThrows(IndexShardClosedException.class, () -> permits.blockOperations(randomInt(10), TimeUnit.MINUTES, () -> { throw new IllegalArgumentException("fake error"); })); + expectThrows(IndexShardClosedException.class, () -> permits.asyncBlockOperations(randomInt(10), TimeUnit.MINUTES, + () -> { throw new IllegalArgumentException("fake error"); }, e -> { throw new AssertionError(e); })); } public void testOperationsDelayedIfBlock() throws ExecutionException, InterruptedException, TimeoutException { @@ -210,6 +212,36 @@ public class IndexShardOperationPermitsTests extends ESTestCase { future.get(1, TimeUnit.HOURS).close(); } + public void testGetBlockWhenBlocked() throws ExecutionException, InterruptedException, TimeoutException { + PlainActionFuture future = new PlainActionFuture<>(); + final CountDownLatch blockAcquired = new CountDownLatch(1); + final CountDownLatch releaseBlock = new CountDownLatch(1); + final AtomicBoolean blocked = new AtomicBoolean(); + try (Releasable ignored = blockAndWait()) { + permits.acquire(future, ThreadPool.Names.GENERIC, true, ""); + + permits.asyncBlockOperations( + 30, + TimeUnit.MINUTES, + () -> { + blocked.set(true); + blockAcquired.countDown(); + releaseBlock.await(); + }, + e -> { + throw new RuntimeException(e); + }); + assertFalse(blocked.get()); + assertFalse(future.isDone()); + } + blockAcquired.await(); + assertTrue(blocked.get()); + assertFalse(future.isDone()); + releaseBlock.countDown(); + + future.get(1, TimeUnit.HOURS).close(); + } + /** * Tests that the ThreadContext is restored when a operation is executed after it has been delayed due to a block */ diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 6d9e15c52af..03442be7f06 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -297,7 +297,7 @@ public class IndexShardTests extends IndexShardTestCase { // expected } try { - indexShard.acquireReplicaOperationPermit(indexShard.getPrimaryTerm(), SequenceNumbers.UNASSIGNED_SEQ_NO, null, + indexShard.acquireReplicaOperationPermit(indexShard.getPendingPrimaryTerm(), SequenceNumbers.UNASSIGNED_SEQ_NO, null, ThreadPool.Names.WRITE, ""); fail("we should not be able to increment anymore"); } catch (IndexShardClosedException e) { @@ -308,7 +308,7 @@ public class IndexShardTests extends IndexShardTestCase { public void testRejectOperationPermitWithHigherTermWhenNotStarted() throws IOException { IndexShard indexShard = newShard(false); expectThrows(IndexShardNotStartedException.class, () -> - indexShard.acquireReplicaOperationPermit(indexShard.getPrimaryTerm() + randomIntBetween(1, 100), + indexShard.acquireReplicaOperationPermit(indexShard.getPendingPrimaryTerm() + randomIntBetween(1, 100), SequenceNumbers.UNASSIGNED_SEQ_NO, null, ThreadPool.Names.WRITE, "")); closeShards(indexShard); } @@ -331,7 +331,7 @@ public class IndexShardTests extends IndexShardTestCase { throw new RuntimeException(e); } indexShard.acquireReplicaOperationPermit( - indexShard.getPrimaryTerm(), + indexShard.getPendingPrimaryTerm(), indexShard.getGlobalCheckpoint(), new ActionListener() { @Override @@ -418,16 +418,13 @@ public class IndexShardTests extends IndexShardTestCase { } /** - * This test makes sure that people can use the shard routing entry to check whether a shard was already promoted to - * a primary. Concretely this means, that when we publish the routing entry via {@link IndexShard#routingEntry()} the following - * should have happened - * 1) Internal state (ala ReplicationTracker) have been updated - * 2) Primary term is set to the new term + * This test makes sure that people can use the shard routing entry + take an operation permit to check whether a shard was already + * promoted to a primary. */ public void testPublishingOrderOnPromotion() throws IOException, InterruptedException, BrokenBarrierException { final IndexShard indexShard = newShard(false); recoveryEmptyReplica(indexShard, randomBoolean()); - final long promotedTerm = indexShard.getPrimaryTerm() + 1; + final long promotedTerm = indexShard.getPendingPrimaryTerm() + 1; final CyclicBarrier barrier = new CyclicBarrier(2); final AtomicBoolean stop = new AtomicBoolean(); final Thread thread = new Thread(() -> { @@ -438,8 +435,12 @@ public class IndexShardTests extends IndexShardTestCase { } while(stop.get() == false) { if (indexShard.routingEntry().primary()) { - assertThat(indexShard.getPrimaryTerm(), equalTo(promotedTerm)); - assertThat(indexShard.getReplicationGroup(), notNullValue()); + assertThat(indexShard.getPendingPrimaryTerm(), equalTo(promotedTerm)); + final PlainActionFuture permitAcquiredFuture = new PlainActionFuture<>(); + indexShard.acquirePrimaryOperationPermit(permitAcquiredFuture, ThreadPool.Names.SAME, "bla"); + try (Releasable ignored = permitAcquiredFuture.actionGet()) { + assertThat(indexShard.getReplicationGroup(), notNullValue()); + } } } }); @@ -504,7 +505,7 @@ public class IndexShardTests extends IndexShardTestCase { // promote the replica final ShardRouting replicaRouting = indexShard.routingEntry(); - final long newPrimaryTerm = indexShard.getPrimaryTerm() + between(1, 10000); + final long newPrimaryTerm = indexShard.getPendingPrimaryTerm() + between(1, 10000); final ShardRouting primaryRouting = newShardRouting( replicaRouting.shardId(), @@ -558,7 +559,7 @@ public class IndexShardTests extends IndexShardTestCase { ShardRouting replicaRouting = indexShard.routingEntry(); ShardRouting primaryRouting = newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), null, true, ShardRoutingState.STARTED, replicaRouting.allocationId()); - final long newPrimaryTerm = indexShard.getPrimaryTerm() + between(1, 1000); + final long newPrimaryTerm = indexShard.getPendingPrimaryTerm() + between(1, 1000); indexShard.updateShardState(primaryRouting, newPrimaryTerm, (shard, listener) -> { assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(newPrimaryTerm)); }, 0L, @@ -568,11 +569,14 @@ public class IndexShardTests extends IndexShardTestCase { } else { indexShard = newStartedShard(true); } - final long primaryTerm = indexShard.getPrimaryTerm(); + final long primaryTerm = indexShard.getPendingPrimaryTerm(); assertEquals(0, indexShard.getActiveOperationsCount()); if (indexShard.routingEntry().isRelocationTarget() == false) { try { - indexShard.acquireReplicaOperationPermit(primaryTerm, indexShard.getGlobalCheckpoint(), null, ThreadPool.Names.WRITE, ""); + final PlainActionFuture permitAcquiredFuture = new PlainActionFuture<>(); + indexShard.acquireReplicaOperationPermit(primaryTerm, indexShard.getGlobalCheckpoint(), permitAcquiredFuture, + ThreadPool.Names.WRITE, ""); + permitAcquiredFuture.actionGet(); fail("shard shouldn't accept operations as replica"); } catch (IllegalStateException ignored) { @@ -650,7 +654,7 @@ public class IndexShardTests extends IndexShardTestCase { assertThat(e, hasToString(containsString("shard " + shardRouting + " is not a primary"))); } - final long primaryTerm = indexShard.getPrimaryTerm(); + final long primaryTerm = indexShard.getPendingPrimaryTerm(); final long translogGen = engineClosed ? -1 : getTranslog(indexShard).getGeneration().translogFileGeneration; final Releasable operation1; @@ -728,7 +732,7 @@ public class IndexShardTests extends IndexShardTestCase { ActionListener listener = new ActionListener() { @Override public void onResponse(Releasable releasable) { - assertThat(indexShard.getPrimaryTerm(), equalTo(newPrimaryTerm)); + assertThat(indexShard.getPendingPrimaryTerm(), equalTo(newPrimaryTerm)); assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(newPrimaryTerm)); assertThat(indexShard.getLocalCheckpoint(), equalTo(expectedLocalCheckpoint)); assertThat(indexShard.getGlobalCheckpoint(), equalTo(newGlobalCheckPoint)); @@ -765,7 +769,7 @@ public class IndexShardTests extends IndexShardTestCase { barrier.await(); if (indexShard.state() == IndexShardState.CREATED || indexShard.state() == IndexShardState.RECOVERING) { barrier.await(); - assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm)); + assertThat(indexShard.getPendingPrimaryTerm(), equalTo(primaryTerm)); assertFalse(onResponse.get()); assertThat(onFailure.get(), instanceOf(IndexShardNotStartedException.class)); Releasables.close(operation1); @@ -774,18 +778,19 @@ public class IndexShardTests extends IndexShardTestCase { // our operation should be blocked until the previous operations complete assertFalse(onResponse.get()); assertNull(onFailure.get()); - assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm)); + assertThat(indexShard.operationPrimaryTerm, equalTo(primaryTerm)); assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(primaryTerm)); Releasables.close(operation1); // our operation should still be blocked assertFalse(onResponse.get()); assertNull(onFailure.get()); - assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm)); + assertThat(indexShard.operationPrimaryTerm, equalTo(primaryTerm)); assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(primaryTerm)); Releasables.close(operation2); barrier.await(); // now lock acquisition should have succeeded - assertThat(indexShard.getPrimaryTerm(), equalTo(newPrimaryTerm)); + assertThat(indexShard.operationPrimaryTerm, equalTo(newPrimaryTerm)); + assertThat(indexShard.getPendingPrimaryTerm(), equalTo(newPrimaryTerm)); assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(newPrimaryTerm)); if (engineClosed) { assertFalse(onResponse.get()); @@ -884,7 +889,7 @@ public class IndexShardTests extends IndexShardTestCase { final CountDownLatch latch = new CountDownLatch(1); indexShard.acquireReplicaOperationPermit( - indexShard.getPrimaryTerm() + 1, + indexShard.getPendingPrimaryTerm() + 1, globalCheckpoint, new ActionListener() { @Override @@ -906,7 +911,7 @@ public class IndexShardTests extends IndexShardTestCase { final CountDownLatch resyncLatch = new CountDownLatch(1); indexShard.updateShardState( newRouting, - indexShard.getPrimaryTerm() + 1, + indexShard.getPendingPrimaryTerm() + 1, (s, r) -> resyncLatch.countDown(), 1L, Collections.singleton(newRouting.allocationId().getId()), @@ -938,7 +943,7 @@ public class IndexShardTests extends IndexShardTestCase { Math.toIntExact(indexShard.getLocalCheckpoint())); final CountDownLatch latch = new CountDownLatch(1); indexShard.acquireReplicaOperationPermit( - indexShard.primaryTerm + 1, + indexShard.pendingPrimaryTerm + 1, globalCheckpoint, new ActionListener() { @Override @@ -975,7 +980,7 @@ public class IndexShardTests extends IndexShardTestCase { final CyclicBarrier barrier = new CyclicBarrier(3); final CountDownLatch latch = new CountDownLatch(2); - final long primaryTerm = indexShard.getPrimaryTerm(); + final long primaryTerm = indexShard.getPendingPrimaryTerm(); final AtomicLong counter = new AtomicLong(); final AtomicReference onFailure = new AtomicReference<>(); @@ -993,7 +998,7 @@ public class IndexShardTests extends IndexShardTestCase { @Override public void onResponse(Releasable releasable) { counter.incrementAndGet(); - assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm + increment)); + assertThat(indexShard.getPendingPrimaryTerm(), equalTo(primaryTerm + increment)); latch.countDown(); releasable.close(); } @@ -1037,7 +1042,7 @@ public class IndexShardTests extends IndexShardTestCase { assertThat(counter.get(), equalTo(2L)); } - assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm + Math.max(firstIncrement, secondIncrement))); + assertThat(indexShard.getPendingPrimaryTerm(), equalTo(primaryTerm + Math.max(firstIncrement, secondIncrement))); closeShards(indexShard); } @@ -1416,14 +1421,14 @@ public class IndexShardTests extends IndexShardTestCase { recoveryThread.start(); latch.await(); // recovery can only be finalized after we release the current primaryOperationLock - assertTrue(shard.isPrimaryMode()); + assertFalse(shard.isRelocatedPrimary()); } // recovery can be now finalized recoveryThread.join(); - assertFalse(shard.isPrimaryMode()); + assertTrue(shard.isRelocatedPrimary()); try (Releasable ignored = acquirePrimaryOperationPermitBlockingly(shard)) { // lock can again be acquired - assertFalse(shard.isPrimaryMode()); + assertTrue(shard.isRelocatedPrimary()); } closeShards(shard); @@ -1465,7 +1470,7 @@ public class IndexShardTests extends IndexShardTestCase { public void testStressRelocated() throws Exception { final IndexShard shard = newStartedShard(true); - assertTrue(shard.isPrimaryMode()); + assertFalse(shard.isRelocatedPrimary()); IndexShardTestCase.updateRoutingEntry(shard, ShardRoutingHelper.relocate(shard.routingEntry(), "other_node")); final int numThreads = randomIntBetween(2, 4); Thread[] indexThreads = new Thread[numThreads]; @@ -1501,14 +1506,14 @@ public class IndexShardTests extends IndexShardTestCase { assertThat(relocated.get(), equalTo(false)); assertThat(shard.getActiveOperationsCount(), greaterThan(0)); // ensure we only transition after pending operations completed - assertTrue(shard.isPrimaryMode()); + assertFalse(shard.isRelocatedPrimary()); // complete pending operations barrier.await(); // complete recovery/relocation recoveryThread.join(); // ensure relocated successfully once pending operations are done assertThat(relocated.get(), equalTo(true)); - assertFalse(shard.isPrimaryMode()); + assertTrue(shard.isRelocatedPrimary()); assertThat(shard.getActiveOperationsCount(), equalTo(0)); for (Thread indexThread : indexThreads) { @@ -1572,7 +1577,7 @@ public class IndexShardTests extends IndexShardTestCase { cyclicBarrier.await(); relocationThread.join(); cancellingThread.join(); - if (shard.isPrimaryMode() == false) { + if (shard.isRelocatedPrimary()) { logger.debug("shard was relocated successfully"); assertThat(cancellingException.get(), instanceOf(IllegalIndexShardStateException.class)); assertThat("current routing:" + shard.routingEntry(), shard.routingEntry().relocating(), equalTo(true)); @@ -1719,7 +1724,7 @@ public class IndexShardTests extends IndexShardTestCase { while ((operation = snapshot.next()) != null) { if (operation.opType() == Translog.Operation.Type.NO_OP) { numNoops++; - assertEquals(newShard.getPrimaryTerm(), operation.primaryTerm()); + assertEquals(newShard.getPendingPrimaryTerm(), operation.primaryTerm()); assertEquals(0, operation.seqNo()); } } @@ -1826,7 +1831,7 @@ public class IndexShardTests extends IndexShardTestCase { flushShard(shard); assertThat(getShardDocUIDs(shard), containsInAnyOrder("doc-0", "doc-1")); // Simulate resync (without rollback): Noop #1, index #2 - acquireReplicaOperationPermitBlockingly(shard, shard.primaryTerm + 1); + acquireReplicaOperationPermitBlockingly(shard, shard.pendingPrimaryTerm + 1); shard.markSeqNoAsNoop(1, "test"); shard.applyIndexOperationOnReplica(2, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, SourceToParse.source(indexName, "_doc", "doc-2", new BytesArray("{}"), XContentType.JSON)); @@ -1837,7 +1842,8 @@ public class IndexShardTests extends IndexShardTestCase { IndexShard newShard = reinitShard(shard, newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE)); - newShard.primaryTerm++; + newShard.pendingPrimaryTerm++; + newShard.operationPrimaryTerm++; DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue(newShard.recoverFromStore()); @@ -1852,7 +1858,7 @@ public class IndexShardTests extends IndexShardTestCase { ShardRouting inRecoveryRouting = ShardRoutingHelper.relocate(origRouting, "some_node"); IndexShardTestCase.updateRoutingEntry(shard, inRecoveryRouting); shard.relocated(primaryContext -> {}); - assertFalse(shard.isPrimaryMode()); + assertTrue(shard.isRelocatedPrimary()); try { IndexShardTestCase.updateRoutingEntry(shard, origRouting); fail("Expected IndexShardRelocatedException"); @@ -2160,11 +2166,11 @@ public class IndexShardTests extends IndexShardTestCase { int numCorruptEntries = 0; for (int i = 0; i < numTotalEntries; i++) { if (randomBoolean()) { - operations.add(new Translog.Index("_doc", "1", 0, primary.getPrimaryTerm(), 1, + operations.add(new Translog.Index("_doc", "1", 0, primary.getPendingPrimaryTerm(), 1, "{\"foo\" : \"bar\"}".getBytes(Charset.forName("UTF-8")), null, -1)); } else { // corrupt entry - operations.add(new Translog.Index("_doc", "2", 1, primary.getPrimaryTerm(), 1, + operations.add(new Translog.Index("_doc", "2", 1, primary.getPendingPrimaryTerm(), 1, "{\"foo\" : \"bar}".getBytes(Charset.forName("UTF-8")), null, -1)); numCorruptEntries++; } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java index 037b5d68fb8..7baae06b671 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java @@ -148,7 +148,7 @@ public class IndexingOperationListenerTests extends ESTestCase{ ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null); Engine.Delete delete = new Engine.Delete("test", "1", new Term("_id", Uid.encodeId(doc.id())), randomNonNegativeLong()); Engine.Index index = new Engine.Index(new Term("_id", Uid.encodeId(doc.id())), randomNonNegativeLong(), doc); - compositeListener.postDelete(randomShardId, delete, new Engine.DeleteResult(1, SequenceNumbers.UNASSIGNED_SEQ_NO, true)); + compositeListener.postDelete(randomShardId, delete, new Engine.DeleteResult(1, 0, SequenceNumbers.UNASSIGNED_SEQ_NO, true)); assertEquals(0, preIndex.get()); assertEquals(0, postIndex.get()); assertEquals(0, postIndexException.get()); @@ -172,7 +172,7 @@ public class IndexingOperationListenerTests extends ESTestCase{ assertEquals(2, postDelete.get()); assertEquals(2, postDeleteException.get()); - compositeListener.postIndex(randomShardId, index, new Engine.IndexResult(0, SequenceNumbers.UNASSIGNED_SEQ_NO, false)); + compositeListener.postIndex(randomShardId, index, new Engine.IndexResult(0, 0, SequenceNumbers.UNASSIGNED_SEQ_NO, false)); assertEquals(0, preIndex.get()); assertEquals(2, postIndex.get()); assertEquals(0, postIndexException.get()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java b/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java index 4444f475329..ae2cc84e487 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java @@ -83,7 +83,7 @@ public class PrimaryReplicaSyncerTests extends IndexShardTestCase { boolean syncNeeded = numDocs > 0; String allocationId = shard.routingEntry().allocationId().getId(); - shard.updateShardState(shard.routingEntry(), shard.getPrimaryTerm(), null, 1000L, Collections.singleton(allocationId), + shard.updateShardState(shard.routingEntry(), shard.getPendingPrimaryTerm(), null, 1000L, Collections.singleton(allocationId), new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build(), Collections.emptySet()); shard.updateLocalCheckpointForShard(allocationId, globalCheckPoint); assertEquals(globalCheckPoint, shard.getGlobalCheckpoint()); @@ -142,7 +142,7 @@ public class PrimaryReplicaSyncerTests extends IndexShardTestCase { } String allocationId = shard.routingEntry().allocationId().getId(); - shard.updateShardState(shard.routingEntry(), shard.getPrimaryTerm(), null, 1000L, Collections.singleton(allocationId), + shard.updateShardState(shard.routingEntry(), shard.getPendingPrimaryTerm(), null, 1000L, Collections.singleton(allocationId), new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build(), Collections.emptySet()); CountDownLatch syncCalledLatch = new CountDownLatch(1); diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index b255238c864..dc0d871a7f2 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -2669,7 +2669,7 @@ public class TranslogTests extends ESTestCase { Engine.Index eIndex = new Engine.Index(newUid(doc), doc, randomSeqNum, randomPrimaryTerm, 1, VersionType.INTERNAL, Origin.PRIMARY, 0, 0, false); - Engine.IndexResult eIndexResult = new Engine.IndexResult(1, randomSeqNum, true); + Engine.IndexResult eIndexResult = new Engine.IndexResult(1, randomPrimaryTerm, randomSeqNum, true); Translog.Index index = new Translog.Index(eIndex, eIndexResult); BytesStreamOutput out = new BytesStreamOutput(); @@ -2680,7 +2680,7 @@ public class TranslogTests extends ESTestCase { Engine.Delete eDelete = new Engine.Delete(doc.type(), doc.id(), newUid(doc), randomSeqNum, randomPrimaryTerm, 2, VersionType.INTERNAL, Origin.PRIMARY, 0); - Engine.DeleteResult eDeleteResult = new Engine.DeleteResult(2, randomSeqNum, true); + Engine.DeleteResult eDeleteResult = new Engine.DeleteResult(2, randomPrimaryTerm, randomSeqNum, true); Translog.Delete delete = new Translog.Delete(eDelete, eDeleteResult); out = new BytesStreamOutput(); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 6be6d7e80bc..f0644b029c3 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -179,12 +179,12 @@ public class RecoverySourceHandlerTests extends ESTestCase { final int initialNumberOfDocs = randomIntBetween(16, 64); for (int i = 0; i < initialNumberOfDocs; i++) { final Engine.Index index = getIndex(Integer.toString(i)); - operations.add(new Translog.Index(index, new Engine.IndexResult(1, SequenceNumbers.UNASSIGNED_SEQ_NO, true))); + operations.add(new Translog.Index(index, new Engine.IndexResult(1, 1, SequenceNumbers.UNASSIGNED_SEQ_NO, true))); } final int numberOfDocsWithValidSequenceNumbers = randomIntBetween(16, 64); for (int i = initialNumberOfDocs; i < initialNumberOfDocs + numberOfDocsWithValidSequenceNumbers; i++) { final Engine.Index index = getIndex(Integer.toString(i)); - operations.add(new Translog.Index(index, new Engine.IndexResult(1, i - initialNumberOfDocs, true))); + operations.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i - initialNumberOfDocs, true))); } operations.add(null); final long startingSeqNo = randomIntBetween(0, numberOfDocsWithValidSequenceNumbers - 1); @@ -395,7 +395,7 @@ public class RecoverySourceHandlerTests extends ESTestCase { final IndexShard shard = mock(IndexShard.class); when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class)); when(shard.segmentStats(anyBoolean())).thenReturn(mock(SegmentsStats.class)); - when(shard.isPrimaryMode()).thenReturn(false); + when(shard.isRelocatedPrimary()).thenReturn(true); when(shard.acquireSafeIndexCommit()).thenReturn(mock(Engine.IndexCommitRef.class)); doAnswer(invocation -> { ((ActionListener)invocation.getArguments()[0]).onResponse(() -> {}); @@ -444,7 +444,7 @@ public class RecoverySourceHandlerTests extends ESTestCase { final CancellableThreads cancellableThreads = new CancellableThreads(); final IndexShard shard = mock(IndexShard.class); final AtomicBoolean freed = new AtomicBoolean(true); - when(shard.isPrimaryMode()).thenReturn(true); + when(shard.isRelocatedPrimary()).thenReturn(false); doAnswer(invocation -> { freed.set(false); ((ActionListener)invocation.getArguments()[0]).onResponse(() -> freed.set(true)); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java index aaba17c3151..99848f64571 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java @@ -201,7 +201,7 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase { if (randomBoolean()) { // create a new translog translogUUIDtoUse = Translog.createEmptyTranslog(replica.shardPath().resolveTranslog(), flushedDocs, - replica.shardId(), replica.getPrimaryTerm()); + replica.shardId(), replica.getPendingPrimaryTerm()); translogGenToUse = 1; } else { translogUUIDtoUse = translogGeneration.translogUUID; diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index 2b1841c39ae..a40f950b02e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -84,6 +84,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Future; import java.util.concurrent.FutureTask; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -233,7 +234,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase activeIds.add(primary.routingEntry().allocationId().getId()); ShardRouting startedRoutingEntry = ShardRoutingHelper.moveToStarted(primary.routingEntry()); IndexShardRoutingTable routingTable = routingTable(shr -> shr == primary.routingEntry() ? startedRoutingEntry : shr); - primary.updateShardState(startedRoutingEntry, primary.getPrimaryTerm(), null, + primary.updateShardState(startedRoutingEntry, primary.getPendingPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(), activeIds, routingTable, Collections.emptySet()); for (final IndexShard replica : replicas) { recoverReplica(replica); @@ -279,20 +280,10 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase /** * promotes the specific replica as the new primary */ - public synchronized Future promoteReplicaToPrimary(IndexShard replica) throws IOException { - final long newTerm = indexMetaData.primaryTerm(shardId.id()) + 1; - IndexMetaData.Builder newMetaData = IndexMetaData.builder(indexMetaData).primaryTerm(shardId.id(), newTerm); - indexMetaData = newMetaData.build(); - assertTrue(replicas.remove(replica)); - closeShards(primary); - primary = replica; - assert primary.routingEntry().active() : "only active replicas can be promoted to primary: " + primary.routingEntry(); + public Future promoteReplicaToPrimary(IndexShard replica) throws IOException { PlainActionFuture fut = new PlainActionFuture<>(); - ShardRouting primaryRouting = replica.routingEntry().moveActiveReplicaToPrimary(); - IndexShardRoutingTable routingTable = routingTable(shr -> shr == replica.routingEntry() ? primaryRouting : shr); - - primary.updateShardState(primaryRouting, - newTerm, (shard, listener) -> primaryReplicaSyncer.resync(shard, + promoteReplicaToPrimary(replica, + (shard, listener) -> primaryReplicaSyncer.resync(shard, new ActionListener() { @Override public void onResponse(PrimaryReplicaSyncer.ResyncTask resyncTask) { @@ -305,11 +296,27 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase listener.onFailure(e); fut.onFailure(e); } - }), currentClusterStateVersion.incrementAndGet(), activeIds(), routingTable, Collections.emptySet()); - + })); return fut; } + public synchronized void promoteReplicaToPrimary(IndexShard replica, + BiConsumer> primaryReplicaSyncer) + throws IOException { + final long newTerm = indexMetaData.primaryTerm(shardId.id()) + 1; + IndexMetaData.Builder newMetaData = IndexMetaData.builder(indexMetaData).primaryTerm(shardId.id(), newTerm); + indexMetaData = newMetaData.build(); + assertTrue(replicas.remove(replica)); + closeShards(primary); + primary = replica; + assert primary.routingEntry().active() : "only active replicas can be promoted to primary: " + primary.routingEntry(); + ShardRouting primaryRouting = replica.routingEntry().moveActiveReplicaToPrimary(); + IndexShardRoutingTable routingTable = routingTable(shr -> shr == replica.routingEntry() ? primaryRouting : shr); + + primary.updateShardState(primaryRouting, newTerm, primaryReplicaSyncer, currentClusterStateVersion.incrementAndGet(), + activeIds(), routingTable, Collections.emptySet()); + } + private synchronized Set activeIds() { return shardRoutings().stream() .filter(ShardRouting::active).map(ShardRouting::allocationId).map(AllocationId::getId).collect(Collectors.toSet()); @@ -425,7 +432,8 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase private void updateAllocationIDsOnPrimary() throws IOException { - primary.updateShardState(primary.routingEntry(), primary.getPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(), + primary.updateShardState(primary.routingEntry(), primary.getPendingPrimaryTerm(), null, + currentClusterStateVersion.incrementAndGet(), activeIds(), routingTable(Function.identity()), Collections.emptySet()); } } @@ -527,7 +535,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase IndexShard replica = replicationGroup.replicas.stream() .filter(s -> replicaRouting.isSameAllocation(s.routingEntry())).findFirst().get(); replica.acquireReplicaOperationPermit( - replicationGroup.primary.getPrimaryTerm(), + replicationGroup.primary.getPendingPrimaryTerm(), globalCheckpoint, new ActionListener() { @Override @@ -605,7 +613,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase @Override protected void performOnReplica(BulkShardRequest request, IndexShard replica) throws Exception { - executeShardBulkOnReplica(request, replica, getPrimaryShard().getPrimaryTerm(), getPrimaryShard().getGlobalCheckpoint()); + executeShardBulkOnReplica(request, replica, getPrimaryShard().getPendingPrimaryTerm(), getPrimaryShard().getGlobalCheckpoint()); } } @@ -664,14 +672,18 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase * indexes the given requests on the supplied replica shard */ void indexOnReplica(BulkShardRequest request, ReplicationGroup group, IndexShard replica) throws Exception { - executeShardBulkOnReplica(request, replica, group.primary.getPrimaryTerm(), group.primary.getGlobalCheckpoint()); + indexOnReplica(request, group, replica, group.primary.getPendingPrimaryTerm()); + } + + void indexOnReplica(BulkShardRequest request, ReplicationGroup group, IndexShard replica, long term) throws Exception { + executeShardBulkOnReplica(request, replica, term, group.primary.getGlobalCheckpoint()); } /** * Executes the delete request on the given replica shard. */ void deleteOnReplica(BulkShardRequest request, ReplicationGroup group, IndexShard replica) throws Exception { - executeShardBulkOnReplica(request, replica, group.primary.getPrimaryTerm(), group.primary.getGlobalCheckpoint()); + executeShardBulkOnReplica(request, replica, group.primary.getPendingPrimaryTerm(), group.primary.getGlobalCheckpoint()); } class GlobalCheckpointSync extends ReplicationAction< diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index f9289f65861..d2a84589669 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -425,7 +425,7 @@ public abstract class IndexShardTestCase extends ESTestCase { IndexShardRoutingTable newRoutingTable = new IndexShardRoutingTable.Builder(shardRouting.shardId()) .addShard(shardRouting) .build(); - shard.updateShardState(shardRouting, shard.getPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(), + shard.updateShardState(shardRouting, shard.getPendingPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(), inSyncIds, newRoutingTable, Collections.emptySet()); } @@ -514,8 +514,8 @@ public abstract class IndexShardTestCase extends ESTestCase { request, (int) ByteSizeUnit.MB.toBytes(1), Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), pNode.getName()).build()); - primary.updateShardState(primary.routingEntry(), primary.getPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(), - inSyncIds, routingTable, Collections.emptySet()); + primary.updateShardState(primary.routingEntry(), primary.getPendingPrimaryTerm(), null, + currentClusterStateVersion.incrementAndGet(), inSyncIds, routingTable, Collections.emptySet()); recovery.recoverToTarget(); recoveryTarget.markAsDone(); } @@ -536,9 +536,9 @@ public abstract class IndexShardTestCase extends ESTestCase { Set inSyncIdsWithReplica = new HashSet<>(inSyncIds); inSyncIdsWithReplica.add(replica.routingEntry().allocationId().getId()); // update both primary and replica shard state - primary.updateShardState(primary.routingEntry(), primary.getPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(), - inSyncIdsWithReplica, newRoutingTable, Collections.emptySet()); - replica.updateShardState(replica.routingEntry().moveToStarted(), replica.getPrimaryTerm(), null, + primary.updateShardState(primary.routingEntry(), primary.getPendingPrimaryTerm(), null, + currentClusterStateVersion.incrementAndGet(), inSyncIdsWithReplica, newRoutingTable, Collections.emptySet()); + replica.updateShardState(replica.routingEntry().moveToStarted(), replica.getPendingPrimaryTerm(), null, currentClusterStateVersion.get(), inSyncIdsWithReplica, newRoutingTable, Collections.emptySet()); } @@ -560,7 +560,7 @@ public abstract class IndexShardTestCase extends ESTestCase { .removeShard(replica.routingEntry()) .addShard(routingEntry) .build(); - replica.updateShardState(routingEntry, replica.getPrimaryTerm() + 1, + replica.updateShardState(routingEntry, replica.getPendingPrimaryTerm() + 1, (is, listener) -> listener.onResponse(new PrimaryReplicaSyncer.ResyncTask(1, "type", "action", "desc", null, Collections.emptyMap())), currentClusterStateVersion.incrementAndGet(), From f809d6fff415626b5b9d4f393f9caa638be7f000 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Fri, 3 Aug 2018 09:41:29 +0200 Subject: [PATCH 08/11] Tests: Add rolling upgrade tests for watcher (#32428) These tests ensure, that the basic watch APIs are tested in the rolling upgrade tests. After initially adding a watch, the tests try to get, execute, deactivate and activate a watch. Watcher stats are tested as well, and an own java based test has been added for restarting, as that requires waiting for a state change. Watcher history is also checked. Closes #31216 --- .../upgrades/WatcherRestartIT.java | 48 ++++++++++ .../test/mixed_cluster/60_watcher.yml | 74 +++++++++++++++ .../test/old_cluster/60_watcher.yml | 94 +++++++++++++++++++ .../test/upgraded_cluster/60_watcher.yml | 73 ++++++++++++++ 4 files changed, 289 insertions(+) create mode 100644 x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java create mode 100644 x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_watcher.yml create mode 100644 x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_watcher.yml create mode 100644 x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_watcher.yml diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java new file mode 100644 index 00000000000..395ab375bd2 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.upgrades; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; + +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; + +public class WatcherRestartIT extends AbstractUpgradeTestCase { + + public void testWatcherRestart() throws Exception { + client().performRequest(new Request("POST", "/_xpack/watcher/_stop")); + ensureWatcherStopped(); + + client().performRequest(new Request("POST", "/_xpack/watcher/_start")); + ensureWatcherStarted(); + } + + private void ensureWatcherStopped() throws Exception { + assertBusy(() -> { + Response stats = client().performRequest(new Request("GET", "_xpack/watcher/stats")); + String responseBody = EntityUtils.toString(stats.getEntity(), StandardCharsets.UTF_8); + assertThat(responseBody, containsString("\"watcher_state\":\"stopped\"")); + assertThat(responseBody, not(containsString("\"watcher_state\":\"starting\""))); + assertThat(responseBody, not(containsString("\"watcher_state\":\"started\""))); + assertThat(responseBody, not(containsString("\"watcher_state\":\"stopping\""))); + }); + } + + private void ensureWatcherStarted() throws Exception { + assertBusy(() -> { + Response response = client().performRequest(new Request("GET", "_xpack/watcher/stats")); + String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + assertThat(responseBody, containsString("\"watcher_state\":\"started\"")); + assertThat(responseBody, not(containsString("\"watcher_state\":\"starting\""))); + assertThat(responseBody, not(containsString("\"watcher_state\":\"stopping\""))); + assertThat(responseBody, not(containsString("\"watcher_state\":\"stopped\""))); + }); + } +} diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_watcher.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_watcher.yml new file mode 100644 index 00000000000..30a22e63673 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_watcher.yml @@ -0,0 +1,74 @@ +--- +"CRUD watch APIs": + + # no need to put watch, exists already + - do: + xpack.watcher.get_watch: + id: "my_watch" + - match: { found : true} + - match: { _id: "my_watch" } + + + # execute watch + - do: + xpack.watcher.execute_watch: + id: "my_watch" + body: > + { + "record_execution" : true + } + - set: { _id : record_id } + - match: { watch_record.watch_id: "my_watch" } + - match: { watch_record.trigger_event.type: "manual" } + - match: { watch_record.state: "executed" } + - match: { watch_record.status.execution_state: "executed" } + - match: { watch_record.status.state.active: true } + + # check watch history entry + - do: + indices.refresh: + index: .watcher-history-* + + - do: + search: + index: .watcher-history-* + body: + { + "query" : { "term" : { "_id" : "$record_id" } } + } + - match: { hits.total: 1 } + + # deactivate watch, check with GET API as well + - do: + xpack.watcher.deactivate_watch: + watch_id: "my_watch" + - match: { status.state.active : false } + + - do: + xpack.watcher.get_watch: + id: "my_watch" + - match: { found : true} + - match: { _id: "my_watch" } + - match: { status.state.active: false } + + + # activate watch again, check with GET API as well + - do: + xpack.watcher.activate_watch: + watch_id: "my_watch" + - match: { status.state.active : true } + + - do: + xpack.watcher.get_watch: + id: "my_watch" + - match: { found : true} + - match: { _id: "my_watch" } + - match: { status.state.active: true } + + +--- +"Test watcher stats output": + - do: + xpack.watcher.stats: {} + - match: { "manually_stopped": false } + - match: { "stats.0.watcher_state": "started" } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_watcher.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_watcher.yml new file mode 100644 index 00000000000..2547ddd7023 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_watcher.yml @@ -0,0 +1,94 @@ +--- +"CRUD watch APIs": + + - do: + xpack.watcher.put_watch: + id: "my_watch" + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "simple": {} + }, + "actions": { + "logging": { + "logging": { + "text": "my logging action" + } + } + } + } + - match: { _id: "my_watch" } + + - do: + xpack.watcher.get_watch: + id: "my_watch" + - match: { found : true} + - match: { _id: "my_watch" } + + + # execute watch + - do: + xpack.watcher.execute_watch: + id: "my_watch" + body: > + { + "record_execution" : true + } + - set: { _id : record_id } + - match: { watch_record.watch_id: "my_watch" } + - match: { watch_record.trigger_event.type: "manual" } + - match: { watch_record.state: "executed" } + - match: { watch_record.status.execution_state: "executed" } + - match: { watch_record.status.state.active: true } + + # check watch history entry + - do: + indices.refresh: + index: .watcher-history-* + + - do: + search: + index: .watcher-history-* + body: + { + "query" : { "term" : { "_id" : "$record_id" } } + } + - match: { hits.total: 1 } + + # deactivate watch, check with GET API as well + - do: + xpack.watcher.deactivate_watch: + watch_id: "my_watch" + - match: { status.state.active : false } + + - do: + xpack.watcher.get_watch: + id: "my_watch" + - match: { found : true} + - match: { _id: "my_watch" } + - match: { status.state.active: false } + + + # activate watch again, check with GET API as well + - do: + xpack.watcher.activate_watch: + watch_id: "my_watch" + - match: { status.state.active : true } + + - do: + xpack.watcher.get_watch: + id: "my_watch" + - match: { found : true} + - match: { _id: "my_watch" } + - match: { status.state.active: true } + + +--- +"Test watcher stats output": + - do: + xpack.watcher.stats: {} + - match: { "manually_stopped": false } + - match: { "stats.0.watcher_state": "started" } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_watcher.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_watcher.yml new file mode 100644 index 00000000000..75e31c62a84 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_watcher.yml @@ -0,0 +1,73 @@ +--- +"CRUD watch APIs": + + # no need to put watch, exists already + - do: + xpack.watcher.get_watch: + id: "my_watch" + - match: { found : true} + - match: { _id: "my_watch" } + + + # execute watch + - do: + xpack.watcher.execute_watch: + id: "my_watch" + body: > + { + "record_execution" : true + } + - set: { _id : record_id } + - match: { watch_record.watch_id: "my_watch" } + - match: { watch_record.trigger_event.type: "manual" } + - match: { watch_record.state: "executed" } + - match: { watch_record.status.execution_state: "executed" } + - match: { watch_record.status.state.active: true } + + # check watch history entry + - do: + indices.refresh: + index: .watcher-history-* + + - do: + search: + index: .watcher-history-* + body: + { + "query" : { "term" : { "_id" : "$record_id" } } + } + - match: { hits.total: 1 } + + # deactivate watch, check with GET API as well + - do: + xpack.watcher.deactivate_watch: + watch_id: "my_watch" + - match: { status.state.active : false } + + - do: + xpack.watcher.get_watch: + id: "my_watch" + - match: { found : true} + - match: { _id: "my_watch" } + - match: { status.state.active: false } + + + # activate watch again, check with GET API as well + - do: + xpack.watcher.activate_watch: + watch_id: "my_watch" + - match: { status.state.active : true } + + - do: + xpack.watcher.get_watch: + id: "my_watch" + - match: { found : true} + - match: { _id: "my_watch" } + - match: { status.state.active: true } + +--- +"Test watcher stats output": + - do: + xpack.watcher.stats: {} + - match: { "manually_stopped": false } + - match: { "stats.0.watcher_state": "started" } From 937dcfd716de906e007152aa2e0d9dee2b779672 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 3 Aug 2018 10:01:20 +0200 Subject: [PATCH 09/11] [Rollup] Remove builders from MetricConfig (#32536) Related to #29827 --- .../xpack/core/rollup/job/MetricConfig.java | 126 ++++++++---------- .../core/rollup/job/RollupJobConfig.java | 8 +- .../xpack/core/rollup/ConfigTestHelpers.java | 66 ++++----- ...java => MetricConfigSerializingTests.java} | 83 ++++-------- .../rollup/RollupJobIdentifierUtilTests.java | 54 ++------ .../rollup/RollupRequestTranslationTests.java | 8 +- .../rollup/action/SearchActionTests.java | 2 +- .../xpack/rollup/config/ConfigTests.java | 35 ++--- .../xpack/rollup/job/IndexerUtilsTests.java | 20 ++- .../job/RollupIndexerIndexingTests.java | 6 +- 10 files changed, 155 insertions(+), 253 deletions(-) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/{MetricsConfigSerializingTests.java => MetricConfigSerializingTests.java} (69%) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java index 006d8c35c32..1843db5b3c3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java @@ -12,9 +12,10 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; @@ -32,6 +33,8 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + /** * The configuration object for the metrics portion of a rollup job config * @@ -48,14 +51,7 @@ import java.util.stream.Collectors; * ] * } */ -public class MetricConfig implements Writeable, ToXContentFragment { - private static final String NAME = "metric_config"; - - private String field; - private List metrics; - - private static final ParseField FIELD = new ParseField("field"); - private static final ParseField METRICS = new ParseField("metrics"); +public class MetricConfig implements Writeable, ToXContentObject { // TODO: replace these with an enum private static final ParseField MIN = new ParseField("min"); @@ -64,27 +60,54 @@ public class MetricConfig implements Writeable, ToXContentFragment { private static final ParseField AVG = new ParseField("avg"); private static final ParseField VALUE_COUNT = new ParseField("value_count"); - public static final ObjectParser PARSER = new ObjectParser<>(NAME, MetricConfig.Builder::new); - + private static final String NAME = "metrics"; + private static final String FIELD = "field"; + private static final String METRICS = "metrics"; + private static final ConstructingObjectParser PARSER; static { - PARSER.declareString(MetricConfig.Builder::setField, FIELD); - PARSER.declareStringArray(MetricConfig.Builder::setMetrics, METRICS); + PARSER = new ConstructingObjectParser<>(NAME, args -> { + @SuppressWarnings("unchecked") List metrics = (List) args[1]; + return new MetricConfig((String) args[0], metrics); + }); + PARSER.declareString(constructorArg(), new ParseField(FIELD)); + PARSER.declareStringArray(constructorArg(), new ParseField(METRICS)); } - MetricConfig(String name, List metrics) { - this.field = name; + private final String field; + private final List metrics; + + public MetricConfig(final String field, final List metrics) { + if (field == null || field.isEmpty()) { + throw new IllegalArgumentException("Field must be a non-null, non-empty string"); + } + if (metrics == null || metrics.isEmpty()) { + throw new IllegalArgumentException("Metrics must be a non-null, non-empty array of strings"); + } + metrics.forEach(m -> { + if (RollupField.SUPPORTED_METRICS.contains(m) == false) { + throw new IllegalArgumentException("Unsupported metric [" + m + "]. " + + "Supported metrics include: " + RollupField.SUPPORTED_METRICS); + } + }); + this.field = field; this.metrics = metrics; } - MetricConfig(StreamInput in) throws IOException { + MetricConfig(final StreamInput in) throws IOException { field = in.readString(); metrics = in.readList(StreamInput::readString); } + /** + * @return the name of the field used in the metric configuration. Never {@code null}. + */ public String getField() { return field; } + /** + * @return the names of the metrics used in the metric configuration. Never {@code null}. + */ public List getMetrics() { return metrics; } @@ -159,10 +182,13 @@ public class MetricConfig implements Writeable, ToXContentFragment { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(FIELD.getPreferredName(), field); - builder.field(METRICS.getPreferredName(), metrics); - return builder; + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + { + builder.field(FIELD, field); + builder.field(METRICS, metrics); + } + return builder.endObject(); } @Override @@ -172,19 +198,16 @@ public class MetricConfig implements Writeable, ToXContentFragment { } @Override - public boolean equals(Object other) { + public boolean equals(final Object other) { if (this == other) { return true; } - if (other == null || getClass() != other.getClass()) { return false; } - MetricConfig that = (MetricConfig) other; - - return Objects.equals(this.field, that.field) - && Objects.equals(this.metrics, that.metrics); + final MetricConfig that = (MetricConfig) other; + return Objects.equals(field, that.field) && Objects.equals(metrics, that.metrics); } @Override @@ -197,52 +220,7 @@ public class MetricConfig implements Writeable, ToXContentFragment { return Strings.toString(this, true, true); } - - public static class Builder { - private String field; - private List metrics; - - public Builder() { - } - - public Builder(MetricConfig config) { - this.field = config.getField(); - this.metrics = config.getMetrics(); - } - - public String getField() { - return field; - } - - public MetricConfig.Builder setField(String field) { - this.field = field; - return this; - } - - public List getMetrics() { - return metrics; - } - - public MetricConfig.Builder setMetrics(List metrics) { - this.metrics = metrics; - return this; - } - - public MetricConfig build() { - if (Strings.isNullOrEmpty(field) == true) { - throw new IllegalArgumentException("Parameter [" + FIELD.getPreferredName() + "] must be a non-null, non-empty string."); - } - if (metrics == null || metrics.isEmpty()) { - throw new IllegalArgumentException("Parameter [" + METRICS.getPreferredName() - + "] must be a non-null, non-empty array of strings."); - } - metrics.forEach(m -> { - if (RollupField.SUPPORTED_METRICS.contains(m) == false) { - throw new IllegalArgumentException("Unsupported metric [" + m + "]. " + - "Supported metrics include: " + RollupField.SUPPORTED_METRICS); - } - }); - return new MetricConfig(field, metrics); - } + public static MetricConfig fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java index 422ecdd5fd9..1abec72ef53 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java @@ -63,7 +63,7 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject { static { PARSER.declareString(RollupJobConfig.Builder::setId, RollupField.ID); PARSER.declareObject(RollupJobConfig.Builder::setGroupConfig, (p, c) -> GroupConfig.PARSER.apply(p,c).build(), GROUPS); - PARSER.declareObjectArray(RollupJobConfig.Builder::setMetricsConfig, (p, c) -> MetricConfig.PARSER.apply(p, c).build(), METRICS); + PARSER.declareObjectArray(RollupJobConfig.Builder::setMetricsConfig, (p, c) -> MetricConfig.fromXContent(p), METRICS); PARSER.declareString((params, val) -> params.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT); PARSER.declareString(RollupJobConfig.Builder::setIndexPattern, INDEX_PATTERN); @@ -160,10 +160,8 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject { } if (metricsConfig != null) { builder.startArray(METRICS.getPreferredName()); - for (MetricConfig config : metricsConfig) { - builder.startObject(); - config.toXContent(builder, params); - builder.endObject(); + for (MetricConfig metric : metricsConfig) { + metric.toXContent(builder, params); } builder.endArray(); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java index 36b15a1e1f5..603faafdc36 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig; import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Random; import java.util.stream.Collectors; @@ -38,11 +39,7 @@ public class ConfigTestHelpers { builder.setGroupConfig(ConfigTestHelpers.getGroupConfig().build()); builder.setPageSize(ESTestCase.randomIntBetween(1,10)); if (ESTestCase.randomBoolean()) { - List metrics = IntStream.range(1, ESTestCase.randomIntBetween(1,10)) - .mapToObj(n -> ConfigTestHelpers.getMetricConfig().build()) - .collect(Collectors.toList()); - - builder.setMetricsConfig(metrics); + builder.setMetricsConfig(randomMetricsConfigs(ESTestCase.random())); } return builder; } @@ -59,32 +56,6 @@ public class ConfigTestHelpers { return groupBuilder; } - public static MetricConfig.Builder getMetricConfig() { - MetricConfig.Builder builder = new MetricConfig.Builder(); - builder.setField(ESTestCase.randomAlphaOfLength(15)); // large names so we don't accidentally collide - List metrics = new ArrayList<>(); - if (ESTestCase.randomBoolean()) { - metrics.add("min"); - } - if (ESTestCase.randomBoolean()) { - metrics.add("max"); - } - if (ESTestCase.randomBoolean()) { - metrics.add("sum"); - } - if (ESTestCase.randomBoolean()) { - metrics.add("avg"); - } - if (ESTestCase.randomBoolean()) { - metrics.add("value_count"); - } - if (metrics.size() == 0) { - metrics.add("min"); - } - builder.setMetrics(metrics); - return builder; - } - private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m"}; public static String randomPositiveTimeValue() { return ESTestCase.randomIntBetween(1, 1000) + ESTestCase.randomFrom(TIME_SUFFIXES); @@ -123,6 +94,39 @@ public class ConfigTestHelpers { return new HistogramGroupConfig(randomInterval(random), randomFields(random)); } + public static List randomMetricsConfigs(final Random random) { + final int numMetrics = randomIntBetween(random, 1, 10); + final List metrics = new ArrayList<>(numMetrics); + for (int i = 0; i < numMetrics; i++) { + metrics.add(randomMetricConfig(random)); + } + return Collections.unmodifiableList(metrics); + } + + public static MetricConfig randomMetricConfig(final Random random) { + final String field = randomAsciiAlphanumOfLengthBetween(random, 15, 25); // large names so we don't accidentally collide + final List metrics = new ArrayList<>(); + if (random.nextBoolean()) { + metrics.add("min"); + } + if (random.nextBoolean()) { + metrics.add("max"); + } + if (random.nextBoolean()) { + metrics.add("sum"); + } + if (random.nextBoolean()) { + metrics.add("avg"); + } + if (random.nextBoolean()) { + metrics.add("value_count"); + } + if (metrics.size() == 0) { + metrics.add("min"); + } + return new MetricConfig(field, Collections.unmodifiableList(metrics)); + } + public static TermsGroupConfig randomTermsGroupConfig(final Random random) { return new TermsGroupConfig(randomFields(random)); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricsConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java similarity index 69% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricsConfigSerializingTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java index 9b330e71650..a5b8d9afead 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricsConfigSerializingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java @@ -17,14 +17,16 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class MetricsConfigSerializingTests extends AbstractSerializingTestCase { +public class MetricConfigSerializingTests extends AbstractSerializingTestCase { + @Override - protected MetricConfig doParseInstance(XContentParser parser) throws IOException { - return MetricConfig.PARSER.apply(parser, null).build(); + protected MetricConfig doParseInstance(final XContentParser parser) throws IOException { + return MetricConfig.fromXContent(parser); } @Override @@ -34,24 +36,20 @@ public class MetricsConfigSerializingTests extends AbstractSerializingTestCase> responseMap = new HashMap<>(); - MetricConfig config = new MetricConfig.Builder() - .setField("my_field") - .setMetrics(Collections.singletonList("max")) - .build(); + MetricConfig config = new MetricConfig("my_field", singletonList("max")); config.validateMappings(responseMap, e); assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] field with name [my_field] in any of the " + "indices matching the index pattern.")); } - public void testValidateNomatchingField() throws IOException { - + public void testValidateNomatchingField() { ActionRequestValidationException e = new ActionRequestValidationException(); Map> responseMap = new HashMap<>(); @@ -59,17 +57,13 @@ public class MetricsConfigSerializingTests extends AbstractSerializingTestCase> responseMap = new HashMap<>(); @@ -77,17 +71,13 @@ public class MetricsConfigSerializingTests extends AbstractSerializingTestCase> responseMap = new HashMap<>(); @@ -96,15 +86,12 @@ public class MetricsConfigSerializingTests extends AbstractSerializingTestCase> responseMap = new HashMap<>(); @@ -113,10 +100,7 @@ public class MetricsConfigSerializingTests extends AbstractSerializingTestCase caps = singletonSet(cap); @@ -168,10 +165,7 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig(); group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build()); job.setGroupConfig(group.build()); - job.setMetricsConfig(Collections.singletonList(new MetricConfig.Builder() - .setField("bar") - .setMetrics(Collections.singletonList("max")) - .build())); + job.setMetricsConfig(singletonList(new MetricConfig("bar", singletonList("max")))); RollupJobCaps cap = new RollupJobCaps(job.build()); Set caps = new HashSet<>(2); caps.add(cap); @@ -180,10 +174,7 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { GroupConfig.Builder group2 = ConfigTestHelpers.getGroupConfig(); group2.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build()); job2.setGroupConfig(group.build()); - job.setMetricsConfig(Collections.singletonList(new MetricConfig.Builder() - .setField("bar") - .setMetrics(Collections.singletonList("min")) - .build())); + job.setMetricsConfig(singletonList(new MetricConfig("bar", singletonList("min")))); RollupJobCaps cap2 = new RollupJobCaps(job2.build()); caps.add(cap2); @@ -331,12 +322,8 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { .build()) .setHisto(new HistogramGroupConfig(1L, "baz")) // <-- NOTE right type but wrong name .build()) - .setMetricsConfig(Arrays.asList(new MetricConfig.Builder() - .setField("max_field") - .setMetrics(Collections.singletonList("max")).build(), - new MetricConfig.Builder() - .setField("avg_field") - .setMetrics(Collections.singletonList("avg")).build())) + .setMetricsConfig( + Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")))) .build(); Set caps = singletonSet(new RollupJobCaps(job)); @@ -360,12 +347,8 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { .setTimeZone(DateTimeZone.UTC) .build()) .build()) - .setMetricsConfig(Arrays.asList(new MetricConfig.Builder() - .setField("max_field") - .setMetrics(Collections.singletonList("max")).build(), - new MetricConfig.Builder() - .setField("avg_field") - .setMetrics(Collections.singletonList("avg")).build())) + .setMetricsConfig( + Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")))) .build(); Set caps = singletonSet(new RollupJobCaps(job)); @@ -412,12 +395,8 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { .setTimeZone(DateTimeZone.UTC) .build()) .build()) - .setMetricsConfig(Arrays.asList(new MetricConfig.Builder() - .setField("max_field") - .setMetrics(Collections.singletonList("max")).build(), - new MetricConfig.Builder() - .setField("avg_field") - .setMetrics(Collections.singletonList("avg")).build())) + .setMetricsConfig( + Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")))) .build(); Set caps = singletonSet(new RollupJobCaps(job)); @@ -442,12 +421,8 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { .build()) .setHisto(new HistogramGroupConfig(1L, "baz")) // <-- NOTE right type but wrong name .build()) - .setMetricsConfig(Arrays.asList(new MetricConfig.Builder() - .setField("max_field") - .setMetrics(Collections.singletonList("max")).build(), - new MetricConfig.Builder() - .setField("avg_field") - .setMetrics(Collections.singletonList("avg")).build())) + .setMetricsConfig( + Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")))) .build(); Set caps = singletonSet(new RollupJobCaps(job)); @@ -485,9 +460,8 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { int i = ESTestCase.randomIntBetween(0, 3); Set caps = singletonSet(new RollupJobCaps(ConfigTestHelpers - .getRollupJob("foo").setMetricsConfig(Collections.singletonList(new MetricConfig.Builder() - .setField("foo") - .setMetrics(Arrays.asList("avg", "max", "min", "sum")).build())) + .getRollupJob("foo") + .setMetricsConfig(singletonList(new MetricConfig("foo", Arrays.asList("avg", "max", "min", "sum")))) .build())); String aggType; diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java index 9b29c907b3b..08663eb9bba 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java @@ -45,6 +45,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.rollup.RollupRequestTranslator.translateAggregation; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.IsInstanceOf.instanceOf; @@ -153,9 +154,8 @@ public class RollupRequestTranslationTests extends ESTestCase { public void testUnsupportedMetric() { Set caps = singletonSet(new RollupJobCaps(ConfigTestHelpers - .getRollupJob("foo").setMetricsConfig(Collections.singletonList(new MetricConfig.Builder() - .setField("foo") - .setMetrics(Arrays.asList("avg", "max", "min", "sum")).build())) + .getRollupJob("foo") + .setMetricsConfig(singletonList(new MetricConfig("foo", Arrays.asList("avg", "max", "min", "sum")))) .build())); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, @@ -384,7 +384,7 @@ public class RollupRequestTranslationTests extends ESTestCase { assertThat(e.getMessage(), equalTo("Unable to translate aggregation tree into Rollup. Aggregation [test_geo] is of type " + "[GeoDistanceAggregationBuilder] which is currently unsupported.")); } - + private Set singletonSet(RollupJobCaps cap) { Set caps = new HashSet<>(); caps.add(cap); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java index 5b2f22dcd2d..9d8eb4dae09 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java @@ -454,7 +454,7 @@ public class SearchActionTests extends ESTestCase { job2.setGroupConfig(group.build()); // so that the jobs aren't exactly equal - job2.setMetricsConfig(Collections.singletonList(ConfigTestHelpers.getMetricConfig().build())); + job2.setMetricsConfig(ConfigTestHelpers.randomMetricsConfigs(random())); RollupJobCaps cap2 = new RollupJobCaps(job2.build()); Set caps = new HashSet<>(2); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java index d5fd300b687..df389cc7539 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java @@ -16,35 +16,29 @@ import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig; import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig; import org.joda.time.DateTimeZone; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.equalTo; - +//TODO split this into dedicated unit test classes (one for each config object) public class ConfigTests extends ESTestCase { public void testEmptyField() { - MetricConfig.Builder config = ConfigTestHelpers.getMetricConfig(); - config.setField(null); - Exception e = expectThrows(IllegalArgumentException.class, config::build); - assertThat(e.getMessage(), equalTo("Parameter [field] must be a non-null, non-empty string.")); + Exception e = expectThrows(IllegalArgumentException.class, () -> new MetricConfig(null, singletonList("max"))); + assertThat(e.getMessage(), equalTo("Field must be a non-null, non-empty string")); - config.setField(""); - e = expectThrows(IllegalArgumentException.class, config::build); - assertThat(e.getMessage(), equalTo("Parameter [field] must be a non-null, non-empty string.")); + e = expectThrows(IllegalArgumentException.class, () -> new MetricConfig("", singletonList("max"))); + assertThat(e.getMessage(), equalTo("Field must be a non-null, non-empty string")); } public void testEmptyMetrics() { - MetricConfig.Builder config = ConfigTestHelpers.getMetricConfig(); - config.setMetrics(null); - Exception e = expectThrows(IllegalArgumentException.class, config::build); - assertThat(e.getMessage(), equalTo("Parameter [metrics] must be a non-null, non-empty array of strings.")); + Exception e = expectThrows(IllegalArgumentException.class, () -> new MetricConfig("foo", emptyList())); + assertThat(e.getMessage(), equalTo("Metrics must be a non-null, non-empty array of strings")); - config.setMetrics(Collections.emptyList()); - e = expectThrows(IllegalArgumentException.class, config::build); - assertThat(e.getMessage(), equalTo("Parameter [metrics] must be a non-null, non-empty array of strings.")); + e = expectThrows(IllegalArgumentException.class, () -> new MetricConfig("foo", null)); + assertThat(e.getMessage(), equalTo("Metrics must be a non-null, non-empty array of strings")); } public void testEmptyGroup() { @@ -233,11 +227,4 @@ public class ConfigTests extends ESTestCase { assertFalse(json.contains("authentication")); assertFalse(json.contains("security")); } - - public void testUnsupportedMetric() { - MetricConfig.Builder config = ConfigTestHelpers.getMetricConfig(); - config.setMetrics(Arrays.asList("max","foo")); - Exception e = expectThrows(IllegalArgumentException.class, config::build); - assertThat(e.getMessage(), equalTo("Unsupported metric [foo]. Supported metrics include: [max, min, sum, avg, value_count]")); - } } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java index f36ef3b00c7..72d37756572 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java @@ -54,6 +54,7 @@ import java.util.List; import java.util.Map; import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomHistogramGroupConfig; +import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -102,10 +103,7 @@ public class IndexerUtilsTests extends AggregatorTestCase { .build(); CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME, dateHistoGroupConfig.toBuilders()); - MetricConfig metricConfig = new MetricConfig.Builder() - .setField("does_not_exist") - .setMetrics(Collections.singletonList("max")) - .build(); + MetricConfig metricConfig = new MetricConfig("does_not_exist", singletonList("max")); metricConfig.toBuilders().forEach(compositeBuilder::subAggregation); Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, timestampFieldType, valueFieldType); @@ -170,9 +168,9 @@ public class IndexerUtilsTests extends AggregatorTestCase { .interval(1); CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME, - Collections.singletonList(dateHisto)); + singletonList(dateHisto)); - MetricConfig metricConfig = new MetricConfig.Builder().setField(valueField).setMetrics(Collections.singletonList("max")).build(); + MetricConfig metricConfig = new MetricConfig(valueField, singletonList("max")); metricConfig.toBuilders().forEach(compositeBuilder::subAggregation); Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, timestampFieldType, valueFieldType); @@ -226,9 +224,9 @@ public class IndexerUtilsTests extends AggregatorTestCase { TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder("the_terms." + TermsAggregationBuilder.NAME).field(valueField); CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME, - Collections.singletonList(terms)); + singletonList(terms)); - MetricConfig metricConfig = new MetricConfig.Builder().setField(valueField).setMetrics(Collections.singletonList("max")).build(); + MetricConfig metricConfig = new MetricConfig(valueField, singletonList("max")); metricConfig.toBuilders().forEach(compositeBuilder::subAggregation); Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, valueFieldType); @@ -292,9 +290,9 @@ public class IndexerUtilsTests extends AggregatorTestCase { .dateHistogramInterval(new DateHistogramInterval("1d")); CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME, - Collections.singletonList(dateHisto)); + singletonList(dateHisto)); - MetricConfig metricConfig = new MetricConfig.Builder().setField("another_field").setMetrics(Arrays.asList("avg", "sum")).build(); + MetricConfig metricConfig = new MetricConfig("another_field", Arrays.asList("avg", "sum")); metricConfig.toBuilders().forEach(compositeBuilder::subAggregation); Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, timestampFieldType, valueFieldType); @@ -439,7 +437,7 @@ public class IndexerUtilsTests extends AggregatorTestCase { CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME, termsGroupConfig.toBuilders()).size(numDocs*2); - MetricConfig metricConfig = new MetricConfig.Builder().setField(metricField).setMetrics(Collections.singletonList("max")).build(); + MetricConfig metricConfig = new MetricConfig(metricField, singletonList("max")); metricConfig.toBuilders().forEach(compositeBuilder::subAggregation); Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, valueFieldType, metricFieldType); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index f658fa574eb..5fb7f654f36 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -47,13 +47,13 @@ import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation; import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig; import org.elasticsearch.xpack.core.rollup.job.GroupConfig; import org.elasticsearch.xpack.core.rollup.job.IndexerState; import org.elasticsearch.xpack.core.rollup.job.MetricConfig; import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig; -import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.junit.Before; @@ -143,7 +143,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase { DateHistoGroupConfig dateHistoConfig = new DateHistoGroupConfig.Builder() .setField(field) .setInterval(new DateHistogramInterval("1h")).build(); - MetricConfig config = new MetricConfig.Builder().setField("counter").setMetrics(Arrays.asList("avg", "sum", "max", "min")).build(); + MetricConfig config = new MetricConfig("counter", Arrays.asList("avg", "sum", "max", "min")); RollupJobConfig job = createJob(rollupIndex, new GroupConfig.Builder().setDateHisto(dateHistoConfig).build(), Collections.singletonList(config)); final List> dataset = new ArrayList<>(); @@ -417,7 +417,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase { DateHistoGroupConfig dateHistoConfig = new DateHistoGroupConfig.Builder() .setField(timestampField) .setInterval(new DateHistogramInterval(timeInterval)).build(); - MetricConfig metricConfig = new MetricConfig.Builder().setField(valueField).setMetrics(Collections.singletonList("avg")).build(); + MetricConfig metricConfig = new MetricConfig(valueField, Collections.singletonList("avg")); RollupJobConfig job = createJob(rollupIndex, new GroupConfig.Builder().setDateHisto(dateHistoConfig).build(), Collections.singletonList(metricConfig)); From eb17128b9c43ba0dfde374764f4c1e8b92a44a0e Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 3 Aug 2018 10:39:29 +0100 Subject: [PATCH 10/11] [ML] Add Detector config classes to protocol library (#32495) This commit adds the Detector class and its dependencies to the X-Pack protocol library used by the high level REST client. (Future commits will add the remaining config classes, plus results and stats classes.) These classes: - Are immutable, with builders, but the builders do no validation beyond null checks - Are convertible to and from X-Content, but NOT wire transportable - Have lenient parsers to maximize compatibility across versions - Have the same class names, member names and getter/setter names as the corresponding classes in X-Pack core to ease migration for transport client users - Don't reproduce all the methods that do calculations or transformations that the the corresponding classes in X-Pack core have --- .../config/DefaultDetectorDescription.java | 96 +++++ .../xpack/ml/job/config/DetectionRule.java | 153 ++++++++ .../xpack/ml/job/config/Detector.java | 362 ++++++++++++++++++ .../xpack/ml/job/config/DetectorFunction.java | 95 +++++ .../xpack/ml/job/config/FilterRef.java | 107 ++++++ .../xpack/ml/job/config/MlFilter.java | 155 ++++++++ .../xpack/ml/job/config/Operator.java | 71 ++++ .../xpack/ml/job/config/RuleAction.java | 41 ++ .../xpack/ml/job/config/RuleCondition.java | 129 +++++++ .../xpack/ml/job/config/RuleScope.java | 133 +++++++ .../protocol/xpack/ml/package-info.java | 24 ++ .../ml/job/config/DetectionRuleTests.java | 93 +++++ .../xpack/ml/job/config/DetectorTests.java | 137 +++++++ .../xpack/ml/job/config/FilterRefTests.java | 42 ++ .../xpack/ml/job/config/MlFilterTests.java | 86 +++++ .../ml/job/config/RuleConditionTests.java | 80 ++++ .../xpack/ml/job/config/RuleScopeTests.java | 66 ++++ 17 files changed, 1870 insertions(+) create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DefaultDetectorDescription.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRule.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorFunction.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRef.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilter.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Operator.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleAction.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleCondition.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScope.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/package-info.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRuleTests.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorTests.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRefTests.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilterTests.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleConditionTests.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScopeTests.java diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DefaultDetectorDescription.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DefaultDetectorDescription.java new file mode 100644 index 00000000000..081e685fc74 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DefaultDetectorDescription.java @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.Strings; + +public final class DefaultDetectorDescription { + private static final String BY_TOKEN = " by "; + private static final String OVER_TOKEN = " over "; + + private static final String USE_NULL_OPTION = " usenull="; + private static final String PARTITION_FIELD_OPTION = " partitionfield="; + private static final String EXCLUDE_FREQUENT_OPTION = " excludefrequent="; + + private DefaultDetectorDescription() { + } + + /** + * Returns the default description for the given {@code detector} + * + * @param detector the {@code Detector} for which a default description is requested + * @return the default description + */ + public static String of(Detector detector) { + StringBuilder sb = new StringBuilder(); + appendOn(detector, sb); + return sb.toString(); + } + + /** + * Appends to the given {@code StringBuilder} the default description + * for the given {@code detector} + * + * @param detector the {@code Detector} for which a default description is requested + * @param sb the {@code StringBuilder} to append to + */ + public static void appendOn(Detector detector, StringBuilder sb) { + if (isNotNullOrEmpty(detector.getFunction().getFullName())) { + sb.append(detector.getFunction()); + if (isNotNullOrEmpty(detector.getFieldName())) { + sb.append('(').append(quoteField(detector.getFieldName())) + .append(')'); + } + } else if (isNotNullOrEmpty(detector.getFieldName())) { + sb.append(quoteField(detector.getFieldName())); + } + + if (isNotNullOrEmpty(detector.getByFieldName())) { + sb.append(BY_TOKEN).append(quoteField(detector.getByFieldName())); + } + + if (isNotNullOrEmpty(detector.getOverFieldName())) { + sb.append(OVER_TOKEN).append(quoteField(detector.getOverFieldName())); + } + + if (detector.isUseNull()) { + sb.append(USE_NULL_OPTION).append(detector.isUseNull()); + } + + if (isNotNullOrEmpty(detector.getPartitionFieldName())) { + sb.append(PARTITION_FIELD_OPTION).append(quoteField(detector.getPartitionFieldName())); + } + + if (detector.getExcludeFrequent() != null) { + sb.append(EXCLUDE_FREQUENT_OPTION).append(detector.getExcludeFrequent()); + } + } + + private static String quoteField(String field) { + if (field.matches("\\w*")) { + return field; + } else { + return "\"" + field.replace("\\", "\\\\").replace("\"", "\\\"") + "\""; + } + } + + private static boolean isNotNullOrEmpty(String arg) { + return !Strings.isNullOrEmpty(arg); + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRule.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRule.java new file mode 100644 index 00000000000..9a73afe885b --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRule.java @@ -0,0 +1,153 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Objects; + +public class DetectionRule implements ToXContentObject { + + public static final ParseField DETECTION_RULE_FIELD = new ParseField("detection_rule"); + public static final ParseField ACTIONS_FIELD = new ParseField("actions"); + public static final ParseField SCOPE_FIELD = new ParseField("scope"); + public static final ParseField CONDITIONS_FIELD = new ParseField("conditions"); + + public static final ObjectParser PARSER = + new ObjectParser<>(DETECTION_RULE_FIELD.getPreferredName(), true, Builder::new);; + + static { + PARSER.declareStringArray(Builder::setActions, ACTIONS_FIELD); + PARSER.declareObject(Builder::setScope, RuleScope.parser(), SCOPE_FIELD); + PARSER.declareObjectArray(Builder::setConditions, RuleCondition.PARSER, CONDITIONS_FIELD); + } + + private final EnumSet actions; + private final RuleScope scope; + private final List conditions; + + private DetectionRule(EnumSet actions, RuleScope scope, List conditions) { + this.actions = Objects.requireNonNull(actions); + this.scope = Objects.requireNonNull(scope); + this.conditions = Collections.unmodifiableList(conditions); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ACTIONS_FIELD.getPreferredName(), actions); + if (scope.isEmpty() == false) { + builder.field(SCOPE_FIELD.getPreferredName(), scope); + } + if (conditions.isEmpty() == false) { + builder.field(CONDITIONS_FIELD.getPreferredName(), conditions); + } + builder.endObject(); + return builder; + } + + public EnumSet getActions() { + return actions; + } + + public RuleScope getScope() { + return scope; + } + + public List getConditions() { + return conditions; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof DetectionRule == false) { + return false; + } + + DetectionRule other = (DetectionRule) obj; + return Objects.equals(actions, other.actions) + && Objects.equals(scope, other.scope) + && Objects.equals(conditions, other.conditions); + } + + @Override + public int hashCode() { + return Objects.hash(actions, scope, conditions); + } + + public static class Builder { + private EnumSet actions = EnumSet.of(RuleAction.SKIP_RESULT); + private RuleScope scope = new RuleScope(); + private List conditions = Collections.emptyList(); + + public Builder(RuleScope.Builder scope) { + this.scope = scope.build(); + } + + public Builder(List conditions) { + this.conditions = Objects.requireNonNull(conditions); + } + + Builder() { + } + + public Builder setActions(List actions) { + this.actions.clear(); + actions.stream().map(RuleAction::fromString).forEach(this.actions::add); + return this; + } + + public Builder setActions(EnumSet actions) { + this.actions = Objects.requireNonNull(actions, ACTIONS_FIELD.getPreferredName()); + return this; + } + + public Builder setActions(RuleAction... actions) { + this.actions.clear(); + Arrays.stream(actions).forEach(this.actions::add); + return this; + } + + public Builder setScope(RuleScope scope) { + this.scope = Objects.requireNonNull(scope); + return this; + } + + public Builder setConditions(List conditions) { + this.conditions = Objects.requireNonNull(conditions); + return this; + } + + public DetectionRule build() { + return new DetectionRule(actions, scope, conditions); + } + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java new file mode 100644 index 00000000000..3274b03877f --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Detector.java @@ -0,0 +1,362 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +/** + * Defines the fields and functions used in the analysis. A combination of field_name, + * by_field_name and over_field_name can be used depending on the specific + * function chosen. For more information see + * configuring + * detectors and detector functions. + */ +public class Detector implements ToXContentObject { + + public enum ExcludeFrequent { + ALL, + NONE, + BY, + OVER; + + /** + * Case-insensitive from string method. + * Works with either ALL, All, etc. + * + * @param value String representation + * @return The data format + */ + public static ExcludeFrequent forString(String value) { + return valueOf(value.toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + public static final ParseField DETECTOR_DESCRIPTION_FIELD = new ParseField("detector_description"); + public static final ParseField FUNCTION_FIELD = new ParseField("function"); + public static final ParseField FIELD_NAME_FIELD = new ParseField("field_name"); + public static final ParseField BY_FIELD_NAME_FIELD = new ParseField("by_field_name"); + public static final ParseField OVER_FIELD_NAME_FIELD = new ParseField("over_field_name"); + public static final ParseField PARTITION_FIELD_NAME_FIELD = new ParseField("partition_field_name"); + public static final ParseField USE_NULL_FIELD = new ParseField("use_null"); + public static final ParseField EXCLUDE_FREQUENT_FIELD = new ParseField("exclude_frequent"); + public static final ParseField CUSTOM_RULES_FIELD = new ParseField("custom_rules"); + public static final ParseField DETECTOR_INDEX = new ParseField("detector_index"); + + public static final ObjectParser PARSER = new ObjectParser<>("detector", true, Builder::new); + + static { + PARSER.declareString(Builder::setDetectorDescription, DETECTOR_DESCRIPTION_FIELD); + PARSER.declareString(Builder::setFunction, FUNCTION_FIELD); + PARSER.declareString(Builder::setFieldName, FIELD_NAME_FIELD); + PARSER.declareString(Builder::setByFieldName, BY_FIELD_NAME_FIELD); + PARSER.declareString(Builder::setOverFieldName, OVER_FIELD_NAME_FIELD); + PARSER.declareString(Builder::setPartitionFieldName, PARTITION_FIELD_NAME_FIELD); + PARSER.declareBoolean(Builder::setUseNull, USE_NULL_FIELD); + PARSER.declareField(Builder::setExcludeFrequent, p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return ExcludeFrequent.forString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING); + PARSER.declareObjectArray(Builder::setRules, (p, c) -> DetectionRule.PARSER.apply(p, c).build(), CUSTOM_RULES_FIELD); + PARSER.declareInt(Builder::setDetectorIndex, DETECTOR_INDEX); + } + + private final String detectorDescription; + private final DetectorFunction function; + private final String fieldName; + private final String byFieldName; + private final String overFieldName; + private final String partitionFieldName; + private final boolean useNull; + private final ExcludeFrequent excludeFrequent; + private final List rules; + private final int detectorIndex; + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(DETECTOR_DESCRIPTION_FIELD.getPreferredName(), detectorDescription); + builder.field(FUNCTION_FIELD.getPreferredName(), function); + if (fieldName != null) { + builder.field(FIELD_NAME_FIELD.getPreferredName(), fieldName); + } + if (byFieldName != null) { + builder.field(BY_FIELD_NAME_FIELD.getPreferredName(), byFieldName); + } + if (overFieldName != null) { + builder.field(OVER_FIELD_NAME_FIELD.getPreferredName(), overFieldName); + } + if (partitionFieldName != null) { + builder.field(PARTITION_FIELD_NAME_FIELD.getPreferredName(), partitionFieldName); + } + if (useNull) { + builder.field(USE_NULL_FIELD.getPreferredName(), useNull); + } + if (excludeFrequent != null) { + builder.field(EXCLUDE_FREQUENT_FIELD.getPreferredName(), excludeFrequent); + } + if (rules.isEmpty() == false) { + builder.field(CUSTOM_RULES_FIELD.getPreferredName(), rules); + } + // negative means unknown + if (detectorIndex >= 0) { + builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex); + } + builder.endObject(); + return builder; + } + + private Detector(String detectorDescription, DetectorFunction function, String fieldName, String byFieldName, String overFieldName, + String partitionFieldName, boolean useNull, ExcludeFrequent excludeFrequent, List rules, + int detectorIndex) { + this.function = function; + this.fieldName = fieldName; + this.byFieldName = byFieldName; + this.overFieldName = overFieldName; + this.partitionFieldName = partitionFieldName; + this.useNull = useNull; + this.excludeFrequent = excludeFrequent; + this.rules = Collections.unmodifiableList(rules); + this.detectorDescription = detectorDescription != null ? detectorDescription : DefaultDetectorDescription.of(this); + this.detectorIndex = detectorIndex; + } + + public String getDetectorDescription() { + return detectorDescription; + } + + /** + * The analysis function used e.g. count, rare, min etc. + * + * @return The function or null if not set + */ + public DetectorFunction getFunction() { + return function; + } + + /** + * The Analysis field + * + * @return The field to analyse + */ + public String getFieldName() { + return fieldName; + } + + /** + * The 'by' field or null if not set. + * + * @return The 'by' field + */ + public String getByFieldName() { + return byFieldName; + } + + /** + * The 'over' field or null if not set. + * + * @return The 'over' field + */ + public String getOverFieldName() { + return overFieldName; + } + + /** + * Segments the analysis along another field to have completely + * independent baselines for each instance of partitionfield + * + * @return The Partition Field + */ + public String getPartitionFieldName() { + return partitionFieldName; + } + + /** + * Where there isn't a value for the 'by' or 'over' field should a new + * series be used as the 'null' series. + * + * @return true if the 'null' series should be created + */ + public boolean isUseNull() { + return useNull; + } + + /** + * Excludes frequently-occuring metrics from the analysis; + * can apply to 'by' field, 'over' field, or both + * + * @return the value that the user set + */ + public ExcludeFrequent getExcludeFrequent() { + return excludeFrequent; + } + + public List getRules() { + return rules; + } + + /** + * @return the detector index or a negative number if unknown + */ + public int getDetectorIndex() { + return detectorIndex; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof Detector == false) { + return false; + } + + Detector that = (Detector) other; + + return Objects.equals(this.detectorDescription, that.detectorDescription) && + Objects.equals(this.function, that.function) && + Objects.equals(this.fieldName, that.fieldName) && + Objects.equals(this.byFieldName, that.byFieldName) && + Objects.equals(this.overFieldName, that.overFieldName) && + Objects.equals(this.partitionFieldName, that.partitionFieldName) && + Objects.equals(this.useNull, that.useNull) && + Objects.equals(this.excludeFrequent, that.excludeFrequent) && + Objects.equals(this.rules, that.rules) && + this.detectorIndex == that.detectorIndex; + } + + @Override + public int hashCode() { + return Objects.hash(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName, useNull, + excludeFrequent, rules, detectorIndex); + } + + public static class Builder { + + private String detectorDescription; + private DetectorFunction function; + private String fieldName; + private String byFieldName; + private String overFieldName; + private String partitionFieldName; + private boolean useNull = false; + private ExcludeFrequent excludeFrequent; + private List rules = Collections.emptyList(); + // negative means unknown + private int detectorIndex = -1; + + public Builder() { + } + + public Builder(Detector detector) { + detectorDescription = detector.detectorDescription; + function = detector.function; + fieldName = detector.fieldName; + byFieldName = detector.byFieldName; + overFieldName = detector.overFieldName; + partitionFieldName = detector.partitionFieldName; + useNull = detector.useNull; + excludeFrequent = detector.excludeFrequent; + rules = new ArrayList<>(detector.rules); + detectorIndex = detector.detectorIndex; + } + + public Builder(String function, String fieldName) { + this(DetectorFunction.fromString(function), fieldName); + } + + public Builder(DetectorFunction function, String fieldName) { + this.function = function; + this.fieldName = fieldName; + } + + public Builder setDetectorDescription(String detectorDescription) { + this.detectorDescription = detectorDescription; + return this; + } + + public Builder setFunction(String function) { + this.function = DetectorFunction.fromString(function); + return this; + } + + public Builder setFieldName(String fieldName) { + this.fieldName = fieldName; + return this; + } + + public Builder setByFieldName(String byFieldName) { + this.byFieldName = byFieldName; + return this; + } + + public Builder setOverFieldName(String overFieldName) { + this.overFieldName = overFieldName; + return this; + } + + public Builder setPartitionFieldName(String partitionFieldName) { + this.partitionFieldName = partitionFieldName; + return this; + } + + public Builder setUseNull(boolean useNull) { + this.useNull = useNull; + return this; + } + + public Builder setExcludeFrequent(ExcludeFrequent excludeFrequent) { + this.excludeFrequent = excludeFrequent; + return this; + } + + public Builder setRules(List rules) { + this.rules = rules; + return this; + } + + public Builder setDetectorIndex(int detectorIndex) { + this.detectorIndex = detectorIndex; + return this; + } + + public Detector build() { + return new Detector(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName, + useNull, excludeFrequent, rules, detectorIndex); + } + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorFunction.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorFunction.java new file mode 100644 index 00000000000..5d9a06948d0 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorFunction.java @@ -0,0 +1,95 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; + +public enum DetectorFunction { + + COUNT, + LOW_COUNT, + HIGH_COUNT, + NON_ZERO_COUNT("nzc"), + LOW_NON_ZERO_COUNT("low_nzc"), + HIGH_NON_ZERO_COUNT("high_nzc"), + DISTINCT_COUNT("dc"), + LOW_DISTINCT_COUNT("low_dc"), + HIGH_DISTINCT_COUNT("high_dc"), + RARE, + FREQ_RARE, + INFO_CONTENT, + LOW_INFO_CONTENT, + HIGH_INFO_CONTENT, + METRIC, + MEAN, + LOW_MEAN, + HIGH_MEAN, + AVG, + LOW_AVG, + HIGH_AVG, + MEDIAN, + LOW_MEDIAN, + HIGH_MEDIAN, + MIN, + MAX, + SUM, + LOW_SUM, + HIGH_SUM, + NON_NULL_SUM, + LOW_NON_NULL_SUM, + HIGH_NON_NULL_SUM, + VARP, + LOW_VARP, + HIGH_VARP, + TIME_OF_DAY, + TIME_OF_WEEK, + LAT_LONG; + + private Set shortcuts; + + DetectorFunction() { + shortcuts = Collections.emptySet(); + } + + DetectorFunction(String... shortcuts) { + this.shortcuts = Arrays.stream(shortcuts).collect(Collectors.toSet()); + } + + public String getFullName() { + return name().toLowerCase(Locale.ROOT); + } + + @Override + public String toString() { + return getFullName(); + } + + public static DetectorFunction fromString(String op) { + for (DetectorFunction function : values()) { + if (function.getFullName().equals(op) || function.shortcuts.contains(op)) { + return function; + } + } + throw new IllegalArgumentException("Unknown detector function [" + op + "]"); + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRef.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRef.java new file mode 100644 index 00000000000..9afbdf4876f --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRef.java @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +public class FilterRef implements ToXContentObject { + + public static final ParseField FILTER_REF_FIELD = new ParseField("filter_ref"); + public static final ParseField FILTER_ID = new ParseField("filter_id"); + public static final ParseField FILTER_TYPE = new ParseField("filter_type"); + + public enum FilterType { + INCLUDE, EXCLUDE; + + public static FilterType fromString(String value) { + return valueOf(value.toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(FILTER_REF_FIELD.getPreferredName(), true, a -> new FilterRef((String) a[0], (FilterType) a[1])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), FILTER_ID); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return FilterType.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, FILTER_TYPE, ObjectParser.ValueType.STRING); + } + + private final String filterId; + private final FilterType filterType; + + public FilterRef(String filterId, FilterType filterType) { + this.filterId = Objects.requireNonNull(filterId); + this.filterType = filterType == null ? FilterType.INCLUDE : filterType; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FILTER_ID.getPreferredName(), filterId); + builder.field(FILTER_TYPE.getPreferredName(), filterType); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof FilterRef == false) { + return false; + } + + FilterRef other = (FilterRef) obj; + return Objects.equals(filterId, other.filterId) && Objects.equals(filterType, other.filterType); + } + + @Override + public int hashCode() { + return Objects.hash(filterId, filterType); + } + + public String getFilterId() { + return filterId; + } + + public FilterType getFilterType() { + return filterType; + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilter.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilter.java new file mode 100644 index 00000000000..bcbc0c295c2 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilter.java @@ -0,0 +1,155 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.SortedSet; +import java.util.TreeSet; + +public class MlFilter implements ToXContentObject { + + public static final ParseField TYPE = new ParseField("type"); + public static final ParseField ID = new ParseField("filter_id"); + public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField ITEMS = new ParseField("items"); + + // For QueryPage + public static final ParseField RESULTS_FIELD = new ParseField("filters"); + + public static final ObjectParser PARSER = new ObjectParser<>(TYPE.getPreferredName(), true, Builder::new); + + static { + PARSER.declareString((builder, s) -> {}, TYPE); + PARSER.declareString(Builder::setId, ID); + PARSER.declareStringOrNull(Builder::setDescription, DESCRIPTION); + PARSER.declareStringArray(Builder::setItems, ITEMS); + } + + private final String id; + private final String description; + private final SortedSet items; + + private MlFilter(String id, String description, SortedSet items) { + this.id = Objects.requireNonNull(id); + this.description = description; + this.items = Collections.unmodifiableSortedSet(items); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ID.getPreferredName(), id); + if (description != null) { + builder.field(DESCRIPTION.getPreferredName(), description); + } + builder.field(ITEMS.getPreferredName(), items); + // Don't include TYPE as it's fixed + builder.endObject(); + return builder; + } + + public String getId() { + return id; + } + + public String getDescription() { + return description; + } + + public SortedSet getItems() { + return items; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + + if (obj instanceof MlFilter == false) { + return false; + } + + MlFilter other = (MlFilter) obj; + return id.equals(other.id) && Objects.equals(description, other.description) && items.equals(other.items); + } + + @Override + public int hashCode() { + return Objects.hash(id, description, items); + } + + public static Builder builder(String filterId) { + return new Builder().setId(filterId); + } + + public static class Builder { + + private String id; + private String description; + private SortedSet items = new TreeSet<>(); + + private Builder() { + } + + public Builder setId(String id) { + this.id = Objects.requireNonNull(id); + return this; + } + + @Nullable + public String getId() { + return id; + } + + public Builder setDescription(String description) { + this.description = description; + return this; + } + + public Builder setItems(SortedSet items) { + this.items = Objects.requireNonNull(items); + return this; + } + + public Builder setItems(List items) { + this.items = new TreeSet<>(items); + return this; + } + + public Builder setItems(String... items) { + setItems(Arrays.asList(items)); + return this; + } + + public MlFilter build() { + return new MlFilter(id, description, items); + } + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Operator.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Operator.java new file mode 100644 index 00000000000..c3dc52e5a3c --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Operator.java @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.ParseField; + +import java.util.Locale; + +/** + * Enum representing logical comparisons on doubles + */ +public enum Operator { + GT { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) > 0; + } + }, + GTE { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) >= 0; + } + }, + LT { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) < 0; + } + }, + LTE { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) <= 0; + } + }; + // EQ was considered but given the oddity of such a + // condition and the fact that it would be a numerically + // unstable condition, it was rejected. + + public static final ParseField OPERATOR_FIELD = new ParseField("operator"); + + public boolean test(double lhs, double rhs) { + return false; + } + + public static Operator fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleAction.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleAction.java new file mode 100644 index 00000000000..9e2364b4fd9 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleAction.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import java.util.Locale; + +public enum RuleAction { + SKIP_RESULT, + SKIP_MODEL_UPDATE; + + /** + * Case-insensitive from string method. + * + * @param value String representation + * @return The rule action + */ + public static RuleAction fromString(String value) { + return RuleAction.valueOf(value.toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleCondition.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleCondition.java new file mode 100644 index 00000000000..ec19547fe13 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleCondition.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +public class RuleCondition implements ToXContentObject { + + public static final ParseField RULE_CONDITION_FIELD = new ParseField("rule_condition"); + + public static final ParseField APPLIES_TO_FIELD = new ParseField("applies_to"); + public static final ParseField VALUE_FIELD = new ParseField("value"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RULE_CONDITION_FIELD.getPreferredName(), true, + a -> new RuleCondition((AppliesTo) a[0], (Operator) a[1], (double) a[2])); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return AppliesTo.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, APPLIES_TO_FIELD, ValueType.STRING); + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Operator.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, Operator.OPERATOR_FIELD, ValueType.STRING); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), VALUE_FIELD); + } + + private final AppliesTo appliesTo; + private final Operator operator; + private final double value; + + public RuleCondition(AppliesTo appliesTo, Operator operator, double value) { + this.appliesTo = appliesTo; + this.operator = operator; + this.value = value; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(APPLIES_TO_FIELD.getPreferredName(), appliesTo); + builder.field(Operator.OPERATOR_FIELD.getPreferredName(), operator); + builder.field(VALUE_FIELD.getPreferredName(), value); + builder.endObject(); + return builder; + } + + public AppliesTo getAppliesTo() { + return appliesTo; + } + + public Operator getOperator() { + return operator; + } + + public double getValue() { + return value; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof RuleCondition == false) { + return false; + } + + RuleCondition other = (RuleCondition) obj; + return appliesTo == other.appliesTo && operator == other.operator && value == other.value; + } + + @Override + public int hashCode() { + return Objects.hash(appliesTo, operator, value); + } + + public static RuleCondition createTime(Operator operator, long epochSeconds) { + return new RuleCondition(AppliesTo.TIME, operator, epochSeconds); + } + + public enum AppliesTo { + ACTUAL, + TYPICAL, + DIFF_FROM_TYPICAL, + TIME; + + public static AppliesTo fromString(String value) { + return valueOf(value.toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScope.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScope.java new file mode 100644 index 00000000000..aa12d5ea2a2 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScope.java @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ContextParser; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class RuleScope implements ToXContentObject { + + public static ContextParser parser() { + return (p, c) -> { + Map unparsedScope = p.map(); + if (unparsedScope.isEmpty()) { + return new RuleScope(); + } + Map scope = new HashMap<>(); + for (Map.Entry entry : unparsedScope.entrySet()) { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + @SuppressWarnings("unchecked") + Map value = (Map) entry.getValue(); + builder.map(value); + try (XContentParser scopeParser = XContentFactory.xContent(builder.contentType()).createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, Strings.toString(builder))) { + scope.put(entry.getKey(), FilterRef.PARSER.parse(scopeParser, null)); + } + } + } + return new RuleScope(scope); + }; + } + + private final Map scope; + + public RuleScope() { + scope = Collections.emptyMap(); + } + + public RuleScope(Map scope) { + this.scope = Collections.unmodifiableMap(scope); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.map(scope); + } + + public boolean isEmpty() { + return scope.isEmpty(); + } + + public Set getReferencedFilters() { + return scope.values().stream().map(FilterRef::getFilterId).collect(Collectors.toSet()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof RuleScope == false) { + return false; + } + + RuleScope other = (RuleScope) obj; + return Objects.equals(scope, other.scope); + } + + @Override + public int hashCode() { + return Objects.hash(scope); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private Map scope = new HashMap<>(); + + public Builder() { + } + + public Builder(RuleScope otherScope) { + scope = new HashMap<>(otherScope.scope); + } + + public Builder exclude(String field, String filterId) { + scope.put(field, new FilterRef(filterId, FilterRef.FilterType.EXCLUDE)); + return this; + } + + public Builder include(String field, String filterId) { + scope.put(field, new FilterRef(filterId, FilterRef.FilterType.INCLUDE)); + return this; + } + + public RuleScope build() { + return new RuleScope(scope); + } + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/package-info.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/package-info.java new file mode 100644 index 00000000000..b1e4c6c0d4e --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Request and Response objects for the default distribution's Machine + * Learning APIs. + */ +package org.elasticsearch.protocol.xpack.ml; diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRuleTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRuleTests.java new file mode 100644 index 00000000000..bc70a404894 --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectionRuleTests.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DetectionRuleTests extends AbstractXContentTestCase { + + @Override + protected DetectionRule createTestInstance() { + DetectionRule.Builder builder = new DetectionRule.Builder(); + + if (randomBoolean()) { + EnumSet actions = EnumSet.noneOf(RuleAction.class); + int actionsCount = randomIntBetween(1, RuleAction.values().length); + for (int i = 0; i < actionsCount; ++i) { + actions.add(randomFrom(RuleAction.values())); + } + builder.setActions(actions); + } + + boolean hasScope = randomBoolean(); + boolean hasConditions = randomBoolean(); + + if (!hasScope && !hasConditions) { + // at least one of the two should be present + if (randomBoolean()) { + hasScope = true; + } else { + hasConditions = true; + } + } + + if (hasScope) { + Map scope = new HashMap<>(); + int scopeSize = randomIntBetween(1, 3); + for (int i = 0; i < scopeSize; i++) { + scope.put(randomAlphaOfLength(20), new FilterRef(randomAlphaOfLength(20), randomFrom(FilterRef.FilterType.values()))); + } + builder.setScope(new RuleScope(scope)); + } + + if (hasConditions) { + int size = randomIntBetween(1, 5); + List ruleConditions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + // no need for random condition (it is already tested) + ruleConditions.addAll(createCondition(randomDouble())); + } + builder.setConditions(ruleConditions); + } + + return builder.build(); + } + + @Override + protected DetectionRule doParseInstance(XContentParser parser) { + return DetectionRule.PARSER.apply(parser, null).build(); + } + + private static List createCondition(double value) { + return Collections.singletonList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, value)); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorTests.java new file mode 100644 index 00000000000..0b1ba892acd --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/DetectorTests.java @@ -0,0 +1,137 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; + +public class DetectorTests extends AbstractXContentTestCase { + + public void testEquals_GivenEqual() { + Detector.Builder builder = new Detector.Builder("mean", "field"); + builder.setByFieldName("by_field"); + builder.setOverFieldName("over_field"); + builder.setPartitionFieldName("partition"); + builder.setUseNull(false); + Detector detector1 = builder.build(); + + builder = new Detector.Builder("mean", "field"); + builder.setByFieldName("by_field"); + builder.setOverFieldName("over_field"); + builder.setPartitionFieldName("partition"); + builder.setUseNull(false); + Detector detector2 = builder.build(); + + assertTrue(detector1.equals(detector2)); + assertTrue(detector2.equals(detector1)); + assertEquals(detector1.hashCode(), detector2.hashCode()); + } + + public void testEquals_GivenDifferentDetectorDescription() { + Detector detector1 = createDetector().build(); + Detector.Builder builder = createDetector(); + builder.setDetectorDescription("bar"); + Detector detector2 = builder.build(); + + assertFalse(detector1.equals(detector2)); + } + + public void testEquals_GivenDifferentByFieldName() { + Detector detector1 = createDetector().build(); + Detector detector2 = createDetector().build(); + + assertEquals(detector1, detector2); + + Detector.Builder builder = new Detector.Builder(detector2); + builder.setByFieldName("by2"); + detector2 = builder.build(); + assertFalse(detector1.equals(detector2)); + } + + private Detector.Builder createDetector() { + Detector.Builder detector = new Detector.Builder("mean", "field"); + detector.setByFieldName("by_field"); + detector.setOverFieldName("over_field"); + detector.setPartitionFieldName("partition"); + detector.setUseNull(true); + DetectionRule rule = new DetectionRule.Builder(RuleScope.builder().exclude("partition", "partition_filter")) + .setActions(RuleAction.SKIP_RESULT) + .build(); + detector.setRules(Collections.singletonList(rule)); + return detector; + } + + @Override + protected Detector createTestInstance() { + DetectorFunction function = randomFrom(EnumSet.allOf(DetectorFunction.class)); + Detector.Builder detector = new Detector.Builder(function, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20)); + if (randomBoolean()) { + detector.setDetectorDescription(randomAlphaOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + detector.setPartitionFieldName(randomAlphaOfLengthBetween(6, 20)); + } else if (randomBoolean()) { + detector.setOverFieldName(randomAlphaOfLengthBetween(6, 20)); + } else if (randomBoolean()) { + detector.setByFieldName(randomAlphaOfLengthBetween(6, 20)); + } + if (randomBoolean()) { + detector.setExcludeFrequent(randomFrom(Detector.ExcludeFrequent.values())); + } + if (randomBoolean()) { + int size = randomInt(10); + List rules = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + // no need for random DetectionRule (it is already tested) + rules.add(new DetectionRule.Builder(Collections.singletonList(RuleConditionTests.createRandom())).build()); + } + detector.setRules(rules); + } + if (randomBoolean()) { + detector.setUseNull(randomBoolean()); + } + return detector.build(); + } + + @Override + protected Detector doParseInstance(XContentParser parser) { + return Detector.PARSER.apply(parser, null).build(); + } + + public void testExcludeFrequentForString() { + assertEquals(Detector.ExcludeFrequent.ALL, Detector.ExcludeFrequent.forString("all")); + assertEquals(Detector.ExcludeFrequent.ALL, Detector.ExcludeFrequent.forString("ALL")); + assertEquals(Detector.ExcludeFrequent.NONE, Detector.ExcludeFrequent.forString("none")); + assertEquals(Detector.ExcludeFrequent.NONE, Detector.ExcludeFrequent.forString("NONE")); + assertEquals(Detector.ExcludeFrequent.BY, Detector.ExcludeFrequent.forString("by")); + assertEquals(Detector.ExcludeFrequent.BY, Detector.ExcludeFrequent.forString("BY")); + assertEquals(Detector.ExcludeFrequent.OVER, Detector.ExcludeFrequent.forString("over")); + assertEquals(Detector.ExcludeFrequent.OVER, Detector.ExcludeFrequent.forString("OVER")); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRefTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRefTests.java new file mode 100644 index 00000000000..00862e5307b --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/FilterRefTests.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +public class FilterRefTests extends AbstractXContentTestCase { + + @Override + protected FilterRef createTestInstance() { + return new FilterRef(randomAlphaOfLength(20), randomFrom(FilterRef.FilterType.values())); + } + + @Override + protected FilterRef doParseInstance(XContentParser parser) throws IOException { + return FilterRef.PARSER.parse(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilterTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilterTests.java new file mode 100644 index 00000000000..6c595e2d6da --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/MlFilterTests.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.util.SortedSet; +import java.util.TreeSet; + +import static org.hamcrest.Matchers.contains; + +public class MlFilterTests extends AbstractXContentTestCase { + + public static MlFilter createTestFilter() { + return new MlFilterTests().createTestInstance(); + } + + @Override + protected MlFilter createTestInstance() { + return createRandom(); + } + + public static MlFilter createRandom() { + return createRandom(randomAlphaOfLength(10)); + } + + public static MlFilter createRandom(String filterId) { + String description = null; + if (randomBoolean()) { + description = randomAlphaOfLength(20); + } + + int size = randomInt(10); + SortedSet items = new TreeSet<>(); + for (int i = 0; i < size; i++) { + items.add(randomAlphaOfLengthBetween(1, 20)); + } + return MlFilter.builder(filterId).setDescription(description).setItems(items).build(); + } + + @Override + protected MlFilter doParseInstance(XContentParser parser) { + return MlFilter.PARSER.apply(parser, null).build(); + } + + public void testNullId() { + expectThrows(NullPointerException.class, () -> MlFilter.builder(null).build()); + } + + public void testNullItems() { + expectThrows(NullPointerException.class, + () -> MlFilter.builder(randomAlphaOfLength(10)).setItems((SortedSet) null).build()); + } + + public void testItemsAreSorted() { + MlFilter filter = MlFilter.builder("foo").setItems("c", "b", "a").build(); + assertThat(filter.getItems(), contains("a", "b", "c")); + } + + public void testGetItemsReturnsUnmodifiable() { + MlFilter filter = MlFilter.builder("foo").setItems("c", "b", "a").build(); + expectThrows(UnsupportedOperationException.class, () -> filter.getItems().add("x")); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleConditionTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleConditionTests.java new file mode 100644 index 00000000000..4348ea194d0 --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleConditionTests.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +public class RuleConditionTests extends AbstractXContentTestCase { + + @Override + protected RuleCondition createTestInstance() { + return createRandom(); + } + + public static RuleCondition createRandom() { + RuleCondition.AppliesTo appliesTo = randomFrom(RuleCondition.AppliesTo.values()); + Operator operator = randomFrom(Operator.LT, Operator.LTE, Operator.GT, Operator.GTE); + return new RuleCondition(appliesTo, operator, randomDouble()); + } + + @Override + protected RuleCondition doParseInstance(XContentParser parser) { + return RuleCondition.PARSER.apply(parser, null); + } + + public void testEqualsGivenSameObject() { + RuleCondition condition = createRandom(); + assertTrue(condition.equals(condition)); + } + + public void testEqualsGivenString() { + assertFalse(createRandom().equals("a string")); + } + + public void testCreateTimeBased() { + RuleCondition timeBased = RuleCondition.createTime(Operator.GTE, 100L); + assertEquals(RuleCondition.AppliesTo.TIME, timeBased.getAppliesTo()); + assertEquals(Operator.GTE, timeBased.getOperator()); + assertEquals(100.0, timeBased.getValue(), 0.000001); + } + + public void testAppliesToFromString() { + assertEquals(RuleCondition.AppliesTo.ACTUAL, RuleCondition.AppliesTo.fromString("actual")); + assertEquals(RuleCondition.AppliesTo.ACTUAL, RuleCondition.AppliesTo.fromString("ACTUAL")); + assertEquals(RuleCondition.AppliesTo.TYPICAL, RuleCondition.AppliesTo.fromString("typical")); + assertEquals(RuleCondition.AppliesTo.TYPICAL, RuleCondition.AppliesTo.fromString("TYPICAL")); + assertEquals(RuleCondition.AppliesTo.DIFF_FROM_TYPICAL, RuleCondition.AppliesTo.fromString("diff_from_typical")); + assertEquals(RuleCondition.AppliesTo.DIFF_FROM_TYPICAL, RuleCondition.AppliesTo.fromString("DIFF_FROM_TYPICAL")); + assertEquals(RuleCondition.AppliesTo.TIME, RuleCondition.AppliesTo.fromString("time")); + assertEquals(RuleCondition.AppliesTo.TIME, RuleCondition.AppliesTo.fromString("TIME")); + } + + public void testAppliesToToString() { + assertEquals("actual", RuleCondition.AppliesTo.ACTUAL.toString()); + assertEquals("typical", RuleCondition.AppliesTo.TYPICAL.toString()); + assertEquals("diff_from_typical", RuleCondition.AppliesTo.DIFF_FROM_TYPICAL.toString()); + assertEquals("time", RuleCondition.AppliesTo.TIME.toString()); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScopeTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScopeTests.java new file mode 100644 index 00000000000..ac97e457ac4 --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/RuleScopeTests.java @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.contains; + +public class RuleScopeTests extends AbstractXContentTestCase { + + @Override + protected RuleScope createTestInstance() { + RuleScope.Builder scope = RuleScope.builder(); + int count = randomIntBetween(0, 3); + for (int i = 0; i < count; ++i) { + if (randomBoolean()) { + scope.include(randomAlphaOfLength(20), randomAlphaOfLength(20)); + } else { + scope.exclude(randomAlphaOfLength(20), randomAlphaOfLength(20)); + } + } + return scope.build(); + } + + public void testGetReferencedFilters_GivenEmpty() { + assertTrue(RuleScope.builder().build().getReferencedFilters().isEmpty()); + } + + public void testGetReferencedFilters_GivenMultipleFields() { + RuleScope scope = RuleScope.builder() + .include("foo", "filter1") + .exclude("bar", "filter2") + .include("foobar", "filter3") + .build(); + assertThat(scope.getReferencedFilters(), contains("filter1", "filter2", "filter3")); + } + + @Override + protected RuleScope doParseInstance(XContentParser parser) throws IOException { + return RuleScope.parser().parse(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } +} From d05f39de8be1286b6360b327a3e9d1645fa2cd36 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Fri, 3 Aug 2018 11:47:02 +0100 Subject: [PATCH 11/11] [TEST} unmutes SearchAsyncActionTests and adds debugging info This unmutes the testFanOutAndCollect()` method and add a check to make sure we aren't accidentally running something twice causing a search phase to still be running after we have counted down the latch Relates to #29242 --- .../elasticsearch/action/search/SearchAsyncActionTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index 2726de23758..ace92159639 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -256,7 +256,6 @@ public class SearchAsyncActionTests extends ESTestCase { assertEquals(10, numRequests.get()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29242") public void testFanOutAndCollect() throws InterruptedException { SearchRequest request = new SearchRequest(); request.allowPartialSearchResults(true); @@ -347,6 +346,9 @@ public class SearchAsyncActionTests extends ESTestCase { sendReleaseSearchContext(result.getRequestId(), new MockConnection(result.node), OriginalIndices.NONE); } responseListener.onResponse(response); + if (latch.getCount() == 0) { + throw new AssertionError("Running a search phase after the latch has reached 0 !!!!"); + } latch.countDown(); } };