security: use lucene automatons and remove dependency on briks

This commit removes the dependency on the briks automatons library and instead uses the lucene
version. Shield was originally implemented using the lucene version, but issues arose with supporting
multiple versions of elasticsearch and API changes, so we moved to using the briks library.

x-pack and elasticsearch are always the same version so we can use the lucene version of the
automatons and remove the briks library. This also brings with it protection from huge automatons
that we did not have before.

Original commit: elastic/x-pack-elasticsearch@e3f34b6b55

elasticsearch/build.gradle

@@ -33,7 +33,6 @@ dependencies {
   // security deps
   compile project(path: ':modules:transport-netty3', configuration: 'runtime')
   compile project(path: ':modules:transport-netty4', configuration: 'runtime')
-  compile 'dk.brics.automaton:automaton:1.11-8'
   compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
   compile 'org.bouncycastle:bcprov-jdk15on:1.55'
   compile 'org.bouncycastle:bcpkix-jdk15on:1.55'

elasticsearch/src/main/java/org/elasticsearch/xpack/security/authz/permission/FieldPermissions.java

@@ -5,14 +5,13 @@
  */
 package org.elasticsearch.xpack.security.authz.permission;
 
-import dk.brics.automaton.Automaton;
+import org.apache.lucene.util.automaton.Automaton;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -27,6 +26,15 @@ import java.util.Collection;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.apache.lucene.util.automaton.MinimizationOperations.minimize;
+import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES;
+import static org.apache.lucene.util.automaton.Operations.isTotal;
+import static org.apache.lucene.util.automaton.Operations.run;
+import static org.apache.lucene.util.automaton.Operations.sameLanguage;
+import static org.apache.lucene.util.automaton.Operations.subsetOf;
+import static org.apache.lucene.util.automaton.Operations.union;
+import static org.elasticsearch.xpack.security.support.Automatons.minusAndDeterminize;
+
 /**
  * Stores patterns to fields which access is granted or denied to and maintains an automaton that can be used to check if permission is
  * allowed for a specific field.
@@ -93,13 +101,13 @@ public class FieldPermissions implements Writeable, ToXContent {
         } else {
             deniedFieldsAutomaton = Automatons.patterns(deniedFieldsArray);
         }
-        if (deniedFieldsAutomaton.subsetOf(grantedFieldsAutomaton) == false) {
+        if (subsetOf(deniedFieldsAutomaton, grantedFieldsAutomaton) == false) {
             throw new ElasticsearchSecurityException("Exceptions for field permissions must be a subset of the " +
                     "granted fields but " + Arrays.toString(deniedFieldsArray) + " is not a subset of " +
                     Arrays.toString(grantedFieldsArray));
         }
 
-        grantedFieldsAutomaton = grantedFieldsAutomaton.minus(deniedFieldsAutomaton);
+        grantedFieldsAutomaton = minusAndDeterminize(grantedFieldsAutomaton, deniedFieldsAutomaton);
         return grantedFieldsAutomaton;
     }
 
@@ -176,27 +184,23 @@ public class FieldPermissions implements Writeable, ToXContent {
      * fieldName can be a wildcard.
      */
     public boolean grantsAccessTo(String fieldName) {
-        if (permittedFieldsAutomaton.isTotal()) {
+        return isTotal(permittedFieldsAutomaton) || run(permittedFieldsAutomaton, fieldName);
-            return true;
-        } else {
-            return permittedFieldsAutomaton.run(fieldName);
-        }
     }
 
     // Also, if one grants no access to fields and the other grants all access, merging should result in all access...
     public static FieldPermissions merge(FieldPermissions p1, FieldPermissions p2) {
         Automaton mergedPermittedFieldsAutomaton;
         // we only allow the union of the two automatons
-        mergedPermittedFieldsAutomaton = p1.permittedFieldsAutomaton.union(p2.permittedFieldsAutomaton);
+        mergedPermittedFieldsAutomaton = union(p1.permittedFieldsAutomaton, p2.permittedFieldsAutomaton);
         // need to minimize otherwise isTotal() might return false even if one of the merged ones returned true before
-        mergedPermittedFieldsAutomaton.minimize();
+        mergedPermittedFieldsAutomaton = minimize(mergedPermittedFieldsAutomaton, DEFAULT_MAX_DETERMINIZED_STATES);
         // if one of them allows access to _all we allow it for the merged too
         boolean allFieldIsAllowedInMerged = p1.allFieldIsAllowed || p2.allFieldIsAllowed;
         return new MergedFieldPermissions(mergedPermittedFieldsAutomaton, allFieldIsAllowedInMerged);
     }
 
     public boolean hasFieldLevelSecurity() {
-        return permittedFieldsAutomaton.isTotal() == false;
+        return isTotal(permittedFieldsAutomaton) == false;
     }
 
     public Set<String> resolveAllowedFields(Set<String> allowedMetaFields, MapperService mapperService) {
@@ -229,7 +233,7 @@ public class FieldPermissions implements Writeable, ToXContent {
         if (!Arrays.equals(grantedFieldsArray, that.grantedFieldsArray)) return false;
         // Probably incorrect - comparing Object[] arrays with Arrays.equals
         if (!Arrays.equals(deniedFieldsArray, that.deniedFieldsArray)) return false;
-        return permittedFieldsAutomaton.equals(that.permittedFieldsAutomaton);
+        return sameLanguage(permittedFieldsAutomaton, that.permittedFieldsAutomaton);
 
     }
 

elasticsearch/src/main/java/org/elasticsearch/xpack/security/authz/privilege/AbstractAutomatonPrivilege.java

@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.security.authz.privilege;
 
-import dk.brics.automaton.Automaton;
+import org.apache.lucene.util.automaton.Automaton;
-import dk.brics.automaton.BasicOperations;
 import org.elasticsearch.xpack.security.support.AutomatonPredicate;
 import org.elasticsearch.xpack.security.support.Automatons;
 
 import java.util.function.Predicate;
 
+import static org.apache.lucene.util.automaton.Operations.subsetOf;
 import static org.elasticsearch.xpack.security.support.Automatons.patterns;
 
 @SuppressWarnings("unchecked")
@@ -49,19 +49,9 @@ abstract class AbstractAutomatonPrivilege<P extends AbstractAutomatonPrivilege<P
         return create(name.add(other.name), Automatons.unionAndDeterminize(automaton, other.automaton));
     }
 
-    protected P minus(P other) {
-        if (other.implies((P) this)) {
-            return none();
-        }
-        if (other == none() || !this.implies(other)) {
-            return (P) this;
-        }
-        return create(name.remove(other.name), Automatons.minusAndDeterminize(automaton, other.automaton));
-    }
-
     @Override
     public boolean implies(P other) {
-        return BasicOperations.subsetOf(other.automaton, automaton);
+        return subsetOf(other.automaton, automaton);
     }
 
     @Override

elasticsearch/src/main/java/org/elasticsearch/xpack/security/authz/privilege/ClusterPrivilege.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.authz.privilege;
 
-import dk.brics.automaton.Automaton;
+import org.apache.lucene.util.automaton.Automaton;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.xpack.security.support.Automatons;
 

elasticsearch/src/main/java/org/elasticsearch/xpack/security/authz/privilege/GeneralPrivilege.java

@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.security.authz.privilege;
 
-import dk.brics.automaton.Automaton;
+import org.apache.lucene.util.automaton.Automaton;
-import dk.brics.automaton.BasicAutomata;
+import org.elasticsearch.xpack.security.support.Automatons;
 
 public class GeneralPrivilege extends AbstractAutomatonPrivilege<GeneralPrivilege> {
 
-    public static final GeneralPrivilege NONE = new GeneralPrivilege(Name.NONE, BasicAutomata.makeEmpty());
+    public static final GeneralPrivilege NONE = new GeneralPrivilege(Name.NONE, Automatons.EMPTY);
-    public static final GeneralPrivilege ALL = new GeneralPrivilege(Name.ALL, "*");
+    public static final GeneralPrivilege ALL = new GeneralPrivilege(Name.ALL, Automatons.MATCH_ALL);
 
     public GeneralPrivilege(String name, String... patterns) {
         super(name, patterns);

elasticsearch/src/main/java/org/elasticsearch/xpack/security/authz/privilege/IndexPrivilege.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.authz.privilege;
 
-import dk.brics.automaton.Automaton;
+import org.apache.lucene.util.automaton.Automaton;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction;
 import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistAction;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;

elasticsearch/src/main/java/org/elasticsearch/xpack/security/support/AutomatonPredicate.java

@@ -5,21 +5,19 @@
  */
 package org.elasticsearch.xpack.security.support;
 
-import dk.brics.automaton.Automaton;
+import org.apache.lucene.util.automaton.Automaton;
-import dk.brics.automaton.RunAutomaton;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 
 import java.util.function.Predicate;
 
+import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES;
+
 public class AutomatonPredicate implements Predicate<String> {
 
-    private final RunAutomaton automaton;
+    private final CharacterRunAutomaton automaton;
 
     public AutomatonPredicate(Automaton automaton) {
-        this(new RunAutomaton(automaton, false));
+        this.automaton = new CharacterRunAutomaton(automaton, DEFAULT_MAX_DETERMINIZED_STATES);
-    }
-
-    public AutomatonPredicate(RunAutomaton automaton) {
-        this.automaton = automaton;
     }
 
     @Override

elasticsearch/src/main/java/org/elasticsearch/xpack/security/support/Automatons.java

@@ -5,24 +5,26 @@
  */
 package org.elasticsearch.xpack.security.support;
 
-import dk.brics.automaton.Automaton;
+import org.apache.lucene.util.automaton.Automata;
-import dk.brics.automaton.BasicAutomata;
+import org.apache.lucene.util.automaton.Automaton;
-import dk.brics.automaton.BasicOperations;
+import org.apache.lucene.util.automaton.RegExp;
-import dk.brics.automaton.RegExp;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 
-import static dk.brics.automaton.BasicOperations.minus;
+import static org.apache.lucene.util.automaton.MinimizationOperations.minimize;
-import static dk.brics.automaton.BasicOperations.union;
+import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES;
-import static dk.brics.automaton.MinimizationOperations.minimize;
+import static org.apache.lucene.util.automaton.Operations.concatenate;
+import static org.apache.lucene.util.automaton.Operations.determinize;
+import static org.apache.lucene.util.automaton.Operations.minus;
+import static org.apache.lucene.util.automaton.Operations.union;
 
 public final class Automatons {
 
-    public static final Automaton EMPTY = BasicAutomata.makeEmpty();
+    public static final Automaton EMPTY = Automata.makeEmpty();
-    public static final Automaton MATCH_ALL = BasicAutomata.makeAnyString();
+    public static final Automaton MATCH_ALL = Automata.makeAnyString();
 
     static final char WILDCARD_STRING = '*';     // String equality with support for wildcards
     static final char WILDCARD_CHAR = '?';       // Char equality with support for wildcards
@@ -43,7 +45,7 @@ public final class Automatons {
      */
     public static Automaton patterns(Collection<String> patterns) {
         if (patterns.isEmpty()) {
-            return BasicAutomata.makeEmpty();
+            return EMPTY;
         }
         Automaton automaton = null;
         for (String pattern : patterns) {
@@ -53,8 +55,7 @@ public final class Automatons {
                 automaton = union(automaton, pattern(pattern));
             }
         }
-        minimize(automaton); // minimal is also deterministic
+        return minimize(automaton, DEFAULT_MAX_DETERMINIZED_STATES); // minimal is also deterministic
-        return automaton;
     }
 
     /**
@@ -84,36 +85,34 @@ public final class Automatons {
             int length = 1;
             switch(c) {
                 case WILDCARD_STRING:
-                    automata.add(BasicAutomata.makeAnyString());
+                    automata.add(Automata.makeAnyString());
                     break;
                 case WILDCARD_CHAR:
-                    automata.add(BasicAutomata.makeAnyChar());
+                    automata.add(Automata.makeAnyChar());
                     break;
                 case WILDCARD_ESCAPE:
                     // add the next codepoint instead, if it exists
                     if (i + length < text.length()) {
                         final char nextChar = text.charAt(i + length);
                         length += 1;
-                        automata.add(BasicAutomata.makeChar(nextChar));
+                        automata.add(Automata.makeChar(nextChar));
                         break;
                     } // else fallthru, lenient parsing with a trailing \
                 default:
-                    automata.add(BasicAutomata.makeChar(c));
+                    automata.add(Automata.makeChar(c));
             }
             i += length;
         }
-        return BasicOperations.concatenate(automata);
+        return concatenate(automata);
     }
 
     public static Automaton unionAndDeterminize(Automaton a1, Automaton a2) {
         Automaton res = union(a1, a2);
-        res.determinize();
+        return determinize(res, DEFAULT_MAX_DETERMINIZED_STATES);
-        return res;
     }
 
     public static Automaton minusAndDeterminize(Automaton a1, Automaton a2) {
-        Automaton res = minus(a1, a2);
+        Automaton res = minus(a1, a2, DEFAULT_MAX_DETERMINIZED_STATES);
-        res.determinize();
+        return determinize(res, DEFAULT_MAX_DETERMINIZED_STATES);
-        return res;
     }
 }

elasticsearch/src/test/java/org/elasticsearch/xpack/security/support/AutomatonsTests.java

@@ -5,14 +5,14 @@
  */
 package org.elasticsearch.xpack.security.support;
 
-import dk.brics.automaton.Automaton;
+import org.apache.lucene.util.automaton.Automaton;
-import dk.brics.automaton.RunAutomaton;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.test.ESTestCase;
 
+import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES;
 import static org.elasticsearch.xpack.security.support.Automatons.pattern;
 import static org.elasticsearch.xpack.security.support.Automatons.patterns;
 import static org.elasticsearch.xpack.security.support.Automatons.wildcard;
-import static org.hamcrest.Matchers.is;
 
 public class AutomatonsTests extends ESTestCase {
     public void testPatternsUnionOfMultiplePatterns() throws Exception {
@@ -54,13 +54,13 @@ public class AutomatonsTests extends ESTestCase {
     }
 
     private void assertMatch(Automaton automaton, String text) {
-        RunAutomaton runAutomaton = new RunAutomaton(automaton, false);
+        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, DEFAULT_MAX_DETERMINIZED_STATES);
-        assertThat(runAutomaton.run(text), is(true));
+        assertTrue(runAutomaton.run(text));
     }
 
     private void assertMismatch(Automaton automaton, String text) {
-        RunAutomaton runAutomaton = new RunAutomaton(automaton, false);
+        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, DEFAULT_MAX_DETERMINIZED_STATES);
-        assertThat(runAutomaton.run(text), is(false));
+        assertFalse(runAutomaton.run(text));
     }
 
     private void assertInvalidPattern(String text) {