Remove the _all metadata field (elastic/x-pack-elasticsearch#2356)

This change removes the `_all` metadata field. This field is deprecated in 6
and cannot be activated for indices created in 6 so it can be safely removed in
the next major version (e.g. 7).

Relates https://github.com/elastic/elasticsearch/pull/26356

Original commit: elastic/x-pack-elasticsearch@a47133c94e

docs/en/security/authorization/field-and-document-access-control.asciidoc

@@ -62,8 +62,7 @@ The following role definition grants read access only to the `category`,
 }
 --------------------------------------------------
 
-To allow access to the `_all` meta field, you must explicitly list it as an
+Access to the following meta fields is always allowed: `_id`,
-allowed field. Access to the following meta fields is always allowed: `_id`,
 `_type`, `_parent`, `_routing`, `_timestamp`, `_ttl`, `_size` and `_index`. If
 you specify an empty list of fields, only these meta fields are accessible.
 

plugin/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -46,11 +46,9 @@ public class DeprecationChecks {
     @SuppressWarnings("unchecked")
     static List<Function<IndexMetaData, DeprecationIssue>> INDEX_SETTINGS_CHECKS =
         Collections.unmodifiableList(Arrays.asList(
-            IndexDeprecationChecks::allMetaFieldIsDisabledByDefaultCheck,
             IndexDeprecationChecks::baseSimilarityDefinedCheck,
             IndexDeprecationChecks::coercionCheck,
             IndexDeprecationChecks::dynamicTemplateWithMatchMappingTypeCheck,
-            IndexDeprecationChecks::includeInAllCheck,
             IndexDeprecationChecks::indexSharedFileSystemCheck,
             IndexDeprecationChecks::indexStoreTypeCheck,
             IndexDeprecationChecks::storeThrottleSettingsCheck));

plugin/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java

@@ -12,7 +12,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.index.mapper.DynamicTemplate;
 
 import java.util.ArrayList;
@@ -95,50 +94,6 @@ public class IndexDeprecationChecks {
         return null;
     }
 
-    @SuppressWarnings("unchecked")
-    static DeprecationIssue allMetaFieldIsDisabledByDefaultCheck(IndexMetaData indexMetaData) {
-        if (indexMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) {
-            List<String> issues = new ArrayList<>();
-            fieldLevelMappingIssue(indexMetaData, (mappingMetaData, sourceAsMap) -> {
-                Map<String, Object> allMetaData = (Map<String, Object>) sourceAsMap.getOrDefault("_all", Collections.emptyMap());
-                Object enabledObj = allMetaData.get("enabled");
-                if (enabledObj != null) {
-                    enabledObj = Booleans.parseBooleanLenient(enabledObj.toString(),
-                        AllFieldMapper.Defaults.ENABLED.enabled);
-                }
-                if (Boolean.TRUE.equals(enabledObj)) {
-                    issues.add(mappingMetaData.type());
-                }
-            });
-            if (issues.size() > 0) {
-                return new DeprecationIssue(DeprecationIssue.Level.INFO,
-                    "The _all meta field is disabled by default on indices created in 6.0",
-                    "https://www.elastic.co/guide/en/elasticsearch/reference/master/" +
-                        "breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default",
-                    "types: " + issues.toString());
-            }
-        }
-        return null;
-    }
-
-    static DeprecationIssue includeInAllCheck(IndexMetaData indexMetaData) {
-        if (indexMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) {
-            List<String> issues = new ArrayList<>();
-            fieldLevelMappingIssue(indexMetaData, (mappingMetaData, sourceAsMap) -> {
-                issues.addAll(findInPropertiesRecursively(mappingMetaData.type(), sourceAsMap,
-                    property -> property.containsKey("include_in_all")));
-            });
-            if (issues.size() > 0) {
-                return new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
-                    "The [include_in_all] mapping parameter is now disallowed",
-                    "https://www.elastic.co/guide/en/elasticsearch/reference/master/" +
-                        "breaking_60_mappings_changes.html#_the_literal_include_in_all_literal_mapping_parameter_is_now_disallowed",
-                    issues.toString());
-            }
-        }
-        return null;
-    }
-
     static DeprecationIssue dynamicTemplateWithMatchMappingTypeCheck(IndexMetaData indexMetaData) {
         if (indexMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) {
             List<String> issues = new ArrayList<>();

plugin/src/main/java/org/elasticsearch/xpack/security/authz/permission/FieldPermissions.java

@@ -16,7 +16,6 @@ import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.xpack.security.authz.accesscontrol.FieldSubsetReader;
 import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsDefinition.FieldGrantExcludeGroup;
 import org.elasticsearch.xpack.security.support.Automatons;
@@ -153,24 +152,10 @@ public final class FieldPermissions implements Accountable {
                     Strings.arrayToCommaDelimitedString(grantedFields));
         }
 
-        if (!containsAllField(grantedFields) && !containsAllField(deniedFields)) {
-            // It is not explicitly stated whether _all should be allowed/denied
-            // In that case we automatically disable _all, unless all fields would match
-            if (Operations.isTotal(grantedFieldsAutomaton) && Operations.isEmpty(deniedFieldsAutomaton)) {
-                // all fields are accepted, so using _all is fine
-            } else {
-                deniedFieldsAutomaton = Operations.union(deniedFieldsAutomaton, Automata.makeString(AllFieldMapper.NAME));
-            }
-        }
-
         grantedFieldsAutomaton = minusAndMinimize(grantedFieldsAutomaton, deniedFieldsAutomaton);
         return grantedFieldsAutomaton;
     }
 
-    private static boolean containsAllField(String[] fields) {
-        return fields != null && Arrays.stream(fields).anyMatch(AllFieldMapper.NAME::equals);
-    }
-
     /**
      * Returns true if this field permission policy allows access to the field and false if not.
      * fieldName can be a wildcard.

plugin/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java

@@ -34,10 +34,6 @@ public class IndexDeprecationChecksTests extends ESTestCase {
     public void testCoerceBooleanDeprecation() throws IOException {
         XContentBuilder mapping = XContentFactory.jsonBuilder();
         mapping.startObject(); {
-            mapping.startObject("_all"); {
-                mapping.field("enabled", false);
-            }
-            mapping.endObject();
             mapping.startObject("properties"); {
                 mapping.startObject("my_boolean"); {
                     mapping.field("type", "boolean");
@@ -86,73 +82,9 @@ public class IndexDeprecationChecksTests extends ESTestCase {
         assertEquals(singletonList(expected), issues);
     }
 
-    public void testAllMetaFieldIsDisabledByDefaultCheck() throws IOException {
-        XContentBuilder mapping = XContentFactory.jsonBuilder();
-        mapping.startObject(); {
-            mapping.startObject("_all"); {
-                mapping.field("enabled", randomFrom("1", 1, "true", true));
-            }
-            mapping.endObject();
-        }
-        mapping.endObject();
-
-        IndexMetaData indexMetaData = IndexMetaData.builder("test")
-            .putMapping("testAllEnabled", mapping.string())
-            .settings(settings(Version.V_5_6_0))
-            .numberOfShards(1)
-            .numberOfReplicas(0)
-            .build();
-
-        DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.INFO,
-            "The _all meta field is disabled by default on indices created in 6.0",
-            "https://www.elastic.co/guide/en/elasticsearch/reference/master/" +
-                "breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default",
-            "types: [testAllEnabled]");
-        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData));
-        assertEquals(singletonList(expected), issues);
-    }
-
-    public void testIncludeInAllCheck() throws IOException {
-        XContentBuilder mapping = XContentFactory.jsonBuilder();
-        mapping.startObject(); {
-            mapping.startObject("_all"); {
-                mapping.field("enabled", false);
-            }
-            mapping.endObject();
-            mapping.startObject("properties"); {
-                mapping.startObject("my_field"); {
-                    mapping.field("type", "text");
-                    mapping.field("include_in_all", false);
-                }
-                mapping.endObject();
-            }
-            mapping.endObject();
-        }
-        mapping.endObject();
-
-        IndexMetaData indexMetaData = IndexMetaData.builder("test")
-            .putMapping("testIncludeInAll", mapping.string())
-            .settings(settings(Version.V_5_6_0))
-            .numberOfShards(1)
-            .numberOfReplicas(0)
-            .build();
-
-        DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
-            "The [include_in_all] mapping parameter is now disallowed",
-            "https://www.elastic.co/guide/en/elasticsearch/reference/master/" +
-                "breaking_60_mappings_changes.html#_the_literal_include_in_all_literal_mapping_parameter_is_now_disallowed",
-            "[[type: testIncludeInAll, field: my_field]]");
-        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData));
-        assertEquals(singletonList(expected), issues);
-    }
-
     public void testMatchMappingTypeCheck() throws IOException {
         XContentBuilder mapping = XContentFactory.jsonBuilder();
         mapping.startObject(); {
-            mapping.startObject("_all"); {
-                mapping.field("enabled", false);
-            }
-            mapping.endObject();
             mapping.startArray("dynamic_templates");
             {
                 mapping.startObject();

plugin/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java

@@ -45,7 +45,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
-import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParentFieldMapper;
@@ -108,14 +107,13 @@ import static org.mockito.Mockito.when;
 
 public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
 
-    private static final Set<String> META_FIELDS_WITHOUT_ALL;
+    private static final Set<String> META_FIELDS;
     static {
-        final Set<String> metaFieldsWithoutAll = new HashSet<>(Arrays.asList(MapperService.getAllMetaFields()));
+        final Set<String> metaFields = new HashSet<>(Arrays.asList(MapperService.getAllMetaFields()));
-        metaFieldsWithoutAll.add(SourceFieldMapper.NAME);
+        metaFields.add(SourceFieldMapper.NAME);
-        metaFieldsWithoutAll.add(FieldNamesFieldMapper.NAME);
+        metaFields.add(FieldNamesFieldMapper.NAME);
-        metaFieldsWithoutAll.add(SeqNoFieldMapper.NAME);
+        metaFields.add(SeqNoFieldMapper.NAME);
-        metaFieldsWithoutAll.remove(AllFieldMapper.NAME);
+        META_FIELDS = Collections.unmodifiableSet(metaFields);
-        META_FIELDS_WITHOUT_ALL = Collections.unmodifiableSet(metaFieldsWithoutAll);
     }
 
     private ThreadContext threadContext;
@@ -178,8 +176,6 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
         assertThat(result.getFilter().run("_index"), is(true));
         assertThat(result.getFilter().run("_field_names"), is(true));
         assertThat(result.getFilter().run("_seq_no"), is(true));
-        // _all contains actual user data and therefor can't be included by default
-        assertThat(result.getFilter().run("_all"), is(false));
         assertThat(result.getFilter().run("_some_random_meta_field"), is(true));
         assertThat(result.getFilter().run("some_random_regular_field"), is(false));
     }
@@ -201,7 +197,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
     }
 
     public void testWildcards() throws Exception {
-        Set<String> expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        Set<String> expected = new HashSet<>(META_FIELDS);
         expected.add("field1_a");
         expected.add("field1_b");
         expected.add("field1_c");
@@ -209,17 +205,17 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
     }
 
     public void testDotNotion() throws Exception {
-        Set<String> expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        Set<String> expected = new HashSet<>(META_FIELDS);
         expected.add("foo.bar");
         assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"foo.bar"}, null)), expected, "foo", "foo.baz", "bar.foo");
 
-        expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        expected = new HashSet<>(META_FIELDS);
         expected.add("foo.bar");
         assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"foo.*"}, null)), expected, "foo", "bar");
     }
 
     public void testParentChild() throws Exception {
-        Set<String> expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        Set<String> expected = new HashSet<>(META_FIELDS);
         expected.add(ParentFieldMapper.joinField("parent1"));
         expected.add("foo");
         assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"foo"}, null)), expected, "bar");
@@ -340,20 +336,19 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
                 new SecurityIndexSearcherWrapper(indexSettings, null, null, threadContext, licenseState, null);
         String[] grantedFields = new String[]{};
         String[] deniedFields;
-        Set<String> expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        Set<String> expected = new HashSet<>(META_FIELDS);
         // Presence of fields in a role with an empty array implies access to no fields except the meta fields
         assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, randomBoolean() ? null : new String[]{})),
                 expected, "foo", "bar");
 
         // make sure meta fields cannot be denied access to
-        deniedFields = META_FIELDS_WITHOUT_ALL.toArray(new String[0]);
+        deniedFields = META_FIELDS.toArray(new String[0]);
         assertResolved(new FieldPermissions(fieldPermissionDef(null, deniedFields)),
                 new HashSet<>(Arrays.asList("foo", "bar", "_some_plugin_meta_field")));
 
         // check we can add all fields with *
         grantedFields = new String[]{"*"};
-        expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        expected = new HashSet<>(META_FIELDS);
-        expected.add(AllFieldMapper.NAME);
         expected.add("foo");
         assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, randomBoolean() ? null : new String[]{})), expected);
 
@@ -364,57 +359,49 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
         // check we remove only excluded fields
         grantedFields = new String[]{"*"};
         deniedFields = new String[]{"xfield"};
-        expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        expected = new HashSet<>(META_FIELDS);
         expected.add("foo");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield", "_all");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield");
 
         // same with null
         grantedFields = null;
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield", "_all");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield");
 
         // some other checks
         grantedFields = new String[]{"field*"};
         deniedFields = new String[]{"field1", "field2"};
-        expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        expected = new HashSet<>(META_FIELDS);
         expected.add("field3");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2", "_all");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2");
 
         grantedFields = new String[]{"field1", "field2"};
         deniedFields = new String[]{"field2"};
-        expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        expected = new HashSet<>(META_FIELDS);
         expected.add("field1");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2", "_all");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2");
 
         grantedFields = new String[]{"field*"};
         deniedFields = new String[]{"field2"};
-        expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        expected = new HashSet<>(META_FIELDS);
         expected.add("field1");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field2", "_all");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field2");
 
         deniedFields = new String[]{"field*"};
         assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)),
-                META_FIELDS_WITHOUT_ALL, "field1", "field2");
+                META_FIELDS, "field1", "field2");
 
         // empty array for allowed fields always means no field is allowed
         grantedFields = new String[]{};
         deniedFields = new String[]{};
         assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)),
-                META_FIELDS_WITHOUT_ALL, "field1", "field2");
+                META_FIELDS, "field1", "field2");
 
         // make sure all field can be explicitly allowed
-        grantedFields = new String[]{"_all", "*"};
+        grantedFields = new String[]{"*"};
         deniedFields = randomBoolean() ? null : new String[]{};
-        expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
+        expected = new HashSet<>(META_FIELDS);
-        expected.add("_all");
         expected.add("field1");
         assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected);
-
-        // make sure all field can be explicitly allowed
-        grantedFields = new String[]{"_all"};
-        deniedFields = randomBoolean() ? null : new String[]{};
-        expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
-        expected.add("_all");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "_source");
     }
 
     private SparseFixedBitSet query(LeafReaderContext leaf, String field, String value) throws IOException {

plugin/src/test/java/org/elasticsearch/xpack/security/authz/permission/FieldPermissionsCacheTests.java

@@ -88,21 +88,6 @@ public class FieldPermissionsCacheTests extends ESTestCase {
         assertTrue(mergedFieldPermissions.grantsAccessTo("a"));
         assertTrue(mergedFieldPermissions.grantsAccessTo("b"));
 
-        // test merge does not remove _all
-        allowed1 = new String[]{"_all"};
-        allowed2 = new String[]{};
-        denied1 = null;
-        denied2 = null;
-        fieldPermissions1 = randomBoolean() ? fieldPermissionsCache.getFieldPermissions(allowed1, denied1) :
-                new FieldPermissions(fieldPermissionDef(allowed1, denied1));
-        fieldPermissions2 = randomBoolean() ? fieldPermissionsCache.getFieldPermissions(allowed2, denied2) :
-                new FieldPermissions(fieldPermissionDef(allowed2, denied2));
-        mergedFieldPermissions =
-                fieldPermissionsCache.getFieldPermissions(Arrays.asList(fieldPermissions1, fieldPermissions2));
-        assertTrue(fieldPermissions1.grantsAccessTo("_all"));
-        assertFalse(fieldPermissions2.grantsAccessTo("_all"));
-        assertTrue(mergedFieldPermissions.grantsAccessTo("_all"));
-
         allowed1 = new String[] { "a*" };
         allowed2 = new String[] { "b*" };
         denied1 = new String[] { "aa*" };

plugin/src/test/java/org/elasticsearch/xpack/security/authz/permission/FieldPermissionsTests.java

@@ -224,25 +224,4 @@ public class FieldPermissionsTests extends ESTestCase {
             assertEquals((Integer) hashCode, hashCodes.get(i));
         }
     }
-
-    public void testAllFieldIsAutomaticallyExcludedIfNotExplicitlyGranted() throws Exception {
-        final FieldPermissions fieldPermissions = new FieldPermissions(
-                new FieldPermissionsDefinition(new String[] { "_a*" }, new String[0]));
-        assertTrue(fieldPermissions.grantsAccessTo("_animal"));
-        assertFalse(fieldPermissions.grantsAccessTo("_all"));
-    }
-
-    public void testAllFieldIsNotExcludedIfExplicitlyGranted() throws Exception {
-        final String[] grant = { "foo", "bar", "baz", "_all" };
-        Collections.shuffle(Arrays.asList(grant), random());
-
-        final FieldPermissions fieldPermissions = new FieldPermissions(
-                new FieldPermissionsDefinition(grant, new String[0]));
-
-        assertTrue(fieldPermissions.grantsAccessTo("_all"));
-        assertTrue(fieldPermissions.grantsAccessTo("foo"));
-        assertTrue(fieldPermissions.grantsAccessTo("bar"));
-        assertTrue(fieldPermissions.grantsAccessTo("baz"));
-        assertFalse(fieldPermissions.grantsAccessTo(randomAlphaOfLengthBetween(5, 8)));
-    }
 }