honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove the _all metadata field (elastic/x-pack-elasticsearch#2356) 

This change removes the `_all` metadata field. This field is deprecated in 6
and cannot be activated for indices created in 6 so it can be safely removed in
the next major version (e.g. 7).

Relates https://github.com/elastic/elasticsearch/pull/26356

Original commit: elastic/x-pack-elasticsearch@a47133c94e
This commit is contained in:
Jim Ferenczi 2017-08-28 13:01:27 +02:00 committed by GitHub
parent e18f04f3eb
commit 27d8b4c79c
8 changed files with 27 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/en/security/authorization/field-and-document-access-control.asciidoc
View File

@ -62,8 +62,7 @@ The following role definition grants read access only to the `category`,
}
--------------------------------------------------
To allow access to the `_all` meta field, you must explicitly list it as an
allowed field. Access to the following meta fields is always allowed: `_id`,
Access to the following meta fields is always allowed: `_id`,
`_type`, `_parent`, `_routing`, `_timestamp`, `_ttl`, `_size` and `_index`. If
you specify an empty list of fields, only these meta fields are accessible.

2
plugin/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
View File

@ -46,11 +46,9 @@ public class DeprecationChecks {
@SuppressWarnings("unchecked")
static List<Function<IndexMetaData, DeprecationIssue>> INDEX_SETTINGS_CHECKS =
Collections.unmodifiableList(Arrays.asList(
IndexDeprecationChecks::allMetaFieldIsDisabledByDefaultCheck,
IndexDeprecationChecks::baseSimilarityDefinedCheck,
IndexDeprecationChecks::coercionCheck,
IndexDeprecationChecks::dynamicTemplateWithMatchMappingTypeCheck,
IndexDeprecationChecks::includeInAllCheck,
IndexDeprecationChecks::indexSharedFileSystemCheck,
IndexDeprecationChecks::indexStoreTypeCheck,
IndexDeprecationChecks::storeThrottleSettingsCheck));

45
plugin/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
View File

@ -12,7 +12,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.AllFieldMapper;
import org.elasticsearch.index.mapper.DynamicTemplate;
import java.util.ArrayList;
@ -95,50 +94,6 @@ public class IndexDeprecationChecks {
return null;
}
@SuppressWarnings("unchecked")
static DeprecationIssue allMetaFieldIsDisabledByDefaultCheck(IndexMetaData indexMetaData) {
if (indexMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) {
List<String> issues = new ArrayList<>();
fieldLevelMappingIssue(indexMetaData, (mappingMetaData, sourceAsMap) -> {
Map<String, Object> allMetaData = (Map<String, Object>) sourceAsMap.getOrDefault("_all", Collections.emptyMap());
Object enabledObj = allMetaData.get("enabled");
if (enabledObj != null) {
enabledObj = Booleans.parseBooleanLenient(enabledObj.toString(),
AllFieldMapper.Defaults.ENABLED.enabled);
}
if (Boolean.TRUE.equals(enabledObj)) {
issues.add(mappingMetaData.type());
}
});
if (issues.size() > 0) {
return new DeprecationIssue(DeprecationIssue.Level.INFO,
"The _all meta field is disabled by default on indices created in 6.0",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/" +
"breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default",
"types: " + issues.toString());
}
}
return null;
}
static DeprecationIssue includeInAllCheck(IndexMetaData indexMetaData) {
if (indexMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) {
List<String> issues = new ArrayList<>();
fieldLevelMappingIssue(indexMetaData, (mappingMetaData, sourceAsMap) -> {
issues.addAll(findInPropertiesRecursively(mappingMetaData.type(), sourceAsMap,
property -> property.containsKey("include_in_all")));
});
if (issues.size() > 0) {
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
"The [include_in_all] mapping parameter is now disallowed",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/" +
"breaking_60_mappings_changes.html#_the_literal_include_in_all_literal_mapping_parameter_is_now_disallowed",
issues.toString());
}
}
return null;
}
static DeprecationIssue dynamicTemplateWithMatchMappingTypeCheck(IndexMetaData indexMetaData) {
if (indexMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) {
List<String> issues = new ArrayList<>();

15
plugin/src/main/java/org/elasticsearch/xpack/security/authz/permission/FieldPermissions.java
View File

@ -16,7 +16,6 @@ import org.apache.lucene.util.automaton.Operations;
import org.elasticsearch.ElasticsearchSecurityException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.index.mapper.AllFieldMapper;
import org.elasticsearch.xpack.security.authz.accesscontrol.FieldSubsetReader;
import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsDefinition.FieldGrantExcludeGroup;
import org.elasticsearch.xpack.security.support.Automatons;
@ -153,24 +152,10 @@ public final class FieldPermissions implements Accountable {
Strings.arrayToCommaDelimitedString(grantedFields));
}
if (!containsAllField(grantedFields) && !containsAllField(deniedFields)) {
// It is not explicitly stated whether _all should be allowed/denied
// In that case we automatically disable _all, unless all fields would match
if (Operations.isTotal(grantedFieldsAutomaton) && Operations.isEmpty(deniedFieldsAutomaton)) {
// all fields are accepted, so using _all is fine
} else {
deniedFieldsAutomaton = Operations.union(deniedFieldsAutomaton, Automata.makeString(AllFieldMapper.NAME));
}
}
grantedFieldsAutomaton = minusAndMinimize(grantedFieldsAutomaton, deniedFieldsAutomaton);
return grantedFieldsAutomaton;
}
private static boolean containsAllField(String[] fields) {
return fields != null && Arrays.stream(fields).anyMatch(AllFieldMapper.NAME::equals);
}
/**
* Returns true if this field permission policy allows access to the field and false if not.
* fieldName can be a wildcard.

68
plugin/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java
View File

@ -34,10 +34,6 @@ public class IndexDeprecationChecksTests extends ESTestCase {
public void testCoerceBooleanDeprecation() throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder();
mapping.startObject(); {
mapping.startObject("_all"); {
mapping.field("enabled", false);
}
mapping.endObject();
mapping.startObject("properties"); {
mapping.startObject("my_boolean"); {
mapping.field("type", "boolean");
@ -86,73 +82,9 @@ public class IndexDeprecationChecksTests extends ESTestCase {
assertEquals(singletonList(expected), issues);
}
public void testAllMetaFieldIsDisabledByDefaultCheck() throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder();
mapping.startObject(); {
mapping.startObject("_all"); {
mapping.field("enabled", randomFrom("1", 1, "true", true));
}
mapping.endObject();
}
mapping.endObject();
IndexMetaData indexMetaData = IndexMetaData.builder("test")
.putMapping("testAllEnabled", mapping.string())
.settings(settings(Version.V_5_6_0))
.numberOfShards(1)
.numberOfReplicas(0)
.build();
DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.INFO,
"The _all meta field is disabled by default on indices created in 6.0",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/" +
"breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default",
"types: [testAllEnabled]");
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData));
assertEquals(singletonList(expected), issues);
}
public void testIncludeInAllCheck() throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder();
mapping.startObject(); {
mapping.startObject("_all"); {
mapping.field("enabled", false);
}
mapping.endObject();
mapping.startObject("properties"); {
mapping.startObject("my_field"); {
mapping.field("type", "text");
mapping.field("include_in_all", false);
}
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();
IndexMetaData indexMetaData = IndexMetaData.builder("test")
.putMapping("testIncludeInAll", mapping.string())
.settings(settings(Version.V_5_6_0))
.numberOfShards(1)
.numberOfReplicas(0)
.build();
DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
"The [include_in_all] mapping parameter is now disallowed",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/" +
"breaking_60_mappings_changes.html#_the_literal_include_in_all_literal_mapping_parameter_is_now_disallowed",
"[[type: testIncludeInAll, field: my_field]]");
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData));
assertEquals(singletonList(expected), issues);
}
public void testMatchMappingTypeCheck() throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder();
mapping.startObject(); {
mapping.startObject("_all"); {
mapping.field("enabled", false);
}
mapping.endObject();
mapping.startArray("dynamic_templates");
{
mapping.startObject();

65
plugin/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java
View File

@ -45,7 +45,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.mapper.AllFieldMapper;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ParentFieldMapper;
@ -108,14 +107,13 @@ import static org.mockito.Mockito.when;
public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
private static final Set<String> META_FIELDS_WITHOUT_ALL;
private static final Set<String> META_FIELDS;
static {
final Set<String> metaFieldsWithoutAll = new HashSet<>(Arrays.asList(MapperService.getAllMetaFields()));
metaFieldsWithoutAll.add(SourceFieldMapper.NAME);
metaFieldsWithoutAll.add(FieldNamesFieldMapper.NAME);
metaFieldsWithoutAll.add(SeqNoFieldMapper.NAME);
metaFieldsWithoutAll.remove(AllFieldMapper.NAME);
META_FIELDS_WITHOUT_ALL = Collections.unmodifiableSet(metaFieldsWithoutAll);
final Set<String> metaFields = new HashSet<>(Arrays.asList(MapperService.getAllMetaFields()));
metaFields.add(SourceFieldMapper.NAME);
metaFields.add(FieldNamesFieldMapper.NAME);
metaFields.add(SeqNoFieldMapper.NAME);
META_FIELDS = Collections.unmodifiableSet(metaFields);
}
private ThreadContext threadContext;
@ -178,8 +176,6 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
assertThat(result.getFilter().run("_index"), is(true));
assertThat(result.getFilter().run("_field_names"), is(true));
assertThat(result.getFilter().run("_seq_no"), is(true));
// _all contains actual user data and therefor can't be included by default
assertThat(result.getFilter().run("_all"), is(false));
assertThat(result.getFilter().run("_some_random_meta_field"), is(true));
assertThat(result.getFilter().run("some_random_regular_field"), is(false));
}
@ -201,7 +197,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
}
public void testWildcards() throws Exception {
Set<String> expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
Set<String> expected = new HashSet<>(META_FIELDS);
expected.add("field1_a");
expected.add("field1_b");
expected.add("field1_c");
@ -209,17 +205,17 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
}
public void testDotNotion() throws Exception {
Set<String> expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
Set<String> expected = new HashSet<>(META_FIELDS);
expected.add("foo.bar");
assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"foo.bar"}, null)), expected, "foo", "foo.baz", "bar.foo");
expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
expected = new HashSet<>(META_FIELDS);
expected.add("foo.bar");
assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"foo.*"}, null)), expected, "foo", "bar");
}
public void testParentChild() throws Exception {
Set<String> expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
Set<String> expected = new HashSet<>(META_FIELDS);
expected.add(ParentFieldMapper.joinField("parent1"));
expected.add("foo");
assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"foo"}, null)), expected, "bar");
@ -340,20 +336,19 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
new SecurityIndexSearcherWrapper(indexSettings, null, null, threadContext, licenseState, null);
String[] grantedFields = new String[]{};
String[] deniedFields;
Set<String> expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
Set<String> expected = new HashSet<>(META_FIELDS);
// Presence of fields in a role with an empty array implies access to no fields except the meta fields
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, randomBoolean() ? null : new String[]{})),
expected, "foo", "bar");
// make sure meta fields cannot be denied access to
deniedFields = META_FIELDS_WITHOUT_ALL.toArray(new String[0]);
deniedFields = META_FIELDS.toArray(new String[0]);
assertResolved(new FieldPermissions(fieldPermissionDef(null, deniedFields)),
new HashSet<>(Arrays.asList("foo", "bar", "_some_plugin_meta_field")));
// check we can add all fields with *
grantedFields = new String[]{"*"};
expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
expected.add(AllFieldMapper.NAME);
expected = new HashSet<>(META_FIELDS);
expected.add("foo");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, randomBoolean() ? null : new String[]{})), expected);
@ -364,57 +359,49 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
// check we remove only excluded fields
grantedFields = new String[]{"*"};
deniedFields = new String[]{"xfield"};
expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
expected = new HashSet<>(META_FIELDS);
expected.add("foo");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield", "_all");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield");
// same with null
grantedFields = null;
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield", "_all");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield");
// some other checks
grantedFields = new String[]{"field*"};
deniedFields = new String[]{"field1", "field2"};
expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
expected = new HashSet<>(META_FIELDS);
expected.add("field3");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2", "_all");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2");
grantedFields = new String[]{"field1", "field2"};
deniedFields = new String[]{"field2"};
expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
expected = new HashSet<>(META_FIELDS);
expected.add("field1");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2", "_all");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2");
grantedFields = new String[]{"field*"};
deniedFields = new String[]{"field2"};
expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
expected = new HashSet<>(META_FIELDS);
expected.add("field1");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field2", "_all");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field2");
deniedFields = new String[]{"field*"};
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)),
META_FIELDS_WITHOUT_ALL, "field1", "field2");
META_FIELDS, "field1", "field2");
// empty array for allowed fields always means no field is allowed
grantedFields = new String[]{};
deniedFields = new String[]{};
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)),
META_FIELDS_WITHOUT_ALL, "field1", "field2");
META_FIELDS, "field1", "field2");
// make sure all field can be explicitly allowed
grantedFields = new String[]{"_all", "*"};
grantedFields = new String[]{"*"};
deniedFields = randomBoolean() ? null : new String[]{};
expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
expected.add("_all");
expected = new HashSet<>(META_FIELDS);
expected.add("field1");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected);
// make sure all field can be explicitly allowed
grantedFields = new String[]{"_all"};
deniedFields = randomBoolean() ? null : new String[]{};
expected = new HashSet<>(META_FIELDS_WITHOUT_ALL);
expected.add("_all");
assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "_source");
}
private SparseFixedBitSet query(LeafReaderContext leaf, String field, String value) throws IOException {

15
plugin/src/test/java/org/elasticsearch/xpack/security/authz/permission/FieldPermissionsCacheTests.java
View File

@ -88,21 +88,6 @@ public class FieldPermissionsCacheTests extends ESTestCase {
assertTrue(mergedFieldPermissions.grantsAccessTo("a"));
assertTrue(mergedFieldPermissions.grantsAccessTo("b"));
// test merge does not remove _all
allowed1 = new String[]{"_all"};
allowed2 = new String[]{};
denied1 = null;
denied2 = null;
fieldPermissions1 = randomBoolean() ? fieldPermissionsCache.getFieldPermissions(allowed1, denied1) :
new FieldPermissions(fieldPermissionDef(allowed1, denied1));
fieldPermissions2 = randomBoolean() ? fieldPermissionsCache.getFieldPermissions(allowed2, denied2) :
new FieldPermissions(fieldPermissionDef(allowed2, denied2));
mergedFieldPermissions =
fieldPermissionsCache.getFieldPermissions(Arrays.asList(fieldPermissions1, fieldPermissions2));
assertTrue(fieldPermissions1.grantsAccessTo("_all"));
assertFalse(fieldPermissions2.grantsAccessTo("_all"));
assertTrue(mergedFieldPermissions.grantsAccessTo("_all"));
allowed1 = new String[] { "a*" };
allowed2 = new String[] { "b*" };
denied1 = new String[] { "aa*" };

21
plugin/src/test/java/org/elasticsearch/xpack/security/authz/permission/FieldPermissionsTests.java
View File

@ -224,25 +224,4 @@ public class FieldPermissionsTests extends ESTestCase {
assertEquals((Integer) hashCode, hashCodes.get(i));
}
}
public void testAllFieldIsAutomaticallyExcludedIfNotExplicitlyGranted() throws Exception {
final FieldPermissions fieldPermissions = new FieldPermissions(
new FieldPermissionsDefinition(new String[] { "_a*" }, new String[0]));
assertTrue(fieldPermissions.grantsAccessTo("_animal"));
assertFalse(fieldPermissions.grantsAccessTo("_all"));
}
public void testAllFieldIsNotExcludedIfExplicitlyGranted() throws Exception {
final String[] grant = { "foo", "bar", "baz", "_all" };
Collections.shuffle(Arrays.asList(grant), random());
final FieldPermissions fieldPermissions = new FieldPermissions(
new FieldPermissionsDefinition(grant, new String[0]));
assertTrue(fieldPermissions.grantsAccessTo("_all"));
assertTrue(fieldPermissions.grantsAccessTo("foo"));
assertTrue(fieldPermissions.grantsAccessTo("bar"));
assertTrue(fieldPermissions.grantsAccessTo("baz"));
assertFalse(fieldPermissions.grantsAccessTo(randomAlphaOfLengthBetween(5, 8)));
}
}