honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Don't omit empty arrays when filtering _source (#56527) 

When using source filtering exclusions, empty arrays are not preserved in documents, and no empty arrays are returned if arrays are empty after applying exclusions. We have special treatment to make sure that we preserve empty objects, but the behaviour for arrays is different.

It looks like this regression was introduced by #22593, shortly after we refactored source filtering to use automata (#20736).

Note that this change affects what the search API returns when using source exclusions, as well as what gets indexed when using source exclusions for the _source field.

Closes #23796
This commit is contained in:
Luca Cavanna 2020-05-13 23:24:21 +02:00 committed by GitHub
parent 126619ae3c
commit 34410814b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 138 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
View File

@ -267,7 +267,7 @@ public class XContentMapValues {
List<Object> filteredValue = filter((Iterable<?>) value, List<Object> filteredValue = filter((Iterable<?>) value,
subIncludeAutomaton, subIncludeState, excludeAutomaton, excludeState, matchAllAutomaton); subIncludeAutomaton, subIncludeState, excludeAutomaton, excludeState, matchAllAutomaton);
if (filteredValue.isEmpty() == false) { if (includeAutomaton.isAccept(includeState) || filteredValue.isEmpty() == false) {
filtered.put(key, filteredValue); filtered.put(key, filteredValue);
} }

122
server/src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java
View File

@ -44,7 +44,7 @@ public abstract class AbstractFilteringTestCase extends ESTestCase {
protected abstract void testFilter(Builder expected, Builder actual, Set<String> includes, Set<String> excludes) throws IOException; protected abstract void testFilter(Builder expected, Builder actual, Set<String> includes, Set<String> excludes) throws IOException;
/** Sample test case **/ /** Sample test case **/
private static final Builder SAMPLE = builder -> builder.startObject() protected static final Builder SAMPLE = builder -> builder.startObject()
.field("title", "My awesome book") .field("title", "My awesome book")
.field("pages", 456) .field("pages", 456)
.field("price", 27.99) .field("price", 27.99)
@ -484,70 +484,70 @@ public abstract class AbstractFilteringTestCase extends ESTestCase {
testFilter(expected, SAMPLE, singleton("authors.*name"), emptySet()); testFilter(expected, SAMPLE, singleton("authors.*name"), emptySet());
} }
public void testSimpleArrayOfObjectsExclusive() throws Exception { protected static final Builder SIMPLE_ARRAY_OF_OBJECTS_EXCLUSIVE = builder -> builder.startObject()
final Builder expected = builder -> builder.startObject() .field("title", "My awesome book")
.field("title", "My awesome book") .field("pages", 456)
.field("pages", 456) .field("price", 27.99)
.field("price", 27.99) .field("timestamp", 1428582942867L)
.field("timestamp", 1428582942867L) .nullField("default")
.nullField("default") .startArray("tags")
.startArray("tags") .value("elasticsearch")
.value("elasticsearch") .value("java")
.value("java") .endArray()
.endArray() .startObject("properties")
.startObject("properties") .field("weight", 0.8d)
.field("weight", 0.8d) .startObject("language")
.startObject("language") .startObject("en")
.startObject("en") .field("lang", "English")
.field("lang", "English") .field("available", true)
.field("available", true) .startArray("distributors")
.startArray("distributors") .startObject()
.startObject() .field("name", "The Book Shop")
.field("name", "The Book Shop") .startArray("addresses")
.startArray("addresses") .startObject()
.startObject() .field("name", "address #1")
.field("name", "address #1") .field("street", "Hampton St")
.field("street", "Hampton St") .field("city", "London")
.field("city", "London")
.endObject()
.startObject()
.field("name", "address #2")
.field("street", "Queen St")
.field("city", "Stornoway")
.endObject()
.endArray()
.endObject()
.startObject()
.field("name", "Sussex Books House")
.endObject()
.endArray()
.endObject() .endObject()
.startObject("fr") .startObject()
.field("lang", "French") .field("name", "address #2")
.field("available", false) .field("street", "Queen St")
.startArray("distributors") .field("city", "Stornoway")
.startObject()
.field("name", "La Maison du Livre")
.startArray("addresses")
.startObject()
.field("name", "address #1")
.field("street", "Rue Mouffetard")
.field("city", "Paris")
.endObject()
.endArray()
.endObject()
.startObject()
.field("name", "Thetra")
.endObject()
.endArray()
.endObject() .endObject()
.endObject() .endArray()
.endObject() .endObject()
.endObject(); .startObject()
.field("name", "Sussex Books House")
.endObject()
.endArray()
.endObject()
.startObject("fr")
.field("lang", "French")
.field("available", false)
.startArray("distributors")
.startObject()
.field("name", "La Maison du Livre")
.startArray("addresses")
.startObject()
.field("name", "address #1")
.field("street", "Rue Mouffetard")
.field("city", "Paris")
.endObject()
.endArray()
.endObject()
.startObject()
.field("name", "Thetra")
.endObject()
.endArray()
.endObject()
.endObject()
.endObject()
.endObject();
testFilter(expected, SAMPLE, emptySet(), singleton("authors")); public void testSimpleArrayOfObjectsExclusive() throws Exception {
testFilter(expected, SAMPLE, emptySet(), singleton("authors.*")); testFilter(SIMPLE_ARRAY_OF_OBJECTS_EXCLUSIVE, SAMPLE, emptySet(), singleton("authors"));
testFilter(expected, SAMPLE, emptySet(), singleton("authors.*name")); testFilter(SIMPLE_ARRAY_OF_OBJECTS_EXCLUSIVE, SAMPLE, emptySet(), singleton("authors.*"));
testFilter(SIMPLE_ARRAY_OF_OBJECTS_EXCLUSIVE, SAMPLE, emptySet(), singleton("authors.*name"));
} }
public void testSimpleArrayOfObjectsPropertyInclusive() throws Exception { public void testSimpleArrayOfObjectsPropertyInclusive() throws Exception {

76
server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.json.JsonXContent;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -37,6 +38,8 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import static java.util.Collections.emptySet;
import static java.util.Collections.singleton;
import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap; import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap;
import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasEntry;
@ -479,6 +482,79 @@ public class XContentMapValuesTests extends AbstractFilteringTestCase {
assertEquals(Collections.singletonMap("foobaz", 3), XContentMapValues.filter(map, new String[0], new String[] {"foobar"})); assertEquals(Collections.singletonMap("foobaz", 3), XContentMapValues.filter(map, new String[0], new String[] {"foobar"}));
} }
@Override
public void testSimpleArrayOfObjectsExclusive() throws Exception {
//Empty arrays are preserved by XContentMapValues, they get removed only if explicitly excluded.
//See following tests around this specific behaviour
testFilter(SIMPLE_ARRAY_OF_OBJECTS_EXCLUSIVE, SAMPLE, emptySet(), singleton("authors"));
}
public void testArraySubFieldExclusion() {
Map<String, Object> map = new HashMap<>();
map.put("field", "value");
List<Map<String, String>> array = new ArrayList<>();
Map<String, String> object = new HashMap<>();
object.put("exclude", "bar");
array.add(object);
map.put("array", array);
Map<String, Object> filtered = XContentMapValues.filter(map, new String[0], new String[]{"array.exclude"});
assertTrue(filtered.containsKey("field"));
assertTrue(filtered.containsKey("array"));
List<?> filteredArray = (List<?>)filtered.get("array");
assertThat(filteredArray, hasSize(0));
}
public void testEmptyArraySubFieldsExclusion() {
Map<String, Object> map = new HashMap<>();
map.put("field", "value");
List<Map<String, String>> array = new ArrayList<>();
map.put("array", array);
Map<String, Object> filtered = XContentMapValues.filter(map, new String[0], new String[]{"array.exclude"});
assertTrue(filtered.containsKey("field"));
assertTrue(filtered.containsKey("array"));
List<?> filteredArray = (List<?>)filtered.get("array");
assertEquals(0, filteredArray.size());
}
public void testEmptyArraySubFieldsInclusion() {
Map<String, Object> map = new HashMap<>();
map.put("field", "value");
List<Map<String, String>> array = new ArrayList<>();
map.put("array", array);
{
Map<String, Object> filtered = XContentMapValues.filter(map, new String[]{"array.include"}, new String[0]);
assertFalse(filtered.containsKey("field"));
assertFalse(filtered.containsKey("array"));
}
{
Map<String, Object> filtered = XContentMapValues.filter(map, new String[]{"array", "array.include"},
new String[0]);
assertFalse(filtered.containsKey("field"));
assertTrue(filtered.containsKey("array"));
List<?> filteredArray = (List<?>)filtered.get("array");
assertEquals(0, filteredArray.size());
}
}
public void testEmptyObjectsSubFieldsInclusion() {
Map<String, Object> map = new HashMap<>();
map.put("field", "value");
map.put("object", new HashMap<>());
{
Map<String, Object> filtered = XContentMapValues.filter(map, new String[]{"object.include"}, new String[0]);
assertFalse(filtered.containsKey("field"));
assertFalse(filtered.containsKey("object"));
}
{
Map<String, Object> filtered = XContentMapValues.filter(map, new String[]{"object", "object.include"},
new String[0]);
assertFalse(filtered.containsKey("field"));
assertTrue(filtered.containsKey("object"));
Map<?, ?> filteredMap = (Map<?, ?>)filtered.get("object");
assertEquals(0, filteredMap.size());
}
}
public void testPrefix() { public void testPrefix() {
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
map.put("photos", Arrays.asList(new String[] {"foo", "bar"})); map.put("photos", Arrays.asList(new String[] {"foo", "bar"}));