honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SQL: change the way unsupported data types fields are handled (#50823) (#51220) 

The hierarchy of fields/sub-fields under a field that is of an
unsupported data type will be marked as unsupported as well. Until this
change, the behavior was to set the unsupported data type field's
hierarchy as empty.

Example, considering the following hierarchy of fields/sub-fields
a -> b -> c -> d, if b would be of type "foo", then b, c and d will
be marked as unsupported.

(cherry picked from commit 7adb286c4c485b9e781f88b0a2f98cab9ec5b7e2)
This commit is contained in:
Andrei Stefan 2020-01-20 16:23:43 +02:00 committed by GitHub
parent 51134d9738
commit df36169220
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 209 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java
View File

@ -217,8 +217,14 @@ public class Analyzer extends RuleExecutor<LogicalPlan> {
// unsupported types
else if (DataTypes.isUnsupported(fa.dataType())) {
UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
named = u.withUnresolvedMessage(
"Cannot use field [" + fa.name() + "] type [" + unsupportedField.getOriginalType() + "] as is unsupported");
if (unsupportedField.hasInherited()) {
named = u.withUnresolvedMessage(
"Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "] "
+ "in hierarchy (field [" + unsupportedField.getInherited() + "])");
} else {
named = u.withUnresolvedMessage(
"Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]");
}
}
// compound fields
else if (allowCompound == false && fa.dataType().isPrimitive() == false) {

22
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java
View File

@ -356,11 +356,12 @@ public class IndexResolver {
int dot = fieldName.lastIndexOf('.');
String fullFieldName = fieldName;
EsField parent = null;
if (dot >= 0) {
String parentName = fieldName.substring(0, dot);
fieldName = fieldName.substring(dot + 1);
EsField parent = flattedMapping.get(parentName);
parent = flattedMapping.get(parentName);
if (parent == null) {
Map<String, FieldCapabilities> map = globalCaps.get(parentName);
Function<String, EsField> fieldFunction;
@ -385,7 +386,22 @@ public class IndexResolver {
}
EsField esField = field.apply(fieldName);
if (parent != null && parent instanceof UnsupportedEsField) {
UnsupportedEsField unsupportedParent = (UnsupportedEsField) parent;
String inherited = unsupportedParent.getInherited();
String type = unsupportedParent.getOriginalType();
if (inherited == null) {
// mark the sub-field as unsupported, just like its parent, setting the first unsupported parent as the current one
esField = new UnsupportedEsField(esField.getName(), type, unsupportedParent.getName(), esField.getProperties());
} else {
// mark the sub-field as unsupported, just like its parent, but setting the first unsupported parent
// as the parent's first unsupported grandparent
esField = new UnsupportedEsField(esField.getName(), type, inherited, esField.getProperties());
}
}
parentProps.put(fieldName, esField);
flattedMapping.put(fullFieldName, esField);
@ -406,7 +422,7 @@ public class IndexResolver {
case DATETIME:
return new DateEsField(fieldName, props, isAggregateable);
case UNSUPPORTED:
return new UnsupportedEsField(fieldName, typeName);
return new UnsupportedEsField(fieldName, typeName, null, props);
default:
return new EsField(fieldName, esType, props, isAggregateable, isAlias);
}

22
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/Types.java
View File

@ -73,7 +73,7 @@ public abstract class Types {
properties = Collections.emptyMap();
}
} else {
properties = Collections.emptyMap();
properties = fromEs(content);
}
boolean docValues = boolSetting(content.get("doc_values"), esDataType.defaultDocValues);
final EsField field;
@ -91,7 +91,8 @@ public abstract class Types {
break;
case UNSUPPORTED:
String type = content.get("type").toString();
field = new UnsupportedEsField(name, type);
field = new UnsupportedEsField(name, type, null, properties);
propagateUnsupportedType(name, type, properties);
break;
default:
field = new EsField(name, esDataType, properties, docValues);
@ -113,4 +114,21 @@ public abstract class Types {
private static int intSetting(Object value, int defaultValue) {
return value == null ? defaultValue : Integer.parseInt(value.toString());
}
private static void propagateUnsupportedType(String inherited, String originalType, Map<String, EsField> properties) {
if (properties != null && properties.isEmpty() == false) {
for (Entry<String, EsField> entry : properties.entrySet()) {
EsField field = entry.getValue();
UnsupportedEsField u;
if (field instanceof UnsupportedEsField) {
u = (UnsupportedEsField) field;
u = new UnsupportedEsField(u.getName(), originalType, inherited, u.getProperties());
} else {
u = new UnsupportedEsField(field.getName(), originalType, inherited, field.getProperties());
}
entry.setValue(u);
propagateUnsupportedType(inherited, originalType, u.getProperties());
}
}
}
}

31
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/UnsupportedEsField.java
View File

@ -5,24 +5,40 @@
*/
package org.elasticsearch.xpack.sql.type;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
/**
* SQL-related information about an index field that cannot be supported by SQL
* SQL-related information about an index field that cannot be supported by SQL.
* All the subfields (properties) of an unsupported type should also be unsupported.
*/
public class UnsupportedEsField extends EsField {
private String originalType;
private final String originalType;
private final String inherited; // for fields belonging to parents (or grandparents) that have an unsupported type
public UnsupportedEsField(String name, String originalType) {
super(name, DataType.UNSUPPORTED, Collections.emptyMap(), false);
this(name, originalType, null, new TreeMap<>());
}
public UnsupportedEsField(String name, String originalType, String inherited, Map<String, EsField> properties) {
super(name, DataType.UNSUPPORTED, properties, false);
this.originalType = originalType;
this.inherited = inherited;
}
public String getOriginalType() {
return originalType;
}
public String getInherited() {
return inherited;
}
public boolean hasInherited() {
return inherited != null;
}
@Override
public boolean equals(Object o) {
@ -36,11 +52,12 @@ public class UnsupportedEsField extends EsField {
return false;
}
UnsupportedEsField that = (UnsupportedEsField) o;
return Objects.equals(originalType, that.originalType);
return Objects.equals(originalType, that.originalType)
&& Objects.equals(inherited, that.inherited);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), originalType);
return Objects.hash(super.hashCode(), originalType, inherited);
}
}

24
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
View File

@ -482,19 +482,33 @@ public class VerifierErrorMessagesTests extends ESTestCase {
}
public void testUnsupportedType() {
assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported",
assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]",
error("SELECT unsupported FROM test"));
}
public void testUnsupportedStarExpansion() {
assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported",
assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]",
error("SELECT unsupported.* FROM test"));
}
public void testUnsupportedTypeInFilter() {
assertEquals("1:26: Cannot use field [unsupported] type [ip_range] as is unsupported",
assertEquals("1:26: Cannot use field [unsupported] with unsupported type [ip_range]",
error("SELECT * FROM test WHERE unsupported > 1"));
}
public void testValidRootFieldWithUnsupportedChildren() {
accept("SELECT x FROM test");
}
public void testUnsupportedTypeInHierarchy() {
assertEquals("1:8: Cannot use field [x.y.z.w] with unsupported type [foobar] in hierarchy (field [y])",
error("SELECT x.y.z.w FROM test"));
assertEquals("1:8: Cannot use field [x.y.z.v] with unsupported type [foobar] in hierarchy (field [y])",
error("SELECT x.y.z.v FROM test"));
assertEquals("1:8: Cannot use field [x.y.z] with unsupported type [foobar] in hierarchy (field [y])",
error("SELECT x.y.z.* FROM test"));
assertEquals("1:8: Cannot use field [x.y] with unsupported type [foobar]", error("SELECT x.y FROM test"));
}
public void testTermEqualitOnInexact() {
assertEquals("1:26: [text = 'value'] cannot operate on first argument field of data type [text]: " +
@ -509,12 +523,12 @@ public class VerifierErrorMessagesTests extends ESTestCase {
}
public void testUnsupportedTypeInFunction() {
assertEquals("1:12: Cannot use field [unsupported] type [ip_range] as is unsupported",
assertEquals("1:12: Cannot use field [unsupported] with unsupported type [ip_range]",
error("SELECT ABS(unsupported) FROM test"));
}
public void testUnsupportedTypeInOrder() {
assertEquals("1:29: Cannot use field [unsupported] type [ip_range] as is unsupported",
assertEquals("1:29: Cannot use field [unsupported] with unsupported type [ip_range]",
error("SELECT * FROM test ORDER BY unsupported"));
}

90
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
View File

@ -125,6 +125,94 @@ public class IndexResolverTests extends ESTestCase {
assertEquals(DataType.INTEGER, esIndex.mapping().get("_meta_field").getDataType());
assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
}
public void testFlattenedHiddenSubfield() throws Exception {
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
addFieldCaps(fieldCaps, "some_field", "flattened", false, false);
addFieldCaps(fieldCaps, "some_field._keyed", "flattened", false, false);
addFieldCaps(fieldCaps, "another_field", "object", true, false);
addFieldCaps(fieldCaps, "another_field._keyed", "keyword", true, false);
addFieldCaps(fieldCaps, "nested_field", "object", false, false);
addFieldCaps(fieldCaps, "nested_field.sub_field", "flattened", true, true);
addFieldCaps(fieldCaps, "nested_field.sub_field._keyed", "flattened", true, true);
addFieldCaps(fieldCaps, "text", "keyword", true, true);
String wildcard = "*";
IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
assertTrue(resolution.isValid());
EsIndex esIndex = resolution.get();
assertEquals(wildcard, esIndex.name());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getProperties().get("_keyed").getDataType());
assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field").getDataType());
assertEquals(DataType.UNSUPPORTED,
esIndex.mapping().get("nested_field").getProperties().get("sub_field").getProperties().get("_keyed").getDataType());
assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType());
assertEquals(DataType.KEYWORD, esIndex.mapping().get("another_field").getProperties().get("_keyed").getDataType());
}
public void testPropagateUnsupportedTypeToSubFields() throws Exception {
// generate a field type having the name of the format "foobar43"
String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100);
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
addFieldCaps(fieldCaps, "a", "text", false, false);
addFieldCaps(fieldCaps, "a.b", esFieldType, false, false);
addFieldCaps(fieldCaps, "a.b.c", "object", true, false);
addFieldCaps(fieldCaps, "a.b.c.d", "keyword", true, false);
addFieldCaps(fieldCaps, "a.b.c.e", "foo", true, true);
String wildcard = "*";
IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
assertTrue(resolution.isValid());
EsIndex esIndex = resolution.get();
assertEquals(wildcard, esIndex.name());
assertEquals(DataType.TEXT, esIndex.mapping().get("a").getDataType());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getDataType());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c").getDataType());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c")
.getProperties().get("d").getDataType());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c")
.getProperties().get("e").getDataType());
}
public void testRandomMappingFieldTypeMappedAsUnsupported() throws Exception {
// generate a field type having the name of the format "foobar43"
String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100);
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
addFieldCaps(fieldCaps, "some_field", esFieldType, false, false);
addFieldCaps(fieldCaps, "another_field", "object", true, false);
addFieldCaps(fieldCaps, "another_field._foo", esFieldType, true, false);
addFieldCaps(fieldCaps, "nested_field", "object", false, false);
addFieldCaps(fieldCaps, "nested_field.sub_field1", esFieldType, true, true);
addFieldCaps(fieldCaps, "nested_field.sub_field1.bar", esFieldType, true, true);
addFieldCaps(fieldCaps, "nested_field.sub_field2", esFieldType, true, true);
// even if this is of a supported type, because it belongs to an UNSUPPORTED type parent, it should also be UNSUPPORTED
addFieldCaps(fieldCaps, "nested_field.sub_field2.bar", "keyword", true, true);
addFieldCaps(fieldCaps, "text", "keyword", true, true);
String wildcard = "*";
IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
assertTrue(resolution.isValid());
EsIndex esIndex = resolution.get();
assertEquals(wildcard, esIndex.name());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType());
assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getDataType());
assertEquals(DataType.UNSUPPORTED,
esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getProperties().get("bar").getDataType());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getDataType());
assertEquals(DataType.UNSUPPORTED,
esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getProperties().get("bar").getDataType());
assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType());
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("another_field").getProperties().get("_foo").getDataType());
}
public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception {
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
@ -327,7 +415,7 @@ public class IndexResolverTests extends ESTestCase {
private void addFieldCaps(Map<String, Map<String, FieldCapabilities>> fieldCaps, String name, String type, boolean isSearchable,
boolean isAggregatable) {
Map<String, FieldCapabilities> cap = new HashMap<>();
cap.put(name, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap()));
cap.put(type, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap()));
fieldCaps.put(name, cap);
}
}

6
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java
View File

@ -463,7 +463,7 @@ public class SysColumnsTests extends ESTestCase {
public void testSysColumnsWithCatalogWildcard() throws Exception {
executeCommand("SYS COLUMNS CATALOG 'cluster' TABLE LIKE 'test' LIKE '%'", emptyList(), r -> {
assertEquals(14, r.size());
assertEquals(15, r.size());
assertEquals(CLUSTER_NAME, r.column(0));
assertEquals("test", r.column(2));
assertEquals("bool", r.column(3));
@ -476,7 +476,7 @@ public class SysColumnsTests extends ESTestCase {
public void testSysColumnsWithMissingCatalog() throws Exception {
executeCommand("SYS COLUMNS TABLE LIKE 'test' LIKE '%'", emptyList(), r -> {
assertEquals(14, r.size());
assertEquals(15, r.size());
assertEquals(CLUSTER_NAME, r.column(0));
assertEquals("test", r.column(2));
assertEquals("bool", r.column(3));
@ -489,7 +489,7 @@ public class SysColumnsTests extends ESTestCase {
public void testSysColumnsWithNullCatalog() throws Exception {
executeCommand("SYS COLUMNS CATALOG ? TABLE LIKE 'test' LIKE '%'", singletonList(new SqlTypedParamValue("keyword", null)), r -> {
assertEquals(14, r.size());
assertEquals(15, r.size());
assertEquals(CLUSTER_NAME, r.column(0));
assertEquals("test", r.column(2));
assertEquals("bool", r.column(3));

4
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java
View File

@ -188,6 +188,10 @@ public class TypesTests extends ESTestCase {
Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
EsField dt = mapping.get("range");
assertThat(dt.getDataType().typeName, is("unsupported"));
dt = mapping.get("time_frame");
assertThat(dt.getDataType().typeName, is("unsupported"));
dt = mapping.get("flat");
assertThat(dt.getDataType().typeName, is("unsupported"));
}
public static Map<String, EsField> loadMapping(String name) {

20
x-pack/plugin/sql/src/test/resources/mapping-multi-field-with-nested.json
View File

@ -7,6 +7,26 @@
"unsupported" : { "type" : "ip_range" },
"date" : { "type" : "date"},
"shape": { "type" : "geo_shape" },
"x" : {
"type" : "text",
"fields" : {
"y" : {
"type" : "foobar",
"fields" : {
"z" : {
"properties" : {
"v" : {
"type" : "keyword"
},
"w" : {
"type" : "foo"
}
}
}
}
}
}
},
"some" : {
"properties" : {
"dotted" : {

3
x-pack/plugin/sql/src/test/resources/mapping-unsupported.json
View File

@ -6,6 +6,9 @@
"time_frame" : {
"type" : "date_range",
"format" : "yyyy-MM-dd"
},
"flat" : {
"type" : "flattened"
}
}
}