SQL: do not ignore all fields whose names start with underscore (#36214)

* Do not ignore fields whose names start with underscore, unless they are
meta fields.
* Filter out _size field.

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java

@@ -136,7 +136,7 @@ public class IndexResolver {
 
     private static final IndicesOptions INDICES_ONLY_OPTIONS = new IndicesOptions(
             EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE, Option.IGNORE_ALIASES), EnumSet.of(WildcardStates.OPEN));
-
+    private static final List<String> FIELD_NAMES_BLACKLIST = Arrays.asList("_size");
 
     private final Client client;
     private final String clusterName;
@@ -272,8 +272,8 @@ public class IndexResolver {
 
             String name = entry.getKey();
 
-            // skip internal fields
-            if (!name.startsWith("_")) {
+            // Skip any of the blacklisted field names.
+            if (!FIELD_NAMES_BLACKLIST.contains(name)) {
                 Map<String, FieldCapabilities> types = entry.getValue();
                 // field is mapped differently across indices
                 if (types.size() > 1) {
@@ -297,6 +297,12 @@ public class IndexResolver {
                 // type is okay, check aggregation
                 else {
                     fieldCap = types.values().iterator().next();
+                    
+                    // Skip internal fields (name starting with underscore and its type reported by field_caps starts with underscore
+                    // as well). A meta field named "_version", for example, has the type named "_version".
+                    if (name.startsWith("_") && fieldCap.getType().startsWith("_")) {
+                        continue;
+                    }
                     // validate search/agg-able
                     if (fieldCap.isAggregatable() && fieldCap.nonAggregatableIndices() != null) {
                         errorMessage.append("mapped as aggregatable except in ");

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java

@@ -112,6 +112,25 @@ public class IndexResolverTests extends ESTestCase {
         assertTrue(resolution.isValid());
         assertEqualsMaps(nestedMapping, resolution.get().mapping());
     }
+    
+    public void testMetaFieldsAreIgnored() throws Exception {
+        Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
+        addFieldCaps(fieldCaps, "_version", "_version", false, false);
+        addFieldCaps(fieldCaps, "_meta_field", "integer", true, true);
+        addFieldCaps(fieldCaps, "_size", "integer", true, true);
+        addFieldCaps(fieldCaps, "text", "keyword", true, true);
+        
+        String wildcard = "*";
+        IndexResolution resolution = IndexResolver.mergedMapping(wildcard, fieldCaps);
+        assertTrue(resolution.isValid());
+
+        EsIndex esIndex = resolution.get();
+        assertEquals(wildcard, esIndex.name());
+        assertNull(esIndex.mapping().get("_version"));
+        assertNull(esIndex.mapping().get("_size"));
+        assertEquals(DataType.INTEGER, esIndex.mapping().get("_meta_field").getDataType());
+        assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
+    }
 
     public static IndexResolution merge(EsIndex... indices) {
         return IndexResolver.mergedMapping("*", fromMappings(indices));
@@ -209,4 +228,11 @@ public class IndexResolverTests extends ESTestCase {
             assertEquals(String.format(Locale.ROOT, "Key [%s] has different values", entry.getKey()), entry.getValue(), rv);
         }
     }
+    
+    private void addFieldCaps(Map<String, Map<String, FieldCapabilities>> fieldCaps, String name, String type, boolean isSearchable,
+            boolean isAggregatable) {
+        Map<String, FieldCapabilities> cap = new HashMap<>();
+        cap.put(name, new FieldCapabilities(name, type, isSearchable, isAggregatable));
+        fieldCaps.put(name, cap);
+    }
 }

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java

@@ -38,12 +38,13 @@ public class TypesTests extends ESTestCase {
 
     public void testBasicMapping() {
         Map<String, EsField> mapping = loadMapping("mapping-basic.json");
-        assertThat(mapping.size(), is(6));
+        assertThat(mapping.size(), is(7));
         assertThat(mapping.get("emp_no").getDataType(), is(INTEGER));
         assertThat(mapping.get("first_name"), instanceOf(TextEsField.class));
         assertThat(mapping.get("last_name").getDataType(), is(TEXT));
         assertThat(mapping.get("gender").getDataType(), is(KEYWORD));
         assertThat(mapping.get("salary").getDataType(), is(INTEGER));
+        assertThat(mapping.get("_meta_field").getDataType(), is(KEYWORD));
     }
 
     public void testDefaultStringMapping() {

x-pack/plugin/sql/src/test/resources/mapping-basic.json

@@ -17,6 +17,9 @@
         },
         "salary" : {
             "type" : "integer"
+        },
+        "_meta_field": {
+            "type" : "keyword"
         }
     }
 }

x-pack/plugin/sql/src/test/resources/mapping-numeric.json

@@ -11,6 +11,14 @@
         },
         "long" : {
             "type" : "long"
+        },
+        "meta_subfield" : {
+            "type" : "text",
+            "fields" : {
+                "_meta" : {
+                     "type" : "keyword"
+                }
+            }
         }
     }
 }