Merge pull request #18212 from jimferenczi/field_stats_null

Do not return fieldstats information for fields that exist in the mapping but not in the index.

core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java

@@ -117,11 +117,13 @@ public class TransportFieldStatsTransportAction extends
                     if (existing != null) {
                         if (existing.getType() != entry.getValue().getType()) {
                             if (conflicts.containsKey(entry.getKey()) == false) {
+                                FieldStats[] fields = new FieldStats[] {entry.getValue(), existing};
+                                Arrays.sort(fields, (o1, o2) -> Byte.compare(o1.getType(), o2.getType()));
                                 conflicts.put(entry.getKey(),
                                     "Field [" + entry.getKey() + "] of type [" +
-                                        FieldStats.typeName(entry.getValue().getType()) +
+                                        FieldStats.typeName(fields[0].getType()) +
                                         "] conflicts with existing field of type [" +
-                                        FieldStats.typeName(existing.getType()) +
+                                        FieldStats.typeName(fields[1].getType()) +
                                         "] in other index.");
                             }
                         } else {

core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java

@@ -404,7 +404,7 @@ public abstract class MappedFieldType extends FieldType {
         int maxDoc = reader.maxDoc();
         Terms terms = MultiFields.getTerms(reader, name());
         if (terms == null) {
-            return new FieldStats.Text(maxDoc, isSearchable(), isAggregatable());
+            return null;
         }
         FieldStats stats = new FieldStats.Text(maxDoc, terms.getDocCount(),
             terms.getSumDocFreq(), terms.getSumTotalTermFreq(),

core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java

@@ -399,7 +399,7 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
             String field = name();
             long size = XPointValues.size(reader, field);
             if (size == 0) {
-                return new FieldStats.Date(reader.maxDoc(), isSearchable(), isAggregatable(), dateTimeFormatter());
+                return null;
             }
             int docCount = XPointValues.getDocCount(reader, field);
             byte[] min = XPointValues.getMinPackedValue(reader, field);

core/src/main/java/org/elasticsearch/index/mapper/core/LegacyByteFieldMapper.java

@@ -174,7 +174,7 @@ public class LegacyByteFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Long(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinInt(terms);
             long maxValue = LegacyNumericUtils.getMaxInt(terms);

core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java

@@ -379,7 +379,7 @@ public class LegacyDateFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Date(maxDoc, isSearchable(), isAggregatable(), dateTimeFormatter());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinLong(terms);
             long maxValue = LegacyNumericUtils.getMaxLong(terms);

core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDoubleFieldMapper.java

@@ -185,7 +185,7 @@ public class LegacyDoubleFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Double(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             double minValue = NumericUtils.sortableLongToDouble(LegacyNumericUtils.getMinLong(terms));
             double maxValue = NumericUtils.sortableLongToDouble(LegacyNumericUtils.getMaxLong(terms));

core/src/main/java/org/elasticsearch/index/mapper/core/LegacyFloatFieldMapper.java

@@ -170,7 +170,7 @@ public class LegacyFloatFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Double(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             float minValue = NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMinInt(terms));
             float maxValue = NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMaxInt(terms));

core/src/main/java/org/elasticsearch/index/mapper/core/LegacyIntegerFieldMapper.java

@@ -174,7 +174,7 @@ public class LegacyIntegerFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Long(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinInt(terms);
             long maxValue = LegacyNumericUtils.getMaxInt(terms);

core/src/main/java/org/elasticsearch/index/mapper/core/LegacyLongFieldMapper.java

@@ -173,8 +173,7 @@ public class LegacyLongFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Long(
+                return null;
-                    maxDoc, isSearchable(), isAggregatable());
             }
             long minValue = LegacyNumericUtils.getMinLong(terms);
             long maxValue = LegacyNumericUtils.getMaxLong(terms);

core/src/main/java/org/elasticsearch/index/mapper/core/LegacyShortFieldMapper.java

@@ -178,7 +178,7 @@ public class LegacyShortFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Long(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinInt(terms);
             long maxValue = LegacyNumericUtils.getMaxInt(terms);

core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java

@@ -262,7 +262,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                                     boolean isSearchable, boolean isAggregatable) throws IOException {
                 long size = XPointValues.size(reader, fieldName);
                 if (size == 0) {
-                    return new FieldStats.Double(reader.maxDoc(), isSearchable, isAggregatable);
+                    return null;
                 }
                 int docCount = XPointValues.getDocCount(reader, fieldName);
                 byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
@@ -353,7 +353,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                                     boolean isSearchable, boolean isAggregatable) throws IOException {
                 long size = XPointValues.size(reader, fieldName);
                 if (size == 0) {
-                    return new FieldStats.Double(reader.maxDoc(), isSearchable, isAggregatable);
+                    return null;
                 }
                 int docCount = XPointValues.getDocCount(reader, fieldName);
                 byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
@@ -588,7 +588,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                                   boolean isSearchable, boolean isAggregatable) throws IOException {
                 long size = XPointValues.size(reader, fieldName);
                 if (size == 0) {
-                    return new FieldStats.Long(reader.maxDoc(), isSearchable, isAggregatable);
+                    return null;
                 }
                 int docCount = XPointValues.getDocCount(reader, fieldName);
                 byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
@@ -691,7 +691,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                                   boolean isSearchable, boolean isAggregatable) throws IOException {
                 long size = XPointValues.size(reader, fieldName);
                 if (size == 0) {
-                    return new FieldStats.Long(reader.maxDoc(), isSearchable, isAggregatable);
+                    return null;
                 }
                 int docCount = XPointValues.getDocCount(reader, fieldName);
                 byte[] min = XPointValues.getMinPackedValue(reader, fieldName);

core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java

@@ -232,7 +232,7 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
             String field = name();
             long size = XPointValues.size(reader, field);
             if (size == 0) {
-                return new FieldStats.Ip(reader.maxDoc(), isSearchable(), isAggregatable());
+                return null;
             }
             int docCount = XPointValues.getDocCount(reader, field);
             byte[] min = XPointValues.getMinPackedValue(reader, field);

core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java

@@ -252,7 +252,7 @@ public class LegacyIpFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Ip(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinLong(terms);
             long maxValue = LegacyNumericUtils.getMaxLong(terms);

core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java

@@ -246,7 +246,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getIndicesMergedFieldStats().get("_all").size(), equalTo(0));
         assertThat(response.getConflicts().size(), equalTo(1));
         assertThat(response.getConflicts().get("value"),
-            equalTo("Field [value] of type [text] conflicts with existing field of type [whole-number] " +
+            equalTo("Field [value] of type [whole-number] conflicts with existing field of type [text] " +
                 "in other index."));
 
         response = client().prepareFieldStats().setFields("value").setLevel("indices").get();
@@ -260,7 +260,6 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
             equalTo(new BytesRef("b")));
     }
 
-    @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-7257")
     public void testIncompatibleFieldTypesMultipleFields() {
         assertAcked(prepareCreate("test1").addMapping(
                 "test", "value", "type=long", "value2", "type=long"
@@ -284,7 +283,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getIndicesMergedFieldStats().get("_all").get("value2").getMaxValue(), equalTo(1L));
         assertThat(response.getConflicts().size(), equalTo(1));
         assertThat(response.getConflicts().get("value"),
-            equalTo("Field [value] of type [text] conflicts with existing field of type [whole-number] " +
+            equalTo("Field [value] of type [whole-number] conflicts with existing field of type [text] " +
                 "in other index."));
 
         response = client().prepareFieldStats().setFields("value", "value2").setLevel("indices").get();
@@ -418,19 +417,32 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
 
     public void testWildcardFields() throws Exception {
         assertAcked(prepareCreate("test1").addMapping(
-            "test", "foo", "type=long", "foobar", "type=text", "barfoo", "type=long"
+            "test", "foo", "type=long", "foobar", "type=long", "barfoo", "type=long"
         ));
         assertAcked(prepareCreate("test2").addMapping(
-            "test", "foobar", "type=text", "barfoo", "type=long"
+            "test", "foobar", "type=long", "barfoo", "type=long"
         ));
         ensureGreen("test1", "test2");
         FieldStatsResponse response = client().prepareFieldStats()
             .setFields("foo*")
             .get();
         assertAllSuccessful(response);
+        assertThat(response.getAllFieldStats().size(), equalTo(0));
+
+        indexRange("test1", "foo", -100, 0);
+        indexRange("test2", "foo", -10, 100);
+        indexRange("test1", "foobar", -10, 100);
+        indexRange("test2", "foobar", -100, 0);
+
+        response = client().prepareFieldStats()
+            .setFields("foo*")
+            .get();
+        assertAllSuccessful(response);
         assertThat(response.getAllFieldStats().size(), equalTo(2));
-        assertThat(response.getAllFieldStats().get("foo").getMinValue(), nullValue());
+        assertThat(response.getAllFieldStats().get("foo").getMinValue(), equalTo(-100L));
-        assertThat(response.getAllFieldStats().get("foobar").getMaxValue(), nullValue());
+        assertThat(response.getAllFieldStats().get("foo").getMaxValue(), equalTo(100L));
+        assertThat(response.getAllFieldStats().get("foobar").getMinValue(), equalTo(-100L));
+        assertThat(response.getAllFieldStats().get("foobar").getMaxValue(), equalTo(100L));
 
         response = client().prepareFieldStats()
             .setFields("foo*")
@@ -439,21 +451,26 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertAllSuccessful(response);
         assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2));
         assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(2));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMinValue(), nullValue());
+        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMinValue(), equalTo(-100L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMaxValue(), nullValue());
+        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMaxValue(), equalTo(0L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMinValue(), nullValue());
+        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMinValue(), equalTo(-10L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMaxValue(), nullValue());
+        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMaxValue(), equalTo(100L));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").size(), equalTo(1));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").size(), equalTo(2));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMinValue(), nullValue());
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMinValue(), equalTo(-100L));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMaxValue(), nullValue());
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMaxValue(), equalTo(0L));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foo").getMinValue(), equalTo(-10L));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foo").getMaxValue(), equalTo(100L));
     }
 
     private void indexRange(String index, long from, long to) throws Exception {
+        indexRange(index, "value", from, to);
+    }
+
+    private void indexRange(String index, String field, long from, long to) throws Exception {
         List<IndexRequestBuilder> requests = new ArrayList<>();
         for (long value = from; value <= to; value++) {
-            requests.add(client().prepareIndex(index, "test").setSource("value", value));
+            requests.add(client().prepareIndex(index, "test").setSource(field, value));
         }
         indexRandom(true, false, requests);
     }
-
 }

core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java

@@ -452,37 +452,23 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
     public void testEmptyIndex() {
         createIndex("test1", Settings.EMPTY, "type", "value", "type=date");
         FieldStatsResponse response = client().prepareFieldStats()
-                .setFields("value")
+            .setFields("*")
             .setLevel("indices")
             .get();
         assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(1));
+        assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(0));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxDoc(), equalTo(0L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getDocCount(), equalTo(0L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getSumDocFreq(), equalTo(0L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getSumTotalTermFreq(), equalTo(0L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").isSearchable(), equalTo(true));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").isAggregatable(), equalTo(true));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(null));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxValue(), equalTo(null));
-
-        response = client().prepareFieldStats()
-                .setFields("value")
-                .setIndexContraints(new IndexConstraint("value", MIN, GTE, "1998-01-01T00:00:00.000Z"))
-                .setLevel("indices")
-                .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0));
     }
 
-    public void testMetaFieldsSearchable() {
+    public void testMetaFieldsNotIndexed() {
-        createIndex("test1", Settings.EMPTY, "type", "value", "type=date");
+        createIndex("test", Settings.EMPTY);
+        client().prepareIndex("test", "type").setSource().get();
+        client().admin().indices().prepareRefresh().get();
+
         FieldStatsResponse response = client().prepareFieldStats()
-            .setFields("_id", "_index")
+            .setFields("_id", "_type")
             .get();
-        assertThat(response.getAllFieldStats().size(), equalTo(2));
+        assertThat(response.getAllFieldStats().size(), equalTo(1));
-        assertThat(response.getAllFieldStats().get("_id").isSearchable(), equalTo(true));
+        assertThat(response.getAllFieldStats().get("_type").isSearchable(), equalTo(true));
-        assertThat(response.getAllFieldStats().get("_index").isSearchable(), equalTo(true));
+        // assertThat(response.getAllFieldStats().get("_type").isAggregatable(), equalTo(true));
-        assertThat(response.getAllFieldStats().get("_id").isAggregatable(), equalTo(false));
-        assertThat(response.getAllFieldStats().get("_index").isAggregatable(), equalTo(true));
     }
 }

rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml

@@ -164,5 +164,5 @@ setup:
   - match: { indices._all.fields.number.min_value_as_string: "123" }
   - match: { indices._all.fields.number.max_value: 456 }
   - match: { indices._all.fields.number.max_value_as_string: "456" }
-  - match: { conflicts.bar: "Field [bar] of type [text] conflicts with existing field of type [whole-number] in other index." }
+  - match: { conflicts.bar: "Field [bar] of type [whole-number] conflicts with existing field of type [text] in other index." }
   - is_false: indices._all.fields.bar