honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #18212 from jimferenczi/field_stats_null 

Do not return fieldstats information for fields that exist in the mapping but not in the index.
This commit is contained in:
Jim Ferenczi 2016-05-09 19:25:38 +02:00
parent 214ce31306 aef78ceb13
commit ded91d5df0
16 changed files with 66 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java
View File

@ -117,11 +117,13 @@ public class TransportFieldStatsTransportAction extends
if (existing != null) {
if (existing.getType() != entry.getValue().getType()) {
if (conflicts.containsKey(entry.getKey()) == false) {
FieldStats[] fields = new FieldStats[] {entry.getValue(), existing};
Arrays.sort(fields, (o1, o2) -> Byte.compare(o1.getType(), o2.getType()));
conflicts.put(entry.getKey(),
"Field [" + entry.getKey() + "] of type [" +
FieldStats.typeName(entry.getValue().getType()) +
FieldStats.typeName(fields[0].getType()) +
"] conflicts with existing field of type [" +
FieldStats.typeName(existing.getType()) +
FieldStats.typeName(fields[1].getType()) +
"] in other index.");
}
} else {

2
core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
View File

@ -404,7 +404,7 @@ public abstract class MappedFieldType extends FieldType {
int maxDoc = reader.maxDoc();
Terms terms = MultiFields.getTerms(reader, name());
if (terms == null) {
return new FieldStats.Text(maxDoc, isSearchable(), isAggregatable());
return null;
}
FieldStats stats = new FieldStats.Text(maxDoc, terms.getDocCount(),
terms.getSumDocFreq(), terms.getSumTotalTermFreq(),

2
core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -399,7 +399,7 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
String field = name();
long size = XPointValues.size(reader, field);
if (size == 0) {
return new FieldStats.Date(reader.maxDoc(), isSearchable(), isAggregatable(), dateTimeFormatter());
return null;
}
int docCount = XPointValues.getDocCount(reader, field);
byte[] min = XPointValues.getMinPackedValue(reader, field);

2
core/src/main/java/org/elasticsearch/index/mapper/core/LegacyByteFieldMapper.java
View File

@ -174,7 +174,7 @@ public class LegacyByteFieldMapper extends LegacyNumberFieldMapper {
int maxDoc = reader.maxDoc();
Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
if (terms == null) {
return new FieldStats.Long(maxDoc, isSearchable(), isAggregatable());
return null;
}
long minValue = LegacyNumericUtils.getMinInt(terms);
long maxValue = LegacyNumericUtils.getMaxInt(terms);

2
core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java
View File

@ -379,7 +379,7 @@ public class LegacyDateFieldMapper extends LegacyNumberFieldMapper {
int maxDoc = reader.maxDoc();
Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
if (terms == null) {
return new FieldStats.Date(maxDoc, isSearchable(), isAggregatable(), dateTimeFormatter());
return null;
}
long minValue = LegacyNumericUtils.getMinLong(terms);
long maxValue = LegacyNumericUtils.getMaxLong(terms);

2
core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDoubleFieldMapper.java
View File

@ -185,7 +185,7 @@ public class LegacyDoubleFieldMapper extends LegacyNumberFieldMapper {
int maxDoc = reader.maxDoc();
Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
if (terms == null) {
return new FieldStats.Double(maxDoc, isSearchable(), isAggregatable());
return null;
}
double minValue = NumericUtils.sortableLongToDouble(LegacyNumericUtils.getMinLong(terms));
double maxValue = NumericUtils.sortableLongToDouble(LegacyNumericUtils.getMaxLong(terms));

2
core/src/main/java/org/elasticsearch/index/mapper/core/LegacyFloatFieldMapper.java
View File

@ -170,7 +170,7 @@ public class LegacyFloatFieldMapper extends LegacyNumberFieldMapper {
int maxDoc = reader.maxDoc();
Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
if (terms == null) {
return new FieldStats.Double(maxDoc, isSearchable(), isAggregatable());
return null;
}
float minValue = NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMinInt(terms));
float maxValue = NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMaxInt(terms));

2
core/src/main/java/org/elasticsearch/index/mapper/core/LegacyIntegerFieldMapper.java
View File

@ -174,7 +174,7 @@ public class LegacyIntegerFieldMapper extends LegacyNumberFieldMapper {
int maxDoc = reader.maxDoc();
Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
if (terms == null) {
return new FieldStats.Long(maxDoc, isSearchable(), isAggregatable());
return null;
}
long minValue = LegacyNumericUtils.getMinInt(terms);
long maxValue = LegacyNumericUtils.getMaxInt(terms);

3
core/src/main/java/org/elasticsearch/index/mapper/core/LegacyLongFieldMapper.java
View File

@ -173,8 +173,7 @@ public class LegacyLongFieldMapper extends LegacyNumberFieldMapper {
int maxDoc = reader.maxDoc();
Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
if (terms == null) {
return new FieldStats.Long(
maxDoc, isSearchable(), isAggregatable());
return null;
}
long minValue = LegacyNumericUtils.getMinLong(terms);
long maxValue = LegacyNumericUtils.getMaxLong(terms);

2
core/src/main/java/org/elasticsearch/index/mapper/core/LegacyShortFieldMapper.java
View File

@ -178,7 +178,7 @@ public class LegacyShortFieldMapper extends LegacyNumberFieldMapper {
int maxDoc = reader.maxDoc();
Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
if (terms == null) {
return new FieldStats.Long(maxDoc, isSearchable(), isAggregatable());
return null;
}
long minValue = LegacyNumericUtils.getMinInt(terms);
long maxValue = LegacyNumericUtils.getMaxInt(terms);

8
core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
View File

@ -262,7 +262,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
boolean isSearchable, boolean isAggregatable) throws IOException {
long size = XPointValues.size(reader, fieldName);
if (size == 0) {
return new FieldStats.Double(reader.maxDoc(), isSearchable, isAggregatable);
return null;
}
int docCount = XPointValues.getDocCount(reader, fieldName);
byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
@ -353,7 +353,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
boolean isSearchable, boolean isAggregatable) throws IOException {
long size = XPointValues.size(reader, fieldName);
if (size == 0) {
return new FieldStats.Double(reader.maxDoc(), isSearchable, isAggregatable);
return null;
}
int docCount = XPointValues.getDocCount(reader, fieldName);
byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
@ -588,7 +588,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
boolean isSearchable, boolean isAggregatable) throws IOException {
long size = XPointValues.size(reader, fieldName);
if (size == 0) {
return new FieldStats.Long(reader.maxDoc(), isSearchable, isAggregatable);
return null;
}
int docCount = XPointValues.getDocCount(reader, fieldName);
byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
@ -691,7 +691,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
boolean isSearchable, boolean isAggregatable) throws IOException {
long size = XPointValues.size(reader, fieldName);
if (size == 0) {
return new FieldStats.Long(reader.maxDoc(), isSearchable, isAggregatable);
return null;
}
int docCount = XPointValues.getDocCount(reader, fieldName);
byte[] min = XPointValues.getMinPackedValue(reader, fieldName);

2
core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
View File

@ -232,7 +232,7 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
String field = name();
long size = XPointValues.size(reader, field);
if (size == 0) {
return new FieldStats.Ip(reader.maxDoc(), isSearchable(), isAggregatable());
return null;
}
int docCount = XPointValues.getDocCount(reader, field);
byte[] min = XPointValues.getMinPackedValue(reader, field);

2
core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
View File

@ -252,7 +252,7 @@ public class LegacyIpFieldMapper extends LegacyNumberFieldMapper {
int maxDoc = reader.maxDoc();
Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
if (terms == null) {
return new FieldStats.Ip(maxDoc, isSearchable(), isAggregatable());
return null;
}
long minValue = LegacyNumericUtils.getMinLong(terms);
long maxValue = LegacyNumericUtils.getMaxLong(terms);

49
core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
View File

@ -246,7 +246,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
assertThat(response.getIndicesMergedFieldStats().get("_all").size(), equalTo(0));
assertThat(response.getConflicts().size(), equalTo(1));
assertThat(response.getConflicts().get("value"),
equalTo("Field [value] of type [text] conflicts with existing field of type [whole-number] " +
equalTo("Field [value] of type [whole-number] conflicts with existing field of type [text] " +
"in other index."));
response = client().prepareFieldStats().setFields("value").setLevel("indices").get();
@ -260,7 +260,6 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
equalTo(new BytesRef("b")));
}
@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-7257")
public void testIncompatibleFieldTypesMultipleFields() {
assertAcked(prepareCreate("test1").addMapping(
"test", "value", "type=long", "value2", "type=long"
@ -284,7 +283,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
assertThat(response.getIndicesMergedFieldStats().get("_all").get("value2").getMaxValue(), equalTo(1L));
assertThat(response.getConflicts().size(), equalTo(1));
assertThat(response.getConflicts().get("value"),
equalTo("Field [value] of type [text] conflicts with existing field of type [whole-number] " +
equalTo("Field [value] of type [whole-number] conflicts with existing field of type [text] " +
"in other index."));
response = client().prepareFieldStats().setFields("value", "value2").setLevel("indices").get();
@ -418,19 +417,32 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
public void testWildcardFields() throws Exception {
assertAcked(prepareCreate("test1").addMapping(
"test", "foo", "type=long", "foobar", "type=text", "barfoo", "type=long"
"test", "foo", "type=long", "foobar", "type=long", "barfoo", "type=long"
));
assertAcked(prepareCreate("test2").addMapping(
"test", "foobar", "type=text", "barfoo", "type=long"
"test", "foobar", "type=long", "barfoo", "type=long"
));
ensureGreen("test1", "test2");
FieldStatsResponse response = client().prepareFieldStats()
.setFields("foo*")
.get();
assertAllSuccessful(response);
assertThat(response.getAllFieldStats().size(), equalTo(0));
indexRange("test1", "foo", -100, 0);
indexRange("test2", "foo", -10, 100);
indexRange("test1", "foobar", -10, 100);
indexRange("test2", "foobar", -100, 0);
response = client().prepareFieldStats()
.setFields("foo*")
.get();
assertAllSuccessful(response);
assertThat(response.getAllFieldStats().size(), equalTo(2));
assertThat(response.getAllFieldStats().get("foo").getMinValue(), nullValue());
assertThat(response.getAllFieldStats().get("foobar").getMaxValue(), nullValue());
assertThat(response.getAllFieldStats().get("foo").getMinValue(), equalTo(-100L));
assertThat(response.getAllFieldStats().get("foo").getMaxValue(), equalTo(100L));
assertThat(response.getAllFieldStats().get("foobar").getMinValue(), equalTo(-100L));
assertThat(response.getAllFieldStats().get("foobar").getMaxValue(), equalTo(100L));
response = client().prepareFieldStats()
.setFields("foo*")
@ -439,21 +451,26 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
assertAllSuccessful(response);
assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2));
assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(2));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMinValue(), nullValue());
assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMaxValue(), nullValue());
assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMinValue(), nullValue());
assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMaxValue(), nullValue());
assertThat(response.getIndicesMergedFieldStats().get("test2").size(), equalTo(1));
assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMinValue(), nullValue());
assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMaxValue(), nullValue());
assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMinValue(), equalTo(-100L));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMaxValue(), equalTo(0L));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMinValue(), equalTo(-10L));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMaxValue(), equalTo(100L));
assertThat(response.getIndicesMergedFieldStats().get("test2").size(), equalTo(2));
assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMinValue(), equalTo(-100L));
assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMaxValue(), equalTo(0L));
assertThat(response.getIndicesMergedFieldStats().get("test2").get("foo").getMinValue(), equalTo(-10L));
assertThat(response.getIndicesMergedFieldStats().get("test2").get("foo").getMaxValue(), equalTo(100L));
}
private void indexRange(String index, long from, long to) throws Exception {
indexRange(index, "value", from, to);
}
private void indexRange(String index, String field, long from, long to) throws Exception {
List<IndexRequestBuilder> requests = new ArrayList<>();
for (long value = from; value <= to; value++) {
requests.add(client().prepareIndex(index, "test").setSource("value", value));
requests.add(client().prepareIndex(index, "test").setSource(field, value));
}
indexRandom(true, false, requests);
}
}

36
core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
View File

@ -452,37 +452,23 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
public void testEmptyIndex() {
createIndex("test1", Settings.EMPTY, "type", "value", "type=date");
FieldStatsResponse response = client().prepareFieldStats()
.setFields("value")
.setFields("*")
.setLevel("indices")
.get();
assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(1));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxDoc(), equalTo(0L));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getDocCount(), equalTo(0L));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getSumDocFreq(), equalTo(0L));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getSumTotalTermFreq(), equalTo(0L));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").isSearchable(), equalTo(true));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").isAggregatable(), equalTo(true));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(null));
assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxValue(), equalTo(null));
response = client().prepareFieldStats()
.setFields("value")
.setIndexContraints(new IndexConstraint("value", MIN, GTE, "1998-01-01T00:00:00.000Z"))
.setLevel("indices")
.get();
assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0));
assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(0));
}
public void testMetaFieldsSearchable() {
createIndex("test1", Settings.EMPTY, "type", "value", "type=date");
public void testMetaFieldsNotIndexed() {
createIndex("test", Settings.EMPTY);
client().prepareIndex("test", "type").setSource().get();
client().admin().indices().prepareRefresh().get();
FieldStatsResponse response = client().prepareFieldStats()
.setFields("_id", "_index")
.setFields("_id", "_type")
.get();
assertThat(response.getAllFieldStats().size(), equalTo(2));
assertThat(response.getAllFieldStats().get("_id").isSearchable(), equalTo(true));
assertThat(response.getAllFieldStats().get("_index").isSearchable(), equalTo(true));
assertThat(response.getAllFieldStats().get("_id").isAggregatable(), equalTo(false));
assertThat(response.getAllFieldStats().get("_index").isAggregatable(), equalTo(true));
assertThat(response.getAllFieldStats().size(), equalTo(1));
assertThat(response.getAllFieldStats().get("_type").isSearchable(), equalTo(true));
// assertThat(response.getAllFieldStats().get("_type").isAggregatable(), equalTo(true));
}
}

2
rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
View File

@ -164,5 +164,5 @@ setup:
- match: { indices._all.fields.number.min_value_as_string: "123" }
- match: { indices._all.fields.number.max_value: 456 }
- match: { indices._all.fields.number.max_value_as_string: "456" }
- match: { conflicts.bar: "Field [bar] of type [text] conflicts with existing field of type [whole-number] in other index." }
- match: { conflicts.bar: "Field [bar] of type [whole-number] conflicts with existing field of type [text] in other index." }
- is_false: indices._all.fields.bar