Fix xcontent rendering of ip terms aggs. #18003

Currently terms on an ip address try to put their binary representation in the
json response. With this commit, they would return a formatted ip address:

```
      "buckets": [
        {
          "key": "192.168.1.7",
          "doc_count": 1
        }
      ]
```
This commit is contained in:
Adrien Grand 2016-04-27 09:17:10 +02:00
parent ddbfda2c68
commit 61b1f4ad0b
10 changed files with 208 additions and 46 deletions

View File

@ -110,7 +110,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
}
if (spare == null) {
spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null);
spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null, format);
}
spare.bucketOrd = bucketOrd;
copy(globalOrds.lookupOrd(globalTermOrd), spare.termBytes);
@ -135,7 +135,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
list[i] = bucket;
}
return new SignificantStringTerms(subsetSize, supersetSize, name, bucketCountThresholds.getRequiredSize(),
return new SignificantStringTerms(subsetSize, supersetSize, name, format, bucketCountThresholds.getRequiredSize(),
bucketCountThresholds.getMinDocCount(), significanceHeuristic, Arrays.asList(list), pipelineAggregators(),
metaData());
}
@ -146,7 +146,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
ContextIndexSearcher searcher = context.searchContext().searcher();
IndexReader topReader = searcher.getIndexReader();
int supersetSize = topReader.numDocs();
return new SignificantStringTerms(0, supersetSize, name, bucketCountThresholds.getRequiredSize(),
return new SignificantStringTerms(0, supersetSize, name, format, bucketCountThresholds.getRequiredSize(),
bucketCountThresholds.getMinDocCount(), significanceHeuristic,
Collections.<InternalSignificantTerms.Bucket> emptyList(), pipelineAggregators(), metaData());
}

View File

@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.bucket.significant;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.InternalAggregations;
@ -33,6 +34,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
*
@ -56,15 +58,19 @@ public abstract class InternalSignificantTerms<A extends InternalSignificantTerm
long bucketOrd;
protected InternalAggregations aggregations;
double score;
transient final DocValueFormat format;
protected Bucket(long subsetSize, long supersetSize) {
protected Bucket(long subsetSize, long supersetSize, DocValueFormat format) {
// for serialization
super(subsetSize, supersetSize);
this.format = format;
}
protected Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, InternalAggregations aggregations) {
protected Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize,
InternalAggregations aggregations, DocValueFormat format) {
super(subsetDf, subsetSize, supersetDf, supersetSize);
this.aggregations = aggregations;
this.format = format;
}
@Override
@ -122,9 +128,11 @@ public abstract class InternalSignificantTerms<A extends InternalSignificantTerm
}
}
protected InternalSignificantTerms(long subsetSize, long supersetSize, String name, int requiredSize, long minDocCount,
SignificanceHeuristic significanceHeuristic, List<? extends Bucket> buckets, List<PipelineAggregator> pipelineAggregators,
Map<String, Object> metaData) {
protected DocValueFormat format;
protected InternalSignificantTerms(long subsetSize, long supersetSize, String name, DocValueFormat format, int requiredSize,
long minDocCount, SignificanceHeuristic significanceHeuristic, List<? extends Bucket> buckets,
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
super(name, pipelineAggregators, metaData);
this.requiredSize = requiredSize;
this.minDocCount = minDocCount;
@ -132,6 +140,7 @@ public abstract class InternalSignificantTerms<A extends InternalSignificantTerm
this.subsetSize = subsetSize;
this.supersetSize = supersetSize;
this.significanceHeuristic = significanceHeuristic;
this.format = Objects.requireNonNull(format);
}
@Override

View File

@ -81,18 +81,15 @@ public class SignificantLongTerms extends InternalSignificantTerms<SignificantLo
static class Bucket extends InternalSignificantTerms.Bucket {
long term;
private transient final DocValueFormat format;
public Bucket(long subsetSize, long supersetSize, DocValueFormat formatter) {
super(subsetSize, supersetSize);
this.format = formatter;
public Bucket(long subsetSize, long supersetSize, DocValueFormat format) {
super(subsetSize, supersetSize, format);
// for serialization
}
public Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, long term, InternalAggregations aggregations,
DocValueFormat format) {
super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations);
this.format = format;
super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
this.term = term;
}
@ -160,7 +157,6 @@ public class SignificantLongTerms extends InternalSignificantTerms<SignificantLo
return builder;
}
}
private DocValueFormat format;
SignificantLongTerms() {
} // for serialization
@ -169,8 +165,7 @@ public class SignificantLongTerms extends InternalSignificantTerms<SignificantLo
long minDocCount, SignificanceHeuristic significanceHeuristic, List<? extends InternalSignificantTerms.Bucket> buckets,
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
super(subsetSize, supersetSize, name, requiredSize, minDocCount, significanceHeuristic, buckets, pipelineAggregators, metaData);
this.format = Objects.requireNonNull(format);
super(subsetSize, supersetSize, name, format, requiredSize, minDocCount, significanceHeuristic, buckets, pipelineAggregators, metaData);
}
@Override

View File

@ -22,6 +22,7 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.AggregationStreams;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.InternalAggregations;
@ -55,7 +56,8 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
@Override
public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
Bucket buckets = new Bucket((long) context.attributes().get("subsetSize"), (long) context.attributes().get("supersetSize"));
Bucket buckets = new Bucket((long) context.attributes().get("subsetSize"),
(long) context.attributes().get("supersetSize"), context.format());
buckets.readFrom(in);
return buckets;
}
@ -84,18 +86,20 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
BytesRef termBytes;
public Bucket(long subsetSize, long supersetSize) {
public Bucket(long subsetSize, long supersetSize, DocValueFormat format) {
// for serialization
super(subsetSize, supersetSize);
super(subsetSize, supersetSize, format);
}
public Bucket(BytesRef term, long subsetDf, long subsetSize, long supersetDf, long supersetSize, InternalAggregations aggregations) {
super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations);
public Bucket(BytesRef term, long subsetDf, long subsetSize, long supersetDf, long supersetSize, InternalAggregations aggregations,
DocValueFormat format) {
super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
this.termBytes = term;
}
public Bucket(BytesRef term, long subsetDf, long subsetSize, long supersetDf, long supersetSize, InternalAggregations aggregations, double score) {
this(term, subsetDf, subsetSize, supersetDf, supersetSize, aggregations);
public Bucket(BytesRef term, long subsetDf, long subsetSize, long supersetDf, long supersetSize,
InternalAggregations aggregations, double score, DocValueFormat format) {
this(term, subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
this.score = score;
}
@ -112,7 +116,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
@Override
public String getKeyAsString() {
return termBytes.utf8ToString();
return format.format(termBytes);
}
@Override
@ -122,7 +126,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
@Override
Bucket newBucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, InternalAggregations aggregations) {
return new Bucket(termBytes, subsetDf, subsetSize, supersetDf, supersetSize, aggregations);
return new Bucket(termBytes, subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
}
@Override
@ -146,7 +150,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.utf8Field(CommonFields.KEY, termBytes);
builder.field(CommonFields.KEY, getKeyAsString());
builder.field(CommonFields.DOC_COUNT, getDocCount());
builder.field("score", score);
builder.field("bg_count", supersetDf);
@ -158,11 +162,11 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
SignificantStringTerms() {} // for serialization
public SignificantStringTerms(long subsetSize, long supersetSize, String name, int requiredSize, long minDocCount,
SignificanceHeuristic significanceHeuristic, List<? extends InternalSignificantTerms.Bucket> buckets,
public SignificantStringTerms(long subsetSize, long supersetSize, String name, DocValueFormat format, int requiredSize,
long minDocCount, SignificanceHeuristic significanceHeuristic, List<? extends InternalSignificantTerms.Bucket> buckets,
List<PipelineAggregator> pipelineAggregators,
Map<String, Object> metaData) {
super(subsetSize, supersetSize, name, requiredSize, minDocCount, significanceHeuristic, buckets, pipelineAggregators, metaData);
super(subsetSize, supersetSize, name, format, requiredSize, minDocCount, significanceHeuristic, buckets, pipelineAggregators, metaData);
}
@Override
@ -172,25 +176,26 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
@Override
public SignificantStringTerms create(List<SignificantStringTerms.Bucket> buckets) {
return new SignificantStringTerms(this.subsetSize, this.supersetSize, this.name, this.requiredSize, this.minDocCount,
return new SignificantStringTerms(this.subsetSize, this.supersetSize, this.name, this.format, this.requiredSize, this.minDocCount,
this.significanceHeuristic, buckets, this.pipelineAggregators(), this.metaData);
}
@Override
public Bucket createBucket(InternalAggregations aggregations, SignificantStringTerms.Bucket prototype) {
return new Bucket(prototype.termBytes, prototype.subsetDf, prototype.subsetSize, prototype.supersetDf, prototype.supersetSize,
aggregations);
aggregations, prototype.format);
}
@Override
protected SignificantStringTerms create(long subsetSize, long supersetSize, List<InternalSignificantTerms.Bucket> buckets,
InternalSignificantTerms prototype) {
return new SignificantStringTerms(subsetSize, supersetSize, prototype.getName(), prototype.requiredSize, prototype.minDocCount,
prototype.significanceHeuristic, buckets, prototype.pipelineAggregators(), prototype.getMetaData());
return new SignificantStringTerms(subsetSize, supersetSize, prototype.getName(), prototype.format, prototype.requiredSize,
prototype.minDocCount, prototype.significanceHeuristic, buckets, prototype.pipelineAggregators(), prototype.getMetaData());
}
@Override
protected void doReadFrom(StreamInput in) throws IOException {
this.format = in.readNamedWriteable(DocValueFormat.class);
this.requiredSize = readSize(in);
this.minDocCount = in.readVLong();
this.subsetSize = in.readVLong();
@ -199,7 +204,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
int size = in.readVInt();
List<InternalSignificantTerms.Bucket> buckets = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
Bucket bucket = new Bucket(subsetSize, supersetSize);
Bucket bucket = new Bucket(subsetSize, supersetSize, format);
bucket.readFrom(in);
buckets.add(bucket);
}
@ -209,6 +214,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
@Override
protected void doWriteTo(StreamOutput out) throws IOException {
out.writeNamedWriteable(format);
writeSize(requiredSize, out);
out.writeVLong(minDocCount);
out.writeVLong(subsetSize);

View File

@ -90,7 +90,7 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator {
}
if (spare == null) {
spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null);
spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null, format);
}
bucketOrds.get(i, spare.termBytes);
@ -117,7 +117,7 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator {
list[i] = bucket;
}
return new SignificantStringTerms(subsetSize, supersetSize, name, bucketCountThresholds.getRequiredSize(),
return new SignificantStringTerms(subsetSize, supersetSize, name, format, bucketCountThresholds.getRequiredSize(),
bucketCountThresholds.getMinDocCount(), significanceHeuristic, Arrays.asList(list), pipelineAggregators(),
metaData());
}
@ -128,7 +128,7 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator {
ContextIndexSearcher searcher = context.searchContext().searcher();
IndexReader topReader = searcher.getIndexReader();
int supersetSize = topReader.numDocs();
return new SignificantStringTerms(0, supersetSize, name, bucketCountThresholds.getRequiredSize(),
return new SignificantStringTerms(0, supersetSize, name, format, bucketCountThresholds.getRequiredSize(),
bucketCountThresholds.getMinDocCount(), significanceHeuristic,
Collections.<InternalSignificantTerms.Bucket> emptyList(), pipelineAggregators(), metaData());
}

View File

@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.bucket.significant;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.AggregationStreams;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.InternalAggregations;
@ -59,8 +60,8 @@ public class UnmappedSignificantTerms extends InternalSignificantTerms<UnmappedS
public UnmappedSignificantTerms(String name, int requiredSize, long minDocCount, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
//We pass zero for index/subset sizes because for the purpose of significant term analysis
// we assume an unmapped index's size is irrelevant to the proceedings.
super(0, 0, name, requiredSize, minDocCount, SignificantTermsAggregatorBuilder.DEFAULT_SIGNIFICANCE_HEURISTIC, BUCKETS,
pipelineAggregators, metaData);
super(0, 0, name, DocValueFormat.RAW, requiredSize, minDocCount, SignificantTermsAggregatorBuilder.DEFAULT_SIGNIFICANCE_HEURISTIC,
BUCKETS, pipelineAggregators, metaData);
}
@Override

View File

@ -141,7 +141,7 @@ public class StringTerms extends InternalTerms<StringTerms, StringTerms.Bucket>
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.utf8Field(CommonFields.KEY, termBytes);
builder.field(CommonFields.KEY, getKeyAsString());
builder.field(CommonFields.DOC_COUNT, getDocCount());
if (showDocCountError) {
builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, getDocCountError());

View File

@ -131,9 +131,9 @@ public class SignificanceHeuristicTests extends ESTestCase {
sTerms[1] = new SignificantLongTerms();
} else {
BytesRef term = new BytesRef("someterm");
buckets.add(new SignificantStringTerms.Bucket(term, 1, 2, 3, 4, InternalAggregations.EMPTY));
sTerms[0] = new SignificantStringTerms(10, 20, "some_name", 1, 1, heuristic, buckets, Collections.emptyList(),
null);
buckets.add(new SignificantStringTerms.Bucket(term, 1, 2, 3, 4, InternalAggregations.EMPTY, DocValueFormat.RAW));
sTerms[0] = new SignificantStringTerms(10, 20, "some_name", DocValueFormat.RAW, 1, 1, heuristic, buckets,
Collections.emptyList(), null);
sTerms[1] = new SignificantStringTerms();
}
return sTerms;
@ -184,7 +184,7 @@ public class SignificanceHeuristicTests extends ESTestCase {
private InternalSignificantTerms createAggregation(String type, SignificanceHeuristic significanceHeuristic, List<InternalSignificantTerms.Bucket> buckets, long subsetSize, long supersetSize) {
if (type.equals("string")) {
return new SignificantStringTerms(subsetSize, supersetSize, "sig_terms", 2, -1, significanceHeuristic, buckets, new ArrayList<PipelineAggregator>(), new HashMap<String, Object>());
return new SignificantStringTerms(subsetSize, supersetSize, "sig_terms", DocValueFormat.RAW, 2, -1, significanceHeuristic, buckets, new ArrayList<PipelineAggregator>(), new HashMap<String, Object>());
} else {
return new SignificantLongTerms(subsetSize, supersetSize, "sig_terms", DocValueFormat.RAW, 2, -1, significanceHeuristic, buckets, new ArrayList<PipelineAggregator>(), new HashMap<String, Object>());
}
@ -192,7 +192,7 @@ public class SignificanceHeuristicTests extends ESTestCase {
private InternalSignificantTerms.Bucket createBucket(String type, long subsetDF, long subsetSize, long supersetDF, long supersetSize, long label) {
if (type.equals("string")) {
return new SignificantStringTerms.Bucket(new BytesRef(Long.toString(label).getBytes(StandardCharsets.UTF_8)), subsetDF, subsetSize, supersetDF, supersetSize, InternalAggregations.EMPTY);
return new SignificantStringTerms.Bucket(new BytesRef(Long.toString(label).getBytes(StandardCharsets.UTF_8)), subsetDF, subsetSize, supersetDF, supersetSize, InternalAggregations.EMPTY, DocValueFormat.RAW);
} else {
return new SignificantLongTerms.Bucket(subsetDF, subsetSize, supersetDF, supersetSize, label, InternalAggregations.EMPTY, DocValueFormat.RAW);
}

View File

@ -0,0 +1,110 @@
setup:
- do:
indices.create:
index: test_1
body:
settings:
number_of_replicas: 0
mappings:
test:
properties:
str:
type: keyword
ip:
type: ip
- do:
cluster.health:
wait_for_status: green
---
"Basic test":
- do:
index:
index: test_1
type: test
id: 1
body: { "str" : "abc" }
- do:
index:
index: test_1
type: test
id: 2
body: { "str": "abc" }
- do:
index:
index: test_1
type: test
id: 3
body: { "str": "bcd" }
- do:
indices.refresh: {}
- do:
search:
body: { "aggs" : { "str_terms" : { "terms" : { "field" : "str" } } } }
- match: { hits.total: 3 }
- length: { aggregations.str_terms.buckets: 2 }
- match: { aggregations.str_terms.buckets.0.key: "abc" }
- is_false: aggregations.str_terms.buckets.0.key_as_string
- match: { aggregations.str_terms.buckets.0.doc_count: 2 }
- match: { aggregations.str_terms.buckets.1.key: "bcd" }
- is_false: aggregations.str_terms.buckets.1.key_as_string
- match: { aggregations.str_terms.buckets.1.doc_count: 1 }
---
"IP test":
- do:
index:
index: test_1
type: test
id: 1
body: { "ip": "::1" }
- do:
index:
index: test_1
type: test
id: 2
body: { "ip": "127.0.0.1" }
- do:
index:
index: test_1
type: test
id: 3
body: { "ip": "::1" }
- do:
indices.refresh: {}
- do:
search:
body: { "aggs" : { "ip_terms" : { "terms" : { "field" : "ip" } } } }
- match: { hits.total: 3 }
- length: { aggregations.ip_terms.buckets: 2 }
- match: { aggregations.ip_terms.buckets.0.key: "::1" }
- is_false: aggregations.ip_terms.buckets.0.key_as_string
- match: { aggregations.ip_terms.buckets.0.doc_count: 2 }
- match: { aggregations.ip_terms.buckets.1.key: "127.0.0.1" }
- is_false: aggregations.ip_terms.buckets.1.key_as_string
- match: { aggregations.ip_terms.buckets.1.doc_count: 1 }

View File

@ -80,3 +80,44 @@
- match: {aggregations.class.buckets.0.sig_terms.buckets.0.key: "bad"}
- match: {aggregations.class.buckets.1.sig_terms.buckets.0.key: "good"}
---
"IP test":
- do:
indices.create:
index: ip_index
body:
mappings:
doc:
properties:
ip:
type: ip
- do:
index:
index: ip_index
type: doc
id: 1
body: { ip: "::1" }
- do:
index:
index: ip_index
type: doc
id: 2
body: { }
- do:
indices.refresh: {}
- do:
search:
body: { "query" : { "exists" : { "field" : "ip" } }, "aggs" : { "ip_terms" : { "significant_terms" : { "field" : "ip", "min_doc_count" : 1 } } } }
- match: { hits.total: 1 }
- length: { aggregations.ip_terms.buckets: 1 }
- match: { aggregations.ip_terms.buckets.0.key: "::1" }
- is_false: aggregations.ip_terms.buckets.0.key_as_string
- match: { aggregations.ip_terms.buckets.0.doc_count: 1 }