honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added more unit test coverage for terms aggregation and 

removed terms agg integration tests that were replaced by unit tests.
This commit is contained in:
Martijn van Groningen 2017-05-29 21:26:19 +02:00
parent 9531ef25ec
commit a089dc9dcd
No known key found for this signature in database
GPG Key ID: AB236F4FCF2AF12A
10 changed files with 817 additions and 1232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
core/src/main/java/org/elasticsearch/search/aggregations/BucketOrder.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.search.aggregations;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContentObject;
@ -124,4 +125,9 @@ public abstract class BucketOrder implements ToXContentObject, Writeable {
public void writeTo(StreamOutput out) throws IOException {
InternalOrder.Streams.writeOrder(this, out);
}
@Override
public String toString() {
return Strings.toString(this);
}
}

4
core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java
View File

@ -25,9 +25,9 @@ import org.elasticsearch.index.fielddata.FieldData;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.support.ValuesSource;
import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
import org.elasticsearch.search.internal.SearchContext;
@ -39,7 +39,7 @@ import java.util.stream.Collectors;
public class DoubleTermsAggregator extends LongTermsAggregator {
public DoubleTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format,
DoubleTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format,
BucketOrder order, BucketCountThresholds bucketCountThresholds, SearchContext aggregationContext, Aggregator parent,
SubAggCollectionMode collectionMode, boolean showTermDocCountError, IncludeExclude.LongFilter longFilter,
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {

8
core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
View File

@ -27,14 +27,14 @@ import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.InternalOrder;
import org.elasticsearch.search.aggregations.LeafBucketCollector;
import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
import org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue;
import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.InternalOrder;
import org.elasticsearch.search.aggregations.support.ValuesSource;
import org.elasticsearch.search.internal.SearchContext;
@ -153,7 +153,7 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
final StringTerms.Bucket[] list = new StringTerms.Bucket[ordered.size()];
long survivingBucketOrds[] = new long[ordered.size()];
for (int i = ordered.size() - 1; i >= 0; --i) {
final StringTerms.Bucket bucket = (StringTerms.Bucket) ordered.pop();
final StringTerms.Bucket bucket = ordered.pop();
survivingBucketOrds[i] = bucket.bucketOrd;
list[i] = bucket;
otherDocCount -= bucket.docCount;
@ -163,7 +163,7 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
// Now build the aggs
for (int i = 0; i < list.length; i++) {
final StringTerms.Bucket bucket = (StringTerms.Bucket)list[i];
final StringTerms.Bucket bucket = list[i];
bucket.termBytes = BytesRef.deepCopyOf(bucket.termBytes);
bucket.aggregations = bucketAggregations(bucket.bucketOrd);
bucket.docCountError = 0;

6
core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
View File

@ -27,14 +27,14 @@ import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.AggregatorFactory;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.InternalOrder;
import org.elasticsearch.search.aggregations.NonCollectingAggregator;
import org.elasticsearch.search.aggregations.bucket.BucketUtils;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.InternalOrder;
import org.elasticsearch.search.aggregations.support.ValuesSource;
import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
@ -53,7 +53,7 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
private boolean showTermDocCountError;
public TermsAggregatorFactory(String name,
TermsAggregatorFactory(String name,
ValuesSourceConfig<ValuesSource> config,
BucketOrder order,
IncludeExclude includeExclude,

323
core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
View File

@ -30,9 +30,8 @@ import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.filter.Filter;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
@ -41,9 +40,7 @@ import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.elasticsearch.search.aggregations.metrics.stats.Stats;
import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.test.ESIntegTestCase;
import org.hamcrest.Matchers;
import java.io.IOException;
import java.util.ArrayList;
@ -61,12 +58,10 @@ import java.util.function.Function;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction;
import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats;
import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
import static org.elasticsearch.search.aggregations.AggregationBuilders.max;
import static org.elasticsearch.search.aggregations.AggregationBuilders.stats;
import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
@ -75,7 +70,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.core.IsNull.notNullValue;
@ESIntegTestCase.SuiteScopeTestCase
@ -287,85 +281,6 @@ public class DoubleTermsIT extends AbstractTermsTestCase {
assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]"));
}
public void testSingleValueField() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + (double) i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + (double)i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
}
}
public void testSingleValueFieldWithMaxSize() throws Exception {
SearchResponse response = client().prepareSearch("high_card_idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.size(20)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.key(true))) // we need to sort by terms cause we're checking the first 20 values
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(20));
for (int i = 0; i < 20; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + (double) i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + (double) i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
}
}
public void testSingleValueFieldWithFiltering() throws Exception {
double includes[] = { 1, 2, 3, 98.2 };
double excludes[] = { 2, 4, 99 };
double empty[] = {};
testIncludeExcludeResults(includes, empty, new double[] { 1, 2, 3 });
testIncludeExcludeResults(includes, excludes, new double[] { 1, 3 });
testIncludeExcludeResults(empty, excludes, new double[] { 0, 1, 3 });
}
private void testIncludeExcludeResults(double[] includes, double[] excludes, double[] expecteds) {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.includeExclude(new IncludeExclude(includes, excludes))
.collectMode(randomFrom(SubAggCollectionMode.values())))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(expecteds.length));
for (int i = 0; i < expecteds.length; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + expecteds[i]);
assertThat(bucket, notNullValue());
assertThat(bucket.getDocCount(), equalTo(1L));
}
}
public void testSingleValueFieldWithPartitionedFiltering() throws Exception {
runTestFieldWithPartitionedFiltering(SINGLE_VALUED_FIELD_NAME);
}
@ -403,120 +318,6 @@ public class DoubleTermsIT extends AbstractTermsTestCase {
assertEquals(expectedCardinality, foundTerms.size());
}
public void testSingleValueFieldOrderedByTermAsc() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.key(true)))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + (double)i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
i++;
}
}
public void testSingleValueFieldOrderedByTermDesc() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.key(false)))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 4;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + (double) i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
i--;
}
}
public void testSingleValueFieldOrderedByTieBreaker() throws Exception {
SearchResponse response = client().prepareSearch("idx").setTypes("type")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.aggregation("max_constant", randomBoolean()))
.subAggregation(max("max_constant").field("constant")))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + (double)i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
i++;
}
}
public void testSingleValuedFieldWithSubAggregation() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.subAggregation(sum("sum").field(MULTI_VALUED_FIELD_NAME)))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
assertThat(((InternalAggregation)terms).getProperty("_bucket_count"), equalTo(5));
Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key");
Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count");
Object[] propertiesCounts = (Object[]) ((InternalAggregation)terms).getProperty("sum.value");
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + (double) i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + (double) i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
Sum sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
assertThat((long) sum.getValue(), equalTo(i+i+1L));
assertThat((double) propertiesKeys[i], equalTo((double) i));
assertThat((long) propertiesDocCounts[i], equalTo(1L));
assertThat((double) propertiesCounts[i], equalTo((double) i + i + 1L));
}
}
public void testSingleValuedFieldWithValueScript() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
@ -542,34 +343,6 @@ public class DoubleTermsIT extends AbstractTermsTestCase {
}
}
public void testMultiValuedField() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(MULTI_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(6));
for (int i = 0; i < 6; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + (double) i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + (double) i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
if (i == 0 || i == 5) {
assertThat(bucket.getDocCount(), equalTo(1L));
} else {
assertThat(bucket.getDocCount(), equalTo(2L));
}
}
}
public void testMultiValuedFieldWithValueScript() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
@ -696,23 +469,6 @@ public class DoubleTermsIT extends AbstractTermsTestCase {
}
}
public void testUnmapped() throws Exception {
SearchResponse response = client().prepareSearch("idx_unmapped")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.size(randomIntBetween(1, 5))
.collectMode(randomFrom(SubAggCollectionMode.values())))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(0));
}
public void testPartiallyUnmapped() throws Exception {
SearchResponse response = client().prepareSearch("idx_unmapped", "idx")
.addAggregation(terms("terms")
@ -763,53 +519,6 @@ public class DoubleTermsIT extends AbstractTermsTestCase {
}
}
public void testEmptyAggregation() throws Exception {
SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1L).minDocCount(0)
.subAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME)))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, Matchers.notNullValue());
Histogram.Bucket bucket = histo.getBuckets().get(1);
assertThat(bucket, Matchers.notNullValue());
Terms terms = bucket.getAggregations().get("terms");
assertThat(terms, Matchers.notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().isEmpty(), is(true));
}
public void testSingleValuedFieldOrderedBySingleValueSubAggregationAsc() throws Exception {
boolean asc = true;
SearchResponse response = client()
.prepareSearch("idx")
.addAggregation(
terms("terms").field(SINGLE_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.aggregation("avg_i", asc)).subAggregation(avg("avg_i").field(SINGLE_VALUED_FIELD_NAME)))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + (double) i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + (double)i));
assertThat(bucket.getDocCount(), equalTo(1L));
Avg avg = bucket.getAggregations().get("avg_i");
assertThat(avg, notNullValue());
assertThat(avg.getValue(), equalTo((double) i));
}
}
public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscWithSubTermsAgg() throws Exception {
boolean asc = true;
SearchResponse response = client()
@ -1019,36 +728,6 @@ public class DoubleTermsIT extends AbstractTermsTestCase {
}
}
public void testSingleValuedFieldOrderedBySingleValueSubAggregationDesc() throws Exception {
boolean asc = false;
SearchResponse response = client()
.prepareSearch("idx")
.addAggregation(
terms("terms").field(SINGLE_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.aggregation("avg_i", asc)).subAggregation(avg("avg_i").field(SINGLE_VALUED_FIELD_NAME)))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
for (int i = 4; i >= 0; i--) {
Terms.Bucket bucket = terms.getBucketByKey("" + (double) i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + (double)i));
assertThat(bucket.getDocCount(), equalTo(1L));
Avg avg = bucket.getAggregations().get("avg_i");
assertThat(avg, notNullValue());
assertThat(avg.getValue(), equalTo((double) i));
}
}
public void testSingleValuedFieldOrderedByMultiValueSubAggregationAsc() throws Exception {
boolean asc = true;
SearchResponse response = client()

30
core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java
View File

@ -18,9 +18,6 @@
*/
package org.elasticsearch.search.aggregations.bucket;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.plugins.Plugin;
@ -35,6 +32,9 @@ import java.util.Collections;
import java.util.Map;
import java.util.function.Function;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
public class IpTermsIT extends AbstractTermsTestCase {
@Override
@ -62,30 +62,6 @@ public class IpTermsIT extends AbstractTermsTestCase {
}
}
public void testBasics() throws Exception {
assertAcked(prepareCreate("index").addMapping("type", "ip", "type=ip"));
indexRandom(true,
client().prepareIndex("index", "type", "1").setSource("ip", "192.168.1.7"),
client().prepareIndex("index", "type", "2").setSource("ip", "192.168.1.7"),
client().prepareIndex("index", "type", "3").setSource("ip", "2001:db8::2:1"));
SearchResponse response = client().prepareSearch("index").addAggregation(
AggregationBuilders.terms("my_terms").field("ip").executionHint(randomExecutionHint())).get();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("my_terms");
assertEquals(2, terms.getBuckets().size());
Terms.Bucket bucket1 = terms.getBuckets().get(0);
assertEquals(2, bucket1.getDocCount());
assertEquals("192.168.1.7", bucket1.getKey());
assertEquals("192.168.1.7", bucket1.getKeyAsString());
Terms.Bucket bucket2 = terms.getBuckets().get(1);
assertEquals(1, bucket2.getDocCount());
assertEquals("2001:db8::2:1", bucket2.getKey());
assertEquals("2001:db8::2:1", bucket2.getKeyAsString());
}
public void testScriptValue() throws Exception {
assertAcked(prepareCreate("index").addMapping("type", "ip", "type=ip"));
indexRandom(true,

337
core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
View File

@ -29,9 +29,8 @@ import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.filter.Filter;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
@ -40,9 +39,7 @@ import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.elasticsearch.search.aggregations.metrics.stats.Stats;
import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.test.ESIntegTestCase;
import org.hamcrest.Matchers;
import java.io.IOException;
import java.util.ArrayList;
@ -59,11 +56,9 @@ import java.util.Set;
import java.util.function.Function;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats;
import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
import static org.elasticsearch.search.aggregations.AggregationBuilders.max;
import static org.elasticsearch.search.aggregations.AggregationBuilders.stats;
import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
@ -72,7 +67,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.core.IsNull.notNullValue;
@ESIntegTestCase.SuiteScopeTestCase
@ -280,74 +274,6 @@ public class LongTermsIT extends AbstractTermsTestCase {
assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]"));
}
public void testSingleValueField() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
}
}
public void testSingleValueFieldWithFiltering() throws Exception {
long includes[] = { 1, 2, 3, 98 };
long excludes[] = { -1, 2, 4 };
long empty[] = {};
testIncludeExcludeResults(1, includes, empty, new long[] { 1, 2, 3 }, new long[0]);
testIncludeExcludeResults(1, includes, excludes, new long[] { 1, 3 }, new long[0]);
testIncludeExcludeResults(1, empty, excludes, new long[] { 0, 1, 3 }, new long[0]);
testIncludeExcludeResults(0, includes, empty, new long[] { 1, 2, 3}, new long[] { 98 });
testIncludeExcludeResults(0, includes, excludes, new long[] { 1, 3 }, new long[] { 98 });
testIncludeExcludeResults(0, empty, excludes, new long[] { 0, 1, 3 }, new long[] {5, 6, 7, 8, 9, 10, 11});
}
private void testIncludeExcludeResults(int minDocCount, long[] includes, long[] excludes,
long[] expectedWithCounts, long[] expectedZeroCounts) {
SearchResponse response = client().prepareSearch("idx", "high_card_idx")
.setQuery(QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("_index", "high_card_idx")))
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.includeExclude(new IncludeExclude(includes, excludes))
.collectMode(randomFrom(SubAggCollectionMode.values()))
.minDocCount(minDocCount))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(expectedWithCounts.length + expectedZeroCounts.length));
for (int i = 0; i < expectedWithCounts.length; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + expectedWithCounts[i]);
assertThat(bucket, notNullValue());
assertThat(bucket.getDocCount(), equalTo(1L));
}
for (int i = 0; i < expectedZeroCounts.length; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + expectedZeroCounts[i]);
assertThat(bucket, notNullValue());
assertThat(bucket.getDocCount(), equalTo(0L));
}
}
public void testSingleValueFieldWithPartitionedFiltering() throws Exception {
runTestFieldWithPartitionedFiltering(SINGLE_VALUED_FIELD_NAME);
}
@ -387,142 +313,6 @@ public class LongTermsIT extends AbstractTermsTestCase {
assertEquals(expectedCardinality, foundTerms.size());
}
public void testSingleValueFieldWithMaxSize() throws Exception {
SearchResponse response = client().prepareSearch("high_card_idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.size(20)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.key(true))) // we need to sort by terms cause we're checking the first 20 values
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(20));
for (int i = 0; i < 20; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
}
}
public void testSingleValueFieldOrderedByTermAsc() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.key(true)))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
i++;
}
}
public void testSingleValueFieldOrderedByTermDesc() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.key(false)))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 4;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
i--;
}
}
public void testSingleValueFieldOrderedByTieBreaker() throws Exception {
SearchResponse response = client().prepareSearch("idx").setTypes("type")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.aggregation("max_constant", randomBoolean()))
.subAggregation(max("max_constant").field("constant")))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
i++;
}
}
public void testSingleValuedFieldWithSubAggregation() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.subAggregation(sum("sum").field(MULTI_VALUED_FIELD_NAME)))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key");
Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count");
Object[] propertiesCounts = (Object[]) ((InternalAggregation)terms).getProperty("sum.value");
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
assertThat(bucket.getDocCount(), equalTo(1L));
Sum sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
assertThat((long) sum.getValue(), equalTo(i+i+1L));
assertThat((long) propertiesKeys[i], equalTo((long) i));
assertThat((long) propertiesDocCounts[i], equalTo(1L));
assertThat((double) propertiesCounts[i], equalTo((double) i + i + 1L));
}
}
public void testSingleValuedFieldWithValueScript() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
@ -548,34 +338,6 @@ public class LongTermsIT extends AbstractTermsTestCase {
}
}
public void testMultiValuedField() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(MULTI_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(6));
for (int i = 0; i < 6; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + i));
assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
if (i == 0 || i == 5) {
assertThat(bucket.getDocCount(), equalTo(1L));
} else {
assertThat(bucket.getDocCount(), equalTo(2L));
}
}
}
public void testMultiValuedFieldWithValueScript() throws Exception {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
@ -704,23 +466,6 @@ public class LongTermsIT extends AbstractTermsTestCase {
}
}
public void testUnmapped() throws Exception {
SearchResponse response = client().prepareSearch("idx_unmapped")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.size(randomIntBetween(1, 5))
.collectMode(randomFrom(SubAggCollectionMode.values())))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(0));
}
public void testPartiallyUnmapped() throws Exception {
SearchResponse response = client().prepareSearch("idx_unmapped", "idx")
.addAggregation(terms("terms")
@ -771,54 +516,6 @@ public class LongTermsIT extends AbstractTermsTestCase {
}
}
public void testEmptyAggregation() throws Exception {
SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1L).minDocCount(0)
.subAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME)))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, Matchers.notNullValue());
Histogram.Bucket bucket = histo.getBuckets().get(1);
assertThat(bucket, Matchers.notNullValue());
Terms terms = bucket.getAggregations().get("terms");
assertThat(terms, Matchers.notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().isEmpty(), is(true));
}
public void testSingleValuedFieldOrderedBySingleValueSubAggregationAsc() throws Exception {
boolean asc = true;
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.aggregation("avg_i", asc))
.subAggregation(avg("avg_i").field(SINGLE_VALUED_FIELD_NAME))
).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
Avg avg = bucket.getAggregations().get("avg_i");
assertThat(avg, notNullValue());
assertThat(avg.getValue(), equalTo((double) i));
}
}
public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscWithTermsSubAgg() throws Exception {
boolean asc = true;
SearchResponse response = client().prepareSearch("idx")
@ -1032,38 +729,6 @@ public class LongTermsIT extends AbstractTermsTestCase {
}
}
public void testSingleValuedFieldOrderedBySingleValueSubAggregationDesc() throws Exception {
boolean asc = false;
SearchResponse response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.aggregation("avg_i", asc))
.subAggregation(avg("avg_i").field(SINGLE_VALUED_FIELD_NAME))
).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
for (int i = 4; i >= 0; i--) {
Terms.Bucket bucket = terms.getBucketByKey("" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
Avg avg = bucket.getAggregations().get("avg_i");
assertThat(avg, notNullValue());
assertThat(avg.getValue(), equalTo((double) i));
}
}
public void testSingleValuedFieldOrderedByMultiValueSubAggregationAsc() throws Exception {
boolean asc = true;
SearchResponse response = client().prepareSearch("idx")

492
core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java
View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.search.aggregations.bucket;
import org.apache.lucene.util.automaton.RegExp;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
@ -34,25 +33,19 @@ import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.aggregations.AggregationExecutionException;
import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.filter.Filter;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode;
import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
import org.elasticsearch.search.aggregations.metrics.avg.Avg;
import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.elasticsearch.search.aggregations.metrics.stats.Stats;
import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.test.ESIntegTestCase;
import org.hamcrest.Matchers;
import java.io.IOException;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -61,20 +54,15 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
import static org.elasticsearch.search.aggregations.AggregationBuilders.count;
import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats;
import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
import static org.elasticsearch.search.aggregations.AggregationBuilders.max;
import static org.elasticsearch.search.aggregations.AggregationBuilders.stats;
import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
@ -82,9 +70,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.core.IsNull.notNullValue;
import static org.hamcrest.core.IsNull.nullValue;
@ESIntegTestCase.SuiteScopeTestCase
public class StringTermsIT extends AbstractTermsTestCase {
@ -255,217 +241,6 @@ public class StringTermsIT extends AbstractTermsTestCase {
assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]"));
}
public void testSingleValueField() throws Exception {
SearchResponse response = client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key");
Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count");
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
assertThat((String) propertiesKeys[i], equalTo("val" + i));
assertThat((long) propertiesDocCounts[i], equalTo(1L));
}
}
public void testSingleValueFieldWithGlobalOrdinals() throws Exception {
ExecutionMode[] executionModes = new ExecutionMode[] { null, ExecutionMode.GLOBAL_ORDINALS, ExecutionMode.GLOBAL_ORDINALS_HASH,
ExecutionMode.GLOBAL_ORDINALS_LOW_CARDINALITY };
for (ExecutionMode executionMode : executionModes) {
logger.info("Execution mode: {}", executionMode);
SearchResponse response = client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(executionMode == null ? null : executionMode.toString())
.field(SINGLE_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values()))).execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
}
}
}
public void testSingleValueFieldWithRegexFiltering() throws Exception {
// include without exclude
// we should be left with: val000, val001, val002, val003, val004, val005, val006, val007, val008, val009
SearchResponse response = client()
.prepareSearch("high_card_idx")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())).includeExclude(new IncludeExclude("val00.+", null)))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(10));
for (int i = 0; i < 10; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val00" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
}
// include and exclude
// we should be left with: val002, val003, val004, val005, val006, val007, val008, val009
response = client()
.prepareSearch("high_card_idx")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.includeExclude(new IncludeExclude("val00.+", "(val000|val001)")))
.execute().actionGet();
assertSearchResponse(response);
terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(8));
for (int i = 2; i < 10; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val00" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
}
// exclude without include
// we should be left with: val000, val001, val002, val003, val004, val005, val006, val007, val008, val009
response = client()
.prepareSearch("high_card_idx")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.includeExclude(new IncludeExclude(null, new RegExp("val0[1-9]+.+"))))
.execute().actionGet();
assertSearchResponse(response);
terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(10));
for (int i = 0; i < 10; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val00" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
}
}
public void testSingleValueFieldWithExactTermFiltering() throws Exception {
// include without exclude
String incVals[] = { "val000", "val001", "val002", "val003", "val004", "val005", "val006", "val007", "val008", "val009" };
SearchResponse response = client()
.prepareSearch("high_card_idx")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())).includeExclude(new IncludeExclude(incVals, null)))
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(incVals.length));
for (String incVal : incVals) {
Terms.Bucket bucket = terms.getBucketByKey(incVal);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo(incVal));
assertThat(bucket.getDocCount(), equalTo(1L));
}
// include and exclude
// Slightly illogical example with exact terms below as include and exclude sets
// are made to overlap but the exclude set should have priority over matches.
// we should be left with: val002, val003, val004, val005, val006, val007, val008, val009
String excVals[] = { "val000", "val001" };
response = client()
.prepareSearch("high_card_idx")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())).includeExclude(new IncludeExclude(incVals, excVals)))
.execute()
.actionGet();
assertSearchResponse(response);
terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(8));
for (int i = 2; i < 10; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val00" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
}
// Check case with only exact term exclude clauses
response = client()
.prepareSearch("high_card_idx")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())).includeExclude(new IncludeExclude(null, excVals)))
.execute().actionGet();
assertSearchResponse(response);
terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(10));
for (String key : excVals) {
Terms.Bucket bucket = terms.getBucketByKey(key);
assertThat(bucket, nullValue());
}
NumberFormat nf = NumberFormat.getIntegerInstance(Locale.ENGLISH);
nf.setMinimumIntegerDigits(3);
for (int i = 2; i < 12; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + nf.format(i));
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + nf.format(i)));
assertThat(bucket.getDocCount(), equalTo(1L));
}
}
public void testSingleValueFieldWithPartitionedFiltering() throws Exception {
runTestFieldWithPartitionedFiltering(SINGLE_VALUED_FIELD_NAME);
}
@ -503,116 +278,6 @@ public class StringTermsIT extends AbstractTermsTestCase {
assertEquals(expectedCardinality, foundTerms.size());
}
public void testSingleValueFieldWithMaxSize() throws Exception {
SearchResponse response = client()
.prepareSearch("high_card_idx")
.addAggregation(
terms("terms")
.executionHint(randomExecutionHint())
.field(SINGLE_VALUED_FIELD_NAME).size(20)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.order(BucketOrder.key(true))) // we need to sort by terms cause we're checking the first 20 values
.execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(20));
for (int i = 0; i < 20; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + Strings.padStart(i + "", 3, '0'));
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + Strings.padStart(i + "", 3, '0')));
assertThat(bucket.getDocCount(), equalTo(1L));
}
}
public void testSingleValueFieldOrderedByTermAsc() throws Exception {
SearchResponse response = client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.key(true))).execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
i++;
}
}
public void testSingleValueFieldOrderedByTermDesc() throws Exception {
SearchResponse response = client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.key(false))).execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 4;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
i--;
}
}
public void testSingleValuedFieldWithSubAggregation() throws Exception {
SearchResponse response = client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.subAggregation(count("count").field(MULTI_VALUED_FIELD_NAME))).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key");
Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count");
Object[] propertiesCounts = (Object[]) ((InternalAggregation)terms).getProperty("count.value");
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
ValueCount valueCount = bucket.getAggregations().get("count");
assertThat(valueCount, notNullValue());
assertThat(valueCount.getValue(), equalTo(2L));
assertThat((String) propertiesKeys[i], equalTo("val" + i));
assertThat((long) propertiesDocCounts[i], equalTo(1L));
assertThat((double) propertiesCounts[i], equalTo(2.0));
}
}
public void testSingleValuedFieldWithValueScript() throws Exception {
SearchResponse response = client()
.prepareSearch("idx")
@ -666,33 +331,6 @@ public class StringTermsIT extends AbstractTermsTestCase {
assertThat(bucket.getDocCount(), equalTo(5L));
}
public void testMultiValuedField() throws Exception {
SearchResponse response = client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(MULTI_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(6));
for (int i = 0; i < 6; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
if (i == 0 || i == 5) {
assertThat(bucket.getDocCount(), equalTo(1L));
} else {
assertThat(bucket.getDocCount(), equalTo(2L));
}
}
}
public void testMultiValuedScript() throws Exception {
SearchResponse response = client()
.prepareSearch("idx")
@ -856,22 +494,6 @@ public class StringTermsIT extends AbstractTermsTestCase {
}
}
public void testUnmapped() throws Exception {
SearchResponse response = client()
.prepareSearch("idx_unmapped")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).size(randomIntBetween(1, 5)).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(0));
}
public void testPartiallyUnmapped() throws Exception {
SearchResponse response = client()
.prepareSearch("idx", "idx_unmapped")
@ -922,87 +544,6 @@ public class StringTermsIT extends AbstractTermsTestCase {
}
}
public void testEmptyAggregation() throws Exception {
SearchResponse searchResponse = client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(1L)
.minDocCount(0)
.subAggregation(terms("terms").field("value")))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, Matchers.notNullValue());
Histogram.Bucket bucket = histo.getBuckets().get(1);
assertThat(bucket, Matchers.notNullValue());
Terms terms = bucket.getAggregations().get("terms");
assertThat(terms, Matchers.notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().isEmpty(), is(true));
}
public void testSingleValuedFieldOrderedBySingleValueSubAggregationAsc() throws Exception {
boolean asc = true;
SearchResponse response = client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.aggregation("avg_i", asc))
.subAggregation(avg("avg_i").field("i"))).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
Avg avg = bucket.getAggregations().get("avg_i");
assertThat(avg, notNullValue());
assertThat(avg.getValue(), equalTo((double) i));
i++;
}
}
public void testSingleValuedFieldOrderedByTieBreaker() throws Exception {
SearchResponse response = client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.aggregation("max_constant", randomBoolean()))
.subAggregation(max("max_constant").field("constant"))).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
Max max = bucket.getAggregations().get("max_constant");
assertThat(max, notNullValue());
assertThat(max.getValue(), equalTo((double) 1));
i++;
}
}
public void testSingleValuedFieldOrderedByIllegalAgg() throws Exception {
boolean asc = true;
try {
@ -1330,37 +871,6 @@ public class StringTermsIT extends AbstractTermsTestCase {
}
}
public void testSingleValuedFieldOrderedBySingleValueSubAggregationDesc() throws Exception {
boolean asc = false;
SearchResponse response = client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
.collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.aggregation("avg_i", asc))
.subAggregation(avg("avg_i").field("i"))).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 4;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
Avg avg = bucket.getAggregations().get("avg_i");
assertThat(avg, notNullValue());
assertThat(avg.getValue(), equalTo((double) i));
i--;
}
}
public void testSingleValuedFieldOrderedByMultiValueSubAggregationAsc() throws Exception {
boolean asc = true;
SearchResponse response = client()

814
core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
View File

@ -19,32 +19,52 @@
package org.elasticsearch.search.aggregations.bucket.terms;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.index.mapper.IpFieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.bucket.filter.Filter;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
import org.elasticsearch.search.aggregations.support.ValueType;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
import java.util.function.Function;
import static org.hamcrest.Matchers.instanceOf;
@ -94,54 +114,754 @@ public class TermsAggregatorTests extends AggregatorTestCase {
directory.close();
}
public void testTermsAggregator() throws Exception {
Directory directory = newDirectory();
RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
Document document = new Document();
document.add(new SortedSetDocValuesField("string", new BytesRef("a")));
document.add(new SortedSetDocValuesField("string", new BytesRef("b")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedSetDocValuesField("string", new BytesRef("c")));
document.add(new SortedSetDocValuesField("string", new BytesRef("a")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedSetDocValuesField("string", new BytesRef("b")));
document.add(new SortedSetDocValuesField("string", new BytesRef("d")));
indexWriter.addDocument(document);
indexWriter.close();
IndexReader indexReader = DirectoryReader.open(directory);
// We do not use LuceneTestCase.newSearcher because we need a DirectoryReader
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
for (TermsAggregatorFactory.ExecutionMode executionMode : TermsAggregatorFactory.ExecutionMode.values()) {
TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
.executionHint(executionMode.toString())
.field("string")
.order(BucketOrder.key(true));
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
fieldType.setName("string");
fieldType.setHasDocValues(true );
TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
Terms result = (Terms) aggregator.buildAggregation(0L);
assertEquals(4, result.getBuckets().size());
assertEquals("a", result.getBuckets().get(0).getKeyAsString());
assertEquals(2L, result.getBuckets().get(0).getDocCount());
assertEquals("b", result.getBuckets().get(1).getKeyAsString());
assertEquals(2L, result.getBuckets().get(1).getDocCount());
assertEquals("c", result.getBuckets().get(2).getKeyAsString());
assertEquals(1L, result.getBuckets().get(2).getDocCount());
assertEquals("d", result.getBuckets().get(3).getKeyAsString());
assertEquals(1L, result.getBuckets().get(3).getDocCount());
public void testSimple() throws Exception {
try (Directory directory = newDirectory()) {
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
Document document = new Document();
document.add(new SortedSetDocValuesField("string", new BytesRef("a")));
document.add(new SortedSetDocValuesField("string", new BytesRef("b")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedSetDocValuesField("string", new BytesRef("c")));
document.add(new SortedSetDocValuesField("string", new BytesRef("a")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedSetDocValuesField("string", new BytesRef("b")));
document.add(new SortedSetDocValuesField("string", new BytesRef("d")));
indexWriter.addDocument(document);
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
for (TermsAggregatorFactory.ExecutionMode executionMode : TermsAggregatorFactory.ExecutionMode.values()) {
TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
.executionHint(executionMode.toString())
.field("string")
.order(BucketOrder.key(true));
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
fieldType.setName("string");
fieldType.setHasDocValues(true );
TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
Terms result = (Terms) aggregator.buildAggregation(0L);
assertEquals(4, result.getBuckets().size());
assertEquals("a", result.getBuckets().get(0).getKeyAsString());
assertEquals(2L, result.getBuckets().get(0).getDocCount());
assertEquals("b", result.getBuckets().get(1).getKeyAsString());
assertEquals(2L, result.getBuckets().get(1).getDocCount());
assertEquals("c", result.getBuckets().get(2).getKeyAsString());
assertEquals(1L, result.getBuckets().get(2).getDocCount());
assertEquals("d", result.getBuckets().get(3).getKeyAsString());
assertEquals(1L, result.getBuckets().get(3).getDocCount());
}
}
}
}
}
public void testStringIncludeExclude() throws Exception {
try (Directory directory = newDirectory()) {
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
Document document = new Document();
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val000")));
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val001")));
document.add(new SortedDocValuesField("sv_field", new BytesRef("val001")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val002")));
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val003")));
document.add(new SortedDocValuesField("sv_field", new BytesRef("val003")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val004")));
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val005")));
document.add(new SortedDocValuesField("sv_field", new BytesRef("val005")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val006")));
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val007")));
document.add(new SortedDocValuesField("sv_field", new BytesRef("val007")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val008")));
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val009")));
document.add(new SortedDocValuesField("sv_field", new BytesRef("val009")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val010")));
document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val011")));
document.add(new SortedDocValuesField("sv_field", new BytesRef("val011")));
indexWriter.addDocument(document);
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
fieldType.setName("mv_field");
fieldType.setHasDocValues(true);
String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString();
TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
.executionHint(executionHint)
.includeExclude(new IncludeExclude("val00.+", null))
.field("mv_field")
.size(12)
.order(BucketOrder.key(true));
Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
Terms result = (Terms) aggregator.buildAggregation(0L);
assertEquals(10, result.getBuckets().size());
assertEquals("val000", result.getBuckets().get(0).getKeyAsString());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
assertEquals("val001", result.getBuckets().get(1).getKeyAsString());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
assertEquals("val002", result.getBuckets().get(2).getKeyAsString());
assertEquals(1L, result.getBuckets().get(2).getDocCount());
assertEquals("val003", result.getBuckets().get(3).getKeyAsString());
assertEquals(1L, result.getBuckets().get(3).getDocCount());
assertEquals("val004", result.getBuckets().get(4).getKeyAsString());
assertEquals(1L, result.getBuckets().get(4).getDocCount());
assertEquals("val005", result.getBuckets().get(5).getKeyAsString());
assertEquals(1L, result.getBuckets().get(5).getDocCount());
assertEquals("val006", result.getBuckets().get(6).getKeyAsString());
assertEquals(1L, result.getBuckets().get(6).getDocCount());
assertEquals("val007", result.getBuckets().get(7).getKeyAsString());
assertEquals(1L, result.getBuckets().get(7).getDocCount());
assertEquals("val008", result.getBuckets().get(8).getKeyAsString());
assertEquals(1L, result.getBuckets().get(8).getDocCount());
assertEquals("val009", result.getBuckets().get(9).getKeyAsString());
assertEquals(1L, result.getBuckets().get(9).getDocCount());
MappedFieldType fieldType2 = new KeywordFieldMapper.KeywordFieldType();
fieldType2.setName("sv_field");
fieldType2.setHasDocValues(true);
aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
.executionHint(executionHint)
.includeExclude(new IncludeExclude("val00.+", null))
.field("sv_field")
.order(BucketOrder.key(true));
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType2);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = (Terms) aggregator.buildAggregation(0L);
assertEquals(5, result.getBuckets().size());
assertEquals("val001", result.getBuckets().get(0).getKeyAsString());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
assertEquals("val003", result.getBuckets().get(1).getKeyAsString());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
assertEquals("val005", result.getBuckets().get(2).getKeyAsString());
assertEquals(1L, result.getBuckets().get(2).getDocCount());
assertEquals("val007", result.getBuckets().get(3).getKeyAsString());
assertEquals(1L, result.getBuckets().get(3).getDocCount());
assertEquals("val009", result.getBuckets().get(4).getKeyAsString());
assertEquals(1L, result.getBuckets().get(4).getDocCount());
aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
.executionHint(executionHint)
.includeExclude(new IncludeExclude("val00.+", "(val000|val001)"))
.field("mv_field")
.order(BucketOrder.key(true));
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = (Terms) aggregator.buildAggregation(0L);
assertEquals(8, result.getBuckets().size());
assertEquals("val002", result.getBuckets().get(0).getKeyAsString());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
assertEquals("val003", result.getBuckets().get(1).getKeyAsString());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
assertEquals("val004", result.getBuckets().get(2).getKeyAsString());
assertEquals(1L, result.getBuckets().get(2).getDocCount());
assertEquals("val005", result.getBuckets().get(3).getKeyAsString());
assertEquals(1L, result.getBuckets().get(3).getDocCount());
assertEquals("val006", result.getBuckets().get(4).getKeyAsString());
assertEquals(1L, result.getBuckets().get(4).getDocCount());
assertEquals("val007", result.getBuckets().get(5).getKeyAsString());
assertEquals(1L, result.getBuckets().get(5).getDocCount());
assertEquals("val008", result.getBuckets().get(6).getKeyAsString());
assertEquals(1L, result.getBuckets().get(6).getDocCount());
assertEquals("val009", result.getBuckets().get(7).getKeyAsString());
assertEquals(1L, result.getBuckets().get(7).getDocCount());
aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
.executionHint(executionHint)
.includeExclude(new IncludeExclude(null, "val00.+"))
.field("mv_field")
.order(BucketOrder.key(true));
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = (Terms) aggregator.buildAggregation(0L);
assertEquals(2, result.getBuckets().size());
assertEquals("val010", result.getBuckets().get(0).getKeyAsString());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
assertEquals("val011", result.getBuckets().get(1).getKeyAsString());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
.executionHint(executionHint)
.includeExclude(new IncludeExclude(new String[]{"val000", "val010"}, null))
.field("mv_field")
.order(BucketOrder.key(true));
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = (Terms) aggregator.buildAggregation(0L);
assertEquals(2, result.getBuckets().size());
assertEquals("val000", result.getBuckets().get(0).getKeyAsString());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
assertEquals("val010", result.getBuckets().get(1).getKeyAsString());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
.executionHint(executionHint)
.includeExclude(new IncludeExclude(null, new String[]{"val001", "val002", "val003", "val004",
"val005", "val006", "val007", "val008", "val009", "val011"}))
.field("mv_field")
.order(BucketOrder.key(true));
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = (Terms) aggregator.buildAggregation(0L);
assertEquals(2, result.getBuckets().size());
assertEquals("val000", result.getBuckets().get(0).getKeyAsString());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
assertEquals("val010", result.getBuckets().get(1).getKeyAsString());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
}
}
}
}
public void testNumericIncludeExclude() throws Exception {
try (Directory directory = newDirectory()) {
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
Document document = new Document();
document.add(new NumericDocValuesField("long_field", 0));
document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(0.0)));
indexWriter.addDocument(document);
document = new Document();
document.add(new NumericDocValuesField("long_field", 1));
document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(1.0)));
indexWriter.addDocument(document);
document = new Document();
document.add(new NumericDocValuesField("long_field", 2));
document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(2.0)));
indexWriter.addDocument(document);
document = new Document();
document.add(new NumericDocValuesField("long_field", 3));
document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(3.0)));
indexWriter.addDocument(document);
document = new Document();
document.add(new NumericDocValuesField("long_field", 4));
document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(4.0)));
indexWriter.addDocument(document);
document = new Document();
document.add(new NumericDocValuesField("long_field", 5));
document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(5.0)));
indexWriter.addDocument(document);
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
fieldType.setName("long_field");
fieldType.setHasDocValues(true );
String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString();
TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG)
.executionHint(executionHint)
.includeExclude(new IncludeExclude(new long[]{0, 5}, null))
.field("long_field")
.order(BucketOrder.key(true));
Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
Terms result = (Terms) aggregator.buildAggregation(0L);
assertEquals(2, result.getBuckets().size());
assertEquals(0L, result.getBuckets().get(0).getKey());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
assertEquals(5L, result.getBuckets().get(1).getKey());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG)
.executionHint(executionHint)
.includeExclude(new IncludeExclude(null, new long[]{0, 5}))
.field("long_field")
.order(BucketOrder.key(true));
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = (Terms) aggregator.buildAggregation(0L);
assertEquals(4, result.getBuckets().size());
assertEquals(1L, result.getBuckets().get(0).getKey());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
assertEquals(2L, result.getBuckets().get(1).getKey());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
assertEquals(3L, result.getBuckets().get(2).getKey());
assertEquals(1L, result.getBuckets().get(2).getDocCount());
assertEquals(4L, result.getBuckets().get(3).getKey());
assertEquals(1L, result.getBuckets().get(3).getDocCount());
fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
fieldType.setName("double_field");
fieldType.setHasDocValues(true );
aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.DOUBLE)
.executionHint(executionHint)
.includeExclude(new IncludeExclude(new double[]{0.0, 5.0}, null))
.field("double_field")
.order(BucketOrder.key(true));
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = (Terms) aggregator.buildAggregation(0L);
assertEquals(2, result.getBuckets().size());
assertEquals(0.0, result.getBuckets().get(0).getKey());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
assertEquals(5.0, result.getBuckets().get(1).getKey());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.DOUBLE)
.executionHint(executionHint)
.includeExclude(new IncludeExclude(null, new double[]{0.0, 5.0}))
.field("double_field")
.order(BucketOrder.key(true));
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = (Terms) aggregator.buildAggregation(0L);
assertEquals(4, result.getBuckets().size());
assertEquals(1.0, result.getBuckets().get(0).getKey());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
assertEquals(2.0, result.getBuckets().get(1).getKey());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
assertEquals(3.0, result.getBuckets().get(2).getKey());
assertEquals(1L, result.getBuckets().get(2).getDocCount());
assertEquals(4.0, result.getBuckets().get(3).getKey());
assertEquals(1L, result.getBuckets().get(3).getDocCount());
}
}
}
}
public void testStringTermsAggregator() throws Exception {
BiFunction<String, Boolean, IndexableField> luceneFieldFactory = (val, mv) -> {
if (mv) {
return new SortedSetDocValuesField("field", new BytesRef(val));
} else {
return new SortedDocValuesField("field", new BytesRef(val));
}
};
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
termsAggregator(ValueType.STRING, fieldType, i -> Integer.toString(i),
String::compareTo, luceneFieldFactory);
termsAggregatorWithNestedMaxAgg(ValueType.STRING, fieldType, i -> Integer.toString(i),
val -> new SortedDocValuesField("field", new BytesRef(val)));
}
public void testLongTermsAggregator() throws Exception {
BiFunction<Long, Boolean, IndexableField> luceneFieldFactory = (val, mv) -> {
if (mv) {
return new SortedNumericDocValuesField("field", val);
} else {
return new NumericDocValuesField("field", val);
}
};
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
termsAggregator(ValueType.LONG, fieldType, Integer::longValue, Long::compareTo, luceneFieldFactory);
termsAggregatorWithNestedMaxAgg(ValueType.LONG, fieldType, Integer::longValue, val -> new NumericDocValuesField("field", val));
}
public void testDoubleTermsAggregator() throws Exception {
BiFunction<Double, Boolean, IndexableField> luceneFieldFactory = (val, mv) -> {
if (mv) {
return new SortedNumericDocValuesField("field", Double.doubleToRawLongBits(val));
} else {
return new NumericDocValuesField("field", Double.doubleToRawLongBits(val));
}
};
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
termsAggregator(ValueType.DOUBLE, fieldType, Integer::doubleValue, Double::compareTo, luceneFieldFactory);
termsAggregatorWithNestedMaxAgg(ValueType.DOUBLE, fieldType, Integer::doubleValue,
val -> new NumericDocValuesField("field", Double.doubleToRawLongBits(val)));
}
public void testIpTermsAggregator() throws Exception {
BiFunction<InetAddress, Boolean, IndexableField> luceneFieldFactory = (val, mv) -> {
if (mv) {
return new SortedSetDocValuesField("field", new BytesRef(InetAddressPoint.encode(val)));
} else {
return new SortedDocValuesField("field", new BytesRef(InetAddressPoint.encode(val)));
}
};
InetAddress[] base = new InetAddress[] {InetAddresses.forString("192.168.0.0")};
Comparator<InetAddress> comparator = (o1, o2) -> {
BytesRef b1 = new BytesRef(InetAddressPoint.encode(o1));
BytesRef b2 = new BytesRef(InetAddressPoint.encode(o2));
return b1.compareTo(b2);
};
termsAggregator(ValueType.IP, new IpFieldMapper.IpFieldType(), i -> {return base[0] = InetAddressPoint.nextUp(base[0]);},
comparator, luceneFieldFactory);
}
private <T> void termsAggregator(ValueType valueType, MappedFieldType fieldType,
Function<Integer, T> valueFactory, Comparator<T> keyComparator,
BiFunction<T, Boolean, IndexableField> luceneFieldFactory) throws Exception {
final Map<T, Integer> counts = new HashMap<>();
final Map<T, Integer> filteredCounts = new HashMap<>();
int numTerms = scaledRandomIntBetween(8, 128);
for (int i = 0; i < numTerms; i++) {
int numDocs = scaledRandomIntBetween(2, 32);
T key = valueFactory.apply(i);
counts.put(key, numDocs);
filteredCounts.put(key, 0);
}
try (Directory directory = newDirectory()) {
boolean multiValued = randomBoolean();
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
if (multiValued == false) {
for (Map.Entry<T, Integer> entry : counts.entrySet()) {
for (int i = 0; i < entry.getValue(); i++) {
Document document = new Document();
document.add(luceneFieldFactory.apply(entry.getKey(), false));
if (randomBoolean()) {
document.add(new StringField("include", "yes", Field.Store.NO));
filteredCounts.computeIfPresent(entry.getKey(), (key, integer) -> integer + 1);
}
indexWriter.addDocument(document);
}
}
} else {
Iterator<Map.Entry<T, Integer>> iterator = counts.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<T, Integer> entry1 = iterator.next();
Map.Entry<T, Integer> entry2 = null;
if (randomBoolean() && iterator.hasNext()) {
entry2 = iterator.next();
if (entry1.getValue().compareTo(entry2.getValue()) < 0) {
Map.Entry<T, Integer> temp = entry1;
entry1 = entry2;
entry2 = temp;
}
}
for (int i = 0; i < entry1.getValue(); i++) {
Document document = new Document();
document.add(luceneFieldFactory.apply(entry1.getKey(), true));
if (entry2 != null && i < entry2.getValue()) {
document.add(luceneFieldFactory.apply(entry2.getKey(), true));
}
indexWriter.addDocument(document);
}
}
}
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
boolean order = randomBoolean();
List<Map.Entry<T, Integer>> expectedBuckets = new ArrayList<>();
expectedBuckets.addAll(counts.entrySet());
BucketOrder bucketOrder;
Comparator<Map.Entry<T, Integer>> comparator;
if (randomBoolean()) {
bucketOrder = BucketOrder.key(order);
comparator = Comparator.comparing(Map.Entry::getKey, keyComparator);
} else {
// if order by count then we need to use compound so that we can also sort by key as tie breaker:
bucketOrder = BucketOrder.compound(BucketOrder.count(order), BucketOrder.key(order));
comparator = Comparator.comparing(Map.Entry::getValue);
comparator = comparator.thenComparing(Comparator.comparing(Map.Entry::getKey, keyComparator));
}
if (order == false) {
comparator = comparator.reversed();
}
expectedBuckets.sort(comparator);
int size = randomIntBetween(1, counts.size());
String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString();
logger.info("bucket_order={} size={} execution_hint={}", bucketOrder, size, executionHint);
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueType)
.executionHint(executionHint)
.size(size)
.shardSize(size)
.field("field")
.order(bucketOrder);
fieldType.setName("field");
fieldType.setHasDocValues(true);
Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
Terms result = (Terms) aggregator.buildAggregation(0L);
assertEquals(size, result.getBuckets().size());
for (int i = 0; i < size; i++) {
Map.Entry<T, Integer> expected = expectedBuckets.get(i);
Terms.Bucket actual = result.getBuckets().get(i);
if (valueType == ValueType.IP) {
assertEquals(String.valueOf(expected.getKey()).substring(1), actual.getKey());
} else {
assertEquals(expected.getKey(), actual.getKey());
}
assertEquals(expected.getValue().longValue(), actual.getDocCount());
}
if (multiValued == false) {
aggregationBuilder = new FilterAggregationBuilder("_name1", QueryBuilders.termQuery("include", "yes"));
aggregationBuilder.subAggregation(new TermsAggregationBuilder("_name2", valueType)
.executionHint(executionHint)
.size(numTerms)
.collectMode(randomFrom(Aggregator.SubAggCollectionMode.values()))
.field("field"));
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = ((Filter) aggregator.buildAggregation(0L)).getAggregations().get("_name2");
int expectedFilteredCounts = 0;
for (Integer count : filteredCounts.values()) {
if (count > 0) {
expectedFilteredCounts++;
}
}
assertEquals(expectedFilteredCounts, result.getBuckets().size());
for (Terms.Bucket actual : result.getBuckets()) {
Integer expectedCount;
if (valueType == ValueType.IP) {
expectedCount = filteredCounts.get(InetAddresses.forString((String)actual.getKey()));
} else {
expectedCount = filteredCounts.get(actual.getKey());
}
assertEquals(expectedCount.longValue(), actual.getDocCount());
}
}
}
}
}
}
private <T> void termsAggregatorWithNestedMaxAgg(ValueType valueType, MappedFieldType fieldType,
Function<Integer, T> valueFactory,
Function<T, IndexableField> luceneFieldFactory) throws Exception {
final Map<T, Long> counts = new HashMap<>();
int numTerms = scaledRandomIntBetween(8, 128);
for (int i = 0; i < numTerms; i++) {
counts.put(valueFactory.apply(i), randomLong());
}
try (Directory directory = newDirectory()) {
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
for (Map.Entry<T, Long> entry : counts.entrySet()) {
Document document = new Document();
document.add(luceneFieldFactory.apply(entry.getKey()));
document.add(new NumericDocValuesField("value", entry.getValue()));
indexWriter.addDocument(document);
}
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
boolean order = randomBoolean();
List<Map.Entry<T, Long>> expectedBuckets = new ArrayList<>();
expectedBuckets.addAll(counts.entrySet());
BucketOrder bucketOrder = BucketOrder.aggregation("_max", order);
Comparator<Map.Entry<T, Long>> comparator = Comparator.comparing(Map.Entry::getValue, Long::compareTo);
if (order == false) {
comparator = comparator.reversed();
}
expectedBuckets.sort(comparator);
int size = randomIntBetween(1, counts.size());
String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString();
Aggregator.SubAggCollectionMode collectionMode = randomFrom(Aggregator.SubAggCollectionMode.values());
logger.info("bucket_order={} size={} execution_hint={}, collect_mode={}",
bucketOrder, size, executionHint, collectionMode);
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueType)
.executionHint(executionHint)
.collectMode(collectionMode)
.size(size)
.shardSize(size)
.field("field")
.order(bucketOrder)
.subAggregation(AggregationBuilders.max("_max").field("value"));
fieldType.setName("field");
fieldType.setHasDocValues(true);
MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
fieldType2.setName("value");
fieldType2.setHasDocValues(true);
Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType, fieldType2);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
Terms result = (Terms) aggregator.buildAggregation(0L);
assertEquals(size, result.getBuckets().size());
for (int i = 0; i < size; i++) {
Map.Entry<T, Long> expected = expectedBuckets.get(i);
Terms.Bucket actual = result.getBuckets().get(i);
assertEquals(expected.getKey(), actual.getKey());
}
}
}
}
}
public void testEmpty() throws Exception {
try (Directory directory = newDirectory()) {
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType();
fieldType1.setName("string");
fieldType1.setHasDocValues(true);
MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
fieldType1.setName("long");
fieldType1.setHasDocValues(true);
MappedFieldType fieldType3 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
fieldType1.setName("double");
fieldType1.setHasDocValues(true);
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
.field("string");
Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
Terms result = (Terms) aggregator.buildAggregation(0L);
assertEquals("_name", result.getName());
assertEquals(0, result.getBuckets().size());
aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG)
.field("long");
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType2);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = (Terms) aggregator.buildAggregation(0L);
assertEquals("_name", result.getName());
assertEquals(0, result.getBuckets().size());
aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.DOUBLE)
.field("double");
aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType3);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
result = (Terms) aggregator.buildAggregation(0L);
assertEquals("_name", result.getName());
assertEquals(0, result.getBuckets().size());
}
}
}
}
public void testUnmapped() throws Exception {
try (Directory directory = newDirectory()) {
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
Document document = new Document();
document.add(new SortedDocValuesField("string", new BytesRef("a")));
document.add(new NumericDocValuesField("long", 0L));
document.add(new NumericDocValuesField("double", Double.doubleToRawLongBits(0L)));
indexWriter.addDocument(document);
MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType();
fieldType1.setName("another_string");
fieldType1.setHasDocValues(true);
MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
fieldType1.setName("another_long");
fieldType1.setHasDocValues(true);
MappedFieldType fieldType3 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
fieldType1.setName("another_double");
fieldType1.setHasDocValues(true);
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
ValueType[] valueTypes = new ValueType[]{ValueType.STRING, ValueType.LONG, ValueType.DOUBLE};
String[] fieldNames = new String[]{"string", "long", "double"};
for (int i = 0; i < fieldNames.length; i++) {
TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueTypes[i])
.field(fieldNames[i]);
Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1, fieldType2, fieldType3);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
Terms result = (Terms) aggregator.buildAggregation(0L);
assertEquals("_name", result.getName());
assertEquals(0, result.getBuckets().size());
}
}
}
}
}
public void testNestedTermsAgg() throws Exception {
try (Directory directory = newDirectory()) {
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
Document document = new Document();
document.add(new SortedDocValuesField("field1", new BytesRef("a")));
document.add(new SortedDocValuesField("field2", new BytesRef("b")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedDocValuesField("field1", new BytesRef("c")));
document.add(new SortedDocValuesField("field2", new BytesRef("d")));
indexWriter.addDocument(document);
document = new Document();
document.add(new SortedDocValuesField("field1", new BytesRef("e")));
document.add(new SortedDocValuesField("field2", new BytesRef("f")));
indexWriter.addDocument(document);
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString();
Aggregator.SubAggCollectionMode collectionMode = randomFrom(Aggregator.SubAggCollectionMode.values());
TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name1", ValueType.STRING)
.executionHint(executionHint)
.collectMode(collectionMode)
.field("field1")
.order(BucketOrder.key(true))
.subAggregation(new TermsAggregationBuilder("_name2", ValueType.STRING)
.executionHint(executionHint)
.collectMode(collectionMode)
.field("field2")
.order(BucketOrder.key(true))
);
MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType();
fieldType1.setName("field1");
fieldType1.setHasDocValues(true);
MappedFieldType fieldType2 = new KeywordFieldMapper.KeywordFieldType();
fieldType2.setName("field2");
fieldType2.setHasDocValues(true);
Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1, fieldType2);
aggregator.preCollection();
indexSearcher.search(new MatchAllDocsQuery(), aggregator);
aggregator.postCollection();
Terms result = (Terms) aggregator.buildAggregation(0L);
assertEquals(3, result.getBuckets().size());
assertEquals("a", result.getBuckets().get(0).getKeyAsString());
assertEquals(1L, result.getBuckets().get(0).getDocCount());
Terms.Bucket nestedBucket = ((Terms) result.getBuckets().get(0).getAggregations().get("_name2")).getBuckets().get(0);
assertEquals("b", nestedBucket.getKeyAsString());
assertEquals("c", result.getBuckets().get(1).getKeyAsString());
assertEquals(1L, result.getBuckets().get(1).getDocCount());
nestedBucket = ((Terms) result.getBuckets().get(1).getAggregations().get("_name2")).getBuckets().get(0);
assertEquals("d", nestedBucket.getKeyAsString());
assertEquals("e", result.getBuckets().get(2).getKeyAsString());
assertEquals(1L, result.getBuckets().get(2).getDocCount());
nestedBucket = ((Terms) result.getBuckets().get(2).getAggregations().get("_name2")).getBuckets().get(0);
assertEquals("f", nestedBucket.getKeyAsString());
}
}
}
indexReader.close();
directory.close();
}
public void testMixLongAndDouble() throws Exception {

29
test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
View File

@ -18,10 +18,13 @@
*/
package org.elasticsearch.search.aggregations;
import org.apache.lucene.index.AssertingDirectoryReader;
import org.apache.lucene.index.CompositeReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.AssertingIndexSearcher;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@ -298,6 +301,32 @@ public abstract class AggregatorTestCase extends ESTestCase {
return ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(new Index("_index", "_na_"), 0));
}
/**
* Added to randomly run with more assertions on the index searcher level,
* like {@link org.apache.lucene.util.LuceneTestCase#newSearcher(IndexReader)}, which can't be used because it also
* wraps in the IndexSearcher's IndexReader with other implementations that we can't handle. (e.g. ParallelCompositeReader)
*/
protected static IndexSearcher newIndexSearcher(IndexReader indexReader) {
if (randomBoolean()) {
return new AssertingIndexSearcher(random(), indexReader);
} else {
return new IndexSearcher(indexReader);
}
}
/**
* Added to randomly run with more assertions on the index reader level,
* like {@link org.apache.lucene.util.LuceneTestCase#wrapReader(IndexReader)}, which can't be used because it also
* wraps in the IndexReader with other implementations that we can't handle. (e.g. ParallelCompositeReader)
*/
protected static IndexReader maybeWrapReaderEs(DirectoryReader reader) throws IOException {
if (randomBoolean()) {
return new AssertingDirectoryReader(reader);
} else {
return reader;
}
}
@After
private void cleanupReleasables() {
Releasables.close(releasables);