diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java deleted file mode 100644 index 5e56123261c..00000000000 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.aggregations.bucket; - -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.missing.Missing; -import org.elasticsearch.search.aggregations.metrics.Avg; -import org.elasticsearch.test.ESIntegTestCase; -import org.hamcrest.Matchers; - -import java.util.ArrayList; -import java.util.List; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; -import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; -import static org.elasticsearch.search.aggregations.AggregationBuilders.missing; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.core.IsNull.notNullValue; - -@ESIntegTestCase.SuiteScopeTestCase -public class MissingIT extends ESIntegTestCase { - - static int numDocs, numDocsMissing, numDocsUnmapped; - - @Override - public void setupSuiteScopeCluster() throws Exception { - assertAcked(client().admin().indices().prepareCreate("idx") - .addMapping("type", "tag", "type=keyword").get()); - List builders = new ArrayList<>(); - numDocs = randomIntBetween(5, 20); - numDocsMissing = randomIntBetween(1, numDocs - 1); - for (int i = 0; i < numDocsMissing; i++) { - builders.add(client().prepareIndex("idx", "type", ""+i).setSource(jsonBuilder() - .startObject() - .field("value", i) - .endObject())); - } - for (int i = numDocsMissing; i < numDocs; i++) { - builders.add(client().prepareIndex("idx", "type", ""+i).setSource(jsonBuilder() - .startObject() - .field("tag", "tag1") - .endObject())); - } - - createIndex("unmapped_idx"); - numDocsUnmapped = randomIntBetween(2, 5); - for (int i = 0; i < numDocsUnmapped; i++) { - builders.add(client().prepareIndex("unmapped_idx", "type", ""+i).setSource(jsonBuilder() - .startObject() - .field("value", i) - .endObject())); - } - - prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer").get(); - for (int i = 0; i < 2; i++) { - builders.add(client().prepareIndex("empty_bucket_idx", "type", ""+i).setSource(jsonBuilder() - .startObject() - .field("value", i*2) - .endObject())); - } - - indexRandom(true, builders); - ensureSearchable(); - } - - public void testUnmapped() throws Exception { - SearchResponse response = client().prepareSearch("unmapped_idx") - .addAggregation(missing("missing_tag").field("tag")) - .get(); - - assertSearchResponse(response); - - - Missing missing = response.getAggregations().get("missing_tag"); - assertThat(missing, notNullValue()); - assertThat(missing.getName(), equalTo("missing_tag")); - assertThat(missing.getDocCount(), equalTo((long) numDocsUnmapped)); - } - - public void testPartiallyUnmapped() throws Exception { - SearchResponse response = client().prepareSearch("idx", "unmapped_idx") - .addAggregation(missing("missing_tag").field("tag")) - .get(); - - assertSearchResponse(response); - - - Missing missing = response.getAggregations().get("missing_tag"); - assertThat(missing, notNullValue()); - assertThat(missing.getName(), equalTo("missing_tag")); - assertThat(missing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped)); - } - - public void testSimple() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(missing("missing_tag").field("tag")) - .get(); - - assertSearchResponse(response); - - - Missing missing = response.getAggregations().get("missing_tag"); - assertThat(missing, notNullValue()); - assertThat(missing.getName(), equalTo("missing_tag")); - assertThat(missing.getDocCount(), equalTo((long) numDocsMissing)); - } - - public void testWithSubAggregation() throws Exception { - SearchResponse response = client().prepareSearch("idx", "unmapped_idx") - .addAggregation(missing("missing_tag").field("tag") - .subAggregation(avg("avg_value").field("value"))) - .get(); - - assertSearchResponse(response); - - assertThat("Not all shards are initialized", response.getSuccessfulShards(), equalTo(response.getTotalShards())); - - Missing missing = response.getAggregations().get("missing_tag"); - assertThat(missing, notNullValue()); - assertThat(missing.getName(), equalTo("missing_tag")); - assertThat(missing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped)); - assertThat((long) ((InternalAggregation)missing).getProperty("_count"), equalTo((long) numDocsMissing + numDocsUnmapped)); - assertThat(missing.getAggregations().asList().isEmpty(), is(false)); - - long sum = 0; - for (int i = 0; i < numDocsMissing; ++i) { - sum += i; - } - for (int i = 0; i < numDocsUnmapped; ++i) { - sum += i; - } - Avg avgValue = missing.getAggregations().get("avg_value"); - assertThat(avgValue, notNullValue()); - assertThat(avgValue.getName(), equalTo("avg_value")); - assertThat(avgValue.getValue(), equalTo((double) sum / (numDocsMissing + numDocsUnmapped))); - assertThat((double) ((InternalAggregation)missing).getProperty("avg_value.value"), - equalTo((double) sum / (numDocsMissing + numDocsUnmapped))); - } - - public void testEmptyAggregation() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0) - .subAggregation(missing("missing").field("value"))) - .get(); - - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L)); - Histogram histo = searchResponse.getAggregations().get("histo"); - assertThat(histo, Matchers.notNullValue()); - Histogram.Bucket bucket = histo.getBuckets().get(1); - assertThat(bucket, Matchers.notNullValue()); - - Missing missing = bucket.getAggregations().get("missing"); - assertThat(missing, Matchers.notNullValue()); - assertThat(missing.getName(), equalTo("missing")); - assertThat(missing.getDocCount(), is(0L)); - } - - -} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index 5a3eb5878aa..be9c4ab2042 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -20,10 +20,12 @@ package org.elasticsearch.search.aggregations.bucket.missing; import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -31,6 +33,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; @@ -44,6 +47,11 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram; +import org.elasticsearch.search.aggregations.metrics.InternalSum; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -53,6 +61,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -64,6 +73,10 @@ import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; import static java.util.stream.Collectors.toList; import static org.elasticsearch.common.lucene.search.Queries.newMatchAllQuery; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.collection.IsEmptyCollection.empty; public class MissingAggregatorTests extends AggregatorTestCase { @@ -91,9 +104,9 @@ public class MissingAggregatorTests extends AggregatorTestCase { writer.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong()))); } }, - internalMissing -> { - assertEquals(0, internalMissing.getDocCount()); - assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); + (InternalMissing missing) -> { + assertEquals(0, missing.getDocCount()); + assertFalse(AggregationInspectionHelper.hasValue(missing)); }, singleton(fieldType) ); @@ -118,9 +131,9 @@ public class MissingAggregatorTests extends AggregatorTestCase { writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong()))); } }, - internalMissing -> { - assertEquals(numDocs, internalMissing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + (InternalMissing missing) -> { + assertEquals(numDocs, missing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(missing)); }, org.elasticsearch.common.collect.List.of(aggFieldType, anotherFieldType) ); @@ -152,9 +165,9 @@ public class MissingAggregatorTests extends AggregatorTestCase { newMatchAllQuery(), builder, writer -> writer.addDocuments(docs), - internalMissing -> { - assertEquals(finalDocsMissingAggField, internalMissing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + (InternalMissing missing) -> { + assertEquals(finalDocsMissingAggField, missing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(missing)); }, org.elasticsearch.common.collect.List.of(aggFieldType, anotherFieldType) ); @@ -191,14 +204,113 @@ public class MissingAggregatorTests extends AggregatorTestCase { newMatchAllQuery(), builder, writer -> writer.addDocuments(docs), - internalMissing -> { - assertEquals(finalDocsMissingAggField, internalMissing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + (InternalMissing missing) -> { + assertEquals(finalDocsMissingAggField, missing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(missing)); }, org.elasticsearch.common.collect.List.of(aggFieldType, anotherFieldType) ); } + public void testSubAggregation() throws IOException { + final MappedFieldType missingAggFieldType = new KeywordFieldMapper.Builder("_name").fieldType(); + missingAggFieldType.setName("missing_agg_field"); + missingAggFieldType.setHasDocValues(true); + final MappedFieldType sumAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + sumAggFieldType.setName("sum_agg_field"); + + final MissingAggregationBuilder builder = new MissingAggregationBuilder("missing", null) + .field(missingAggFieldType.name()) + .subAggregation( + new SumAggregationBuilder("sum") + .field(sumAggFieldType.name()) + ); + + final int numDocs = randomIntBetween(10, 20); + int docsMissingAggField = 0; + long expectedSum = 0; + final List> docs = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + final Set doc = new HashSet<>(); + final long sumFieldValue = randomLongBetween(0, 1000); + doc.add(new SortedNumericDocValuesField(sumAggFieldType.name(), sumFieldValue)); + if (randomBoolean()) { + doc.add(new SortedDocValuesField(missingAggFieldType.name(), new BytesRef(randomUnicodeOfLengthBetween(2, 20)))); + } else { + docsMissingAggField++; + expectedSum += sumFieldValue; + } + docs.add(doc); + } + final int finalDocsMissingAggField = docsMissingAggField; + final long finalExpectedSum = expectedSum; + + testCase( + newMatchAllQuery(), + builder, + writer -> writer.addDocuments(docs), + (InternalMissing internalMissing) -> { + assertEquals(finalDocsMissingAggField, internalMissing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + + assertThat(internalMissing.getAggregations().asList(), not(empty())); + final InternalSum internalSum = internalMissing.getAggregations().get("sum"); + assertEquals(finalExpectedSum, internalSum.getValue(), 0d); + assertTrue(AggregationInspectionHelper.hasValue(internalSum)); + }, + org.elasticsearch.common.collect.List.of(missingAggFieldType, sumAggFieldType) + ); + } + + public void testEmptyBucket() throws IOException { + final MappedFieldType histoAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + histoAggFieldType.setName("histo_agg_field"); + final MappedFieldType missingAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + missingAggFieldType.setName("missing_agg_field"); + + final HistogramAggregationBuilder builder = new HistogramAggregationBuilder("histo") + .field(histoAggFieldType.name()) + .interval(1) + .minDocCount(0) + .subAggregation( + new MissingAggregationBuilder("missing", null) + .field(missingAggFieldType.name()) + ); + + testCaseWithReduce( + newMatchAllQuery(), + builder, + writer -> { + writer.addDocument(singleton(new SortedNumericDocValuesField(histoAggFieldType.name(), 0))); + writer.addDocument(singleton(new SortedNumericDocValuesField(histoAggFieldType.name(), 2))); + }, + (InternalHistogram histogram) -> { + assertThat(histogram.getBuckets(), hasSize(3)); + + { + assertThat(histogram.getBuckets().get(0), notNullValue()); + final InternalMissing missing = histogram.getBuckets().get(0).getAggregations().get("missing"); + assertEquals(1, missing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(missing)); + } + { + assertThat(histogram.getBuckets().get(1), notNullValue()); + final InternalMissing missing = histogram.getBuckets().get(1).getAggregations().get("missing"); + assertEquals(0, missing.getDocCount()); + assertFalse(AggregationInspectionHelper.hasValue(missing)); + } + { + assertThat(histogram.getBuckets().get(2), notNullValue()); + final InternalMissing missing = histogram.getBuckets().get(2).getAggregations().get("missing"); + assertEquals(1, missing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(missing)); + } + }, + org.elasticsearch.common.collect.List.of(histoAggFieldType, missingAggFieldType), + true + ); + } + public void testUnmappedWithoutMissingParam() throws IOException { final int numDocs = randomIntBetween(10, 20); final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); @@ -215,9 +327,9 @@ public class MissingAggregatorTests extends AggregatorTestCase { writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong()))); } }, - internalMissing -> { - assertEquals(numDocs, internalMissing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + (InternalMissing missing) -> { + assertEquals(numDocs, missing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(missing)); }, singleton(aggFieldType) ); @@ -240,14 +352,44 @@ public class MissingAggregatorTests extends AggregatorTestCase { writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong()))); } }, - internalMissing -> { - assertEquals(0, internalMissing.getDocCount()); - assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); + (InternalMissing missing) -> { + assertEquals(0, missing.getDocCount()); + assertFalse(AggregationInspectionHelper.hasValue(missing)); }, singleton(aggFieldType) ); } + public void testSingleValuedFieldPartiallyUnmapped() throws IOException { + final int numDocs = randomIntBetween(10, 20); + + final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + fieldType.setName("field"); + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .field(fieldType.name()); + + try (Directory mappedDirectory = newDirectory(); Directory unmappedDirectory = newDirectory()) { + try (RandomIndexWriter mappedWriter = new RandomIndexWriter(random(), mappedDirectory)) { + for (int i = 0; i < numDocs; i++) { + mappedWriter.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong()))); + } + } + + new RandomIndexWriter(random(), unmappedDirectory).close(); + + try (IndexReader mappedReader = DirectoryReader.open(mappedDirectory); + IndexReader unmappedReader = DirectoryReader.open(unmappedDirectory); + MultiReader multiReader = new MultiReader(mappedReader, unmappedReader)) { + + final IndexSearcher searcher = newSearcher(multiReader, true, true); + + final InternalMissing internalMissing = searchAndReduce(searcher, newMatchAllQuery(), builder, fieldType); + assertEquals(0, internalMissing.getDocCount()); + assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); + } + } + } + public void testMissingParam() throws IOException { final int numDocs = randomIntBetween(10, 20); @@ -268,9 +410,9 @@ public class MissingAggregatorTests extends AggregatorTestCase { writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong()))); } }, - internalMissing -> { - assertEquals(0, internalMissing.getDocCount()); - assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); + (InternalMissing missing) -> { + assertEquals(0, missing.getDocCount()); + assertFalse(AggregationInspectionHelper.hasValue(missing)); }, org.elasticsearch.common.collect.List.of(aggFieldType, anotherFieldType) ); @@ -306,9 +448,9 @@ public class MissingAggregatorTests extends AggregatorTestCase { newMatchAllQuery(), builder, writer -> writer.addDocuments(docs), - internalMissing -> { - assertEquals(finalDocsMissingAggField, internalMissing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + (InternalMissing missing) -> { + assertEquals(finalDocsMissingAggField, missing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(missing)); }, org.elasticsearch.common.collect.List.of(aggFieldType, anotherFieldType) ); @@ -349,9 +491,9 @@ public class MissingAggregatorTests extends AggregatorTestCase { newMatchAllQuery(), builder, writer -> writer.addDocuments(docs), - internalMissing -> { - assertEquals(finalDocsMissingField, internalMissing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + (InternalMissing missing) -> { + assertEquals(finalDocsMissingField, missing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(missing)); }, org.elasticsearch.common.collect.List.of(aggFieldType, anotherFieldType) ); @@ -395,29 +537,32 @@ public class MissingAggregatorTests extends AggregatorTestCase { newMatchAllQuery(), builder, writer -> writer.addDocuments(docs), - internalMissing -> { - assertEquals(finalDocsBelowThreshold, internalMissing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + (InternalMissing missing) -> { + assertEquals(finalDocsBelowThreshold, missing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(missing)); }, singleton(aggFieldType) ); } - private void testCase(Query query, - MissingAggregationBuilder builder, - CheckedConsumer writeIndex, - Consumer verify, - Collection fieldTypes) throws IOException { + private void testCase( + Query query, + B builder, + CheckedConsumer writeIndex, + Consumer verify, + Collection fieldTypes) throws IOException { + testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, false); testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, true); } - private void testCaseWithReduce(Query query, - MissingAggregationBuilder builder, - CheckedConsumer writeIndex, - Consumer verify, - Collection fieldTypes, - boolean reduced) throws IOException { + private void testCaseWithReduce( + Query query, + B builder, + CheckedConsumer writeIndex, + Consumer verify, + Collection fieldTypes, + boolean reduced) throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { @@ -427,13 +572,13 @@ public class MissingAggregatorTests extends AggregatorTestCase { try (IndexReader indexReader = DirectoryReader.open(directory)) { final IndexSearcher indexSearcher = newSearcher(indexReader, true, true); final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]); - final InternalMissing missing; + final I result; if (reduced) { - missing = searchAndReduce(indexSearcher, query, builder, fieldTypesArray); + result = searchAndReduce(indexSearcher, query, builder, fieldTypesArray); } else { - missing = search(indexSearcher, query, builder, fieldTypesArray); + result = search(indexSearcher, query, builder, fieldTypesArray); } - verify.accept(missing); + verify.accept(result); } } }