From 574c4552038e53ac292274efdfebb6320ebba764 Mon Sep 17 00:00:00 2001 From: kimchy Date: Tue, 18 Jan 2011 21:51:16 +0200 Subject: [PATCH] Terms Facet: Add option include counts where term is missing, closes #632. --- .../index/field/data/FieldData.java | 2 + .../index/field/data/NumericFieldData.java | 2 + .../index/field/data/bytes/ByteFieldData.java | 2 + .../data/bytes/MultiValueByteFieldData.java | 15 +++++++ .../data/bytes/SingleValueByteFieldData.java | 3 ++ .../field/data/doubles/DoubleFieldData.java | 2 + .../doubles/MultiValueDoubleFieldData.java | 15 +++++++ .../doubles/SingleValueDoubleFieldData.java | 3 ++ .../field/data/floats/FloatFieldData.java | 2 + .../data/floats/MultiValueFloatFieldData.java | 15 +++++++ .../floats/SingleValueFloatFieldData.java | 3 ++ .../index/field/data/ints/IntFieldData.java | 2 + .../data/ints/MultiValueIntFieldData.java | 15 +++++++ .../data/ints/SingleValueIntFieldData.java | 3 ++ .../index/field/data/longs/LongFieldData.java | 2 + .../data/longs/MultiValueLongFieldData.java | 15 +++++++ .../data/longs/SingleValueLongFieldData.java | 3 ++ .../data/shorts/MultiValueShortFieldData.java | 15 +++++++ .../field/data/shorts/ShortFieldData.java | 2 + .../shorts/SingleValueShortFieldData.java | 3 ++ .../strings/MultiValueStringFieldData.java | 5 +++ .../strings/SingleValueStringFieldData.java | 1 + .../geo/MultiValueGeoPointFieldData.java | 5 +++ .../geo/SingleValueGeoPointFieldData.java | 1 + .../CountAndTotalHistogramFacetCollector.java | 6 +++ .../CountHistogramFacetCollector.java | 6 +++ ...KeyValueScriptHistogramFacetCollector.java | 6 +++ .../facet/range/RangeFacetCollector.java | 6 +++ .../StatisticalFacetCollector.java | 6 +++ .../StatisticalFieldsFacetCollector.java | 6 +++ .../search/facet/terms/TermsFacet.java | 10 +++++ .../terms/bytes/InternalByteTermsFacet.java | 20 ++++++++- .../terms/bytes/TermsByteFacetCollector.java | 14 ++++++- .../doubles/InternalDoubleTermsFacet.java | 21 ++++++++-- .../doubles/TermsDoubleFacetCollector.java | 14 ++++++- .../terms/floats/InternalFloatTermsFacet.java | 20 ++++++++- .../floats/TermsFloatFacetCollector.java | 14 ++++++- .../terms/index/IndexNameFacetCollector.java | 2 +- .../terms/ints/InternalIntTermsFacet.java | 20 ++++++++- .../terms/ints/TermsIntFacetCollector.java | 14 ++++++- .../terms/longs/InternalLongTermsFacet.java | 20 ++++++++- .../terms/longs/TermsLongFacetCollector.java | 14 ++++++- .../terms/shorts/InternalShortTermsFacet.java | 21 ++++++++-- .../shorts/TermsShortFacetCollector.java | 14 ++++++- .../FieldsTermsStringFacetCollector.java | 14 ++++++- .../strings/InternalStringTermsFacet.java | 21 ++++++++-- .../ScriptTermsStringFieldFacetCollector.java | 19 ++++++++- .../strings/TermsStringFacetCollector.java | 14 ++++++- .../search/facet/SimpleFacetsTests.java | 41 +++++++++++++++++++ 49 files changed, 464 insertions(+), 35 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/FieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/FieldData.java index c711902eb95..36fd7052202 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/FieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/FieldData.java @@ -91,6 +91,8 @@ public abstract class FieldData { public static interface StringValueInDocProc { void onValue(int docId, String value); + + void onMissing(int docId); } /** diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/NumericFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/NumericFieldData.java index e148b202576..b2b2fd30051 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/NumericFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/NumericFieldData.java @@ -96,5 +96,7 @@ public abstract class NumericFieldData extends public static interface DoubleValueInDocProc { void onValue(int docId, double value); + + void onMissing(int docId); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldData.java index d53cd929231..f825826a1e4 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldData.java @@ -111,6 +111,8 @@ public abstract class ByteFieldData extends NumericFieldData { public static interface ValueInDocProc { void onValue(int docId, byte value); + + void onMissing(int docID); } public static ByteFieldData load(IndexReader reader, String field) throws IOException { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java index 4e0f42ff75e..6ba79915e46 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java @@ -81,30 +81,45 @@ public class MultiValueByteFieldData extends ByteFieldData { } @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, Byte.toString(values[loc])); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, ValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public double[] doubleValues(int docId) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/SingleValueByteFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/SingleValueByteFieldData.java index 4d4c6c6a093..303dc4b551d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/SingleValueByteFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/SingleValueByteFieldData.java @@ -64,6 +64,7 @@ public class SingleValueByteFieldData extends ByteFieldData { @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, Byte.toString(values[loc])); @@ -72,6 +73,7 @@ public class SingleValueByteFieldData extends ByteFieldData { @Override public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); @@ -80,6 +82,7 @@ public class SingleValueByteFieldData extends ByteFieldData { @Override public void forEachValueInDoc(int docId, ValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldData.java index 2a13204dc8a..acbaca9bf7f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldData.java @@ -111,6 +111,8 @@ public abstract class DoubleFieldData extends NumericFieldData public static interface ValueInDocProc { void onValue(int docId, float value); + + void onMissing(int docId); } public static FloatFieldData load(IndexReader reader, String field) throws IOException { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java index 5190bd13d2c..aba388bed18 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java @@ -81,30 +81,45 @@ public class MultiValueFloatFieldData extends FloatFieldData { } @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, Float.toString(values[loc])); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, ValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public double[] doubleValues(int docId) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/SingleValueFloatFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/SingleValueFloatFieldData.java index 506da482c3c..03d8a75aa65 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/SingleValueFloatFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/SingleValueFloatFieldData.java @@ -64,6 +64,7 @@ public class SingleValueFloatFieldData extends FloatFieldData { @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, Float.toString(values[loc])); @@ -72,6 +73,7 @@ public class SingleValueFloatFieldData extends FloatFieldData { @Override public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); @@ -80,6 +82,7 @@ public class SingleValueFloatFieldData extends FloatFieldData { @Override public void forEachValueInDoc(int docId, ValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldData.java index 6d8145081d6..37779b1b1ff 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldData.java @@ -111,6 +111,8 @@ public abstract class IntFieldData extends NumericFieldData { public static interface ValueInDocProc { void onValue(int docId, int value); + + void onMissing(int docId); } public static IntFieldData load(IndexReader reader, String field) throws IOException { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java index a26fa36911d..5ca04dacbac 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java @@ -81,30 +81,45 @@ public class MultiValueIntFieldData extends IntFieldData { } @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, Integer.toString(values[loc])); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, ValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public double[] doubleValues(int docId) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/SingleValueIntFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/SingleValueIntFieldData.java index 0a4d47c0fb3..8e97a2c1828 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/SingleValueIntFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/SingleValueIntFieldData.java @@ -64,6 +64,7 @@ public class SingleValueIntFieldData extends IntFieldData { @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, Integer.toString(values[loc])); @@ -72,6 +73,7 @@ public class SingleValueIntFieldData extends IntFieldData { @Override public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); @@ -80,6 +82,7 @@ public class SingleValueIntFieldData extends IntFieldData { @Override public void forEachValueInDoc(int docId, ValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldData.java index f28e7f747ed..575848c1261 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldData.java @@ -133,6 +133,8 @@ public abstract class LongFieldData extends NumericFieldData { public static interface ValueInDocProc { void onValue(int docId, long value); + + void onMissing(int docId); } public abstract void forEachValueInDoc(int docId, DateValueInDocProc proc); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java index 5de1c4a29b8..07b6431add3 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java @@ -97,30 +97,45 @@ public class MultiValueLongFieldData extends LongFieldData { } @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, Long.toString(values[loc])); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, ValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, DateValueInDocProc proc) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/SingleValueLongFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/SingleValueLongFieldData.java index 739f25ab0f2..72865ea9a16 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/SingleValueLongFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/SingleValueLongFieldData.java @@ -74,6 +74,7 @@ public class SingleValueLongFieldData extends LongFieldData { @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, Long.toString(values[loc])); @@ -82,6 +83,7 @@ public class SingleValueLongFieldData extends LongFieldData { @Override public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); @@ -90,6 +92,7 @@ public class SingleValueLongFieldData extends LongFieldData { @Override public void forEachValueInDoc(int docId, ValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java index 934d5c023ac..7d4e7caeb01 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java @@ -81,30 +81,45 @@ public class MultiValueShortFieldData extends ShortFieldData { } @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, Short.toString(values[loc])); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public void forEachValueInDoc(int docId, ValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public double[] doubleValues(int docId) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldData.java index 715fc8ba3a3..c47f0dcadd2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldData.java @@ -111,6 +111,8 @@ public abstract class ShortFieldData extends NumericFieldData public static interface ValueInDocProc { void onValue(int docId, short value); + + void onMissing(int docId); } public static ShortFieldData load(IndexReader reader, String field) throws IOException { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/SingleValueShortFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/SingleValueShortFieldData.java index 5d6b9fc2b91..51a76497609 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/SingleValueShortFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/SingleValueShortFieldData.java @@ -64,6 +64,7 @@ public class SingleValueShortFieldData extends ShortFieldData { @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, Short.toString(values[loc])); @@ -72,6 +73,7 @@ public class SingleValueShortFieldData extends ShortFieldData { @Override public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); @@ -80,6 +82,7 @@ public class SingleValueShortFieldData extends ShortFieldData { @Override public void forEachValueInDoc(int docId, ValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java index 4e52e0186c2..d8a690d8314 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java @@ -71,12 +71,17 @@ public class MultiValueStringFieldData extends StringFieldData { } @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, values[loc]); } } + if (!found) { + proc.onMissing(docId); + } } @Override public String value(int docId) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/SingleValueStringFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/SingleValueStringFieldData.java index 93d2d13c845..771cb2b88f9 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/SingleValueStringFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/SingleValueStringFieldData.java @@ -66,6 +66,7 @@ public class SingleValueStringFieldData extends StringFieldData { @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, values[loc]); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/MultiValueGeoPointFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/MultiValueGeoPointFieldData.java index f2e8dc20124..3416c6addd8 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/MultiValueGeoPointFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/MultiValueGeoPointFieldData.java @@ -95,12 +95,17 @@ public class MultiValueGeoPointFieldData extends GeoPointFieldData { } @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { + boolean found = false; for (int[] ordinal : ordinals) { int loc = ordinal[docId]; if (loc != 0) { + found = true; proc.onValue(docId, GeoHashUtils.encode(lat[loc], lon[loc])); } } + if (!found) { + proc.onMissing(docId); + } } @Override public GeoPoint value(int docId) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/SingleValueGeoPointFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/SingleValueGeoPointFieldData.java index ed197973dc4..40cd40043e6 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/SingleValueGeoPointFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/SingleValueGeoPointFieldData.java @@ -74,6 +74,7 @@ public class SingleValueGeoPointFieldData extends GeoPointFieldData { @Override public void forEachValueInDoc(int docId, StringValueInDocProc proc) { int loc = ordinals[docId]; if (loc == 0) { + proc.onMissing(docId); return; } proc.onValue(docId, GeoHashUtils.encode(lat[loc], lon[loc])); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/CountAndTotalHistogramFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/CountAndTotalHistogramFacetCollector.java index b5bbbcf1d19..ebd45f2a01b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/CountAndTotalHistogramFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/CountAndTotalHistogramFacetCollector.java @@ -107,6 +107,8 @@ public class CountAndTotalHistogramFacetCollector extends AbstractFacetCollector private final TLongDoubleHashMap totals = new TLongDoubleHashMap(); + private int missing; + public HistogramProc(long interval) { this.interval = interval; } @@ -117,6 +119,10 @@ public class CountAndTotalHistogramFacetCollector extends AbstractFacetCollector totals.adjustOrPutValue(bucket, value, value); } + @Override public void onMissing(int docId) { + missing++; + } + public TLongLongHashMap counts() { return counts; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/CountHistogramFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/CountHistogramFacetCollector.java index 94458856809..fd0016ad410 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/CountHistogramFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/CountHistogramFacetCollector.java @@ -104,6 +104,8 @@ public class CountHistogramFacetCollector extends AbstractFacetCollector { private final TLongLongHashMap counts = new TLongLongHashMap(); + private int missing; + public HistogramProc(long interval) { this.interval = interval; } @@ -113,6 +115,10 @@ public class CountHistogramFacetCollector extends AbstractFacetCollector { counts.adjustOrPutValue(bucket, 1, 1); } + @Override public void onMissing(int docId) { + missing++; + } + public TLongLongHashMap counts() { return counts; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/KeyValueScriptHistogramFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/KeyValueScriptHistogramFacetCollector.java index 7a17219e53d..1aaeb03577d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/KeyValueScriptHistogramFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/KeyValueScriptHistogramFacetCollector.java @@ -116,6 +116,8 @@ public class KeyValueScriptHistogramFacetCollector extends AbstractFacetCollecto private final TLongDoubleHashMap totals = new TLongDoubleHashMap(); + private int missing; + public HistogramProc(long interval, SearchScript valueScript) { this.interval = interval; this.valueScript = valueScript; @@ -128,6 +130,10 @@ public class KeyValueScriptHistogramFacetCollector extends AbstractFacetCollecto totals.adjustOrPutValue(bucket, scriptValue, scriptValue); } + @Override public void onMissing(int docId) { + missing++; + } + public TLongLongHashMap counts() { return counts; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/RangeFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/RangeFacetCollector.java index 888f825efac..6052772b417 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/RangeFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/RangeFacetCollector.java @@ -88,6 +88,8 @@ public class RangeFacetCollector extends AbstractFacetCollector { private final RangeFacet.Entry[] entries; + private int missing; + public RangeProc(RangeFacet.Entry[] entries) { this.entries = entries; } @@ -100,5 +102,9 @@ public class RangeFacetCollector extends AbstractFacetCollector { } } } + + @Override public void onMissing(int docId) { + missing++; + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetCollector.java index d0c842ba5ca..3218ead6edc 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetCollector.java @@ -91,6 +91,8 @@ public class StatisticalFacetCollector extends AbstractFacetCollector { private long count; + private int missing; + @Override public void onValue(int docId, double value) { if (value < min || Double.isNaN(min)) { min = value; @@ -103,6 +105,10 @@ public class StatisticalFacetCollector extends AbstractFacetCollector { count++; } + @Override public void onMissing(int docId) { + missing++; + } + public final double min() { return min; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFieldsFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFieldsFacetCollector.java index 989bd47f96b..b55ea9be697 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFieldsFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFieldsFacetCollector.java @@ -97,6 +97,8 @@ public class StatisticalFieldsFacetCollector extends AbstractFacetCollector { private long count; + private int missing; + @Override public void onValue(int docId, double value) { if (value < min || Double.isNaN(min)) { min = value; @@ -109,6 +111,10 @@ public class StatisticalFieldsFacetCollector extends AbstractFacetCollector { count++; } + @Override public void onMissing(int docId) { + missing++; + } + public final double min() { return min; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/TermsFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/TermsFacet.java index 491a53660f5..b0b7a99d7ad 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/TermsFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/TermsFacet.java @@ -164,6 +164,16 @@ public interface TermsFacet extends Facet, Iterable { */ ComparatorType getComparatorType(); + /** + * The number of docs missing a value. + */ + long missingCount(); + + /** + * The number of docs missing a value. + */ + long getMissingCount(); + /** * The terms and counts. */ diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/bytes/InternalByteTermsFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/bytes/InternalByteTermsFacet.java index b6b3fa8bdbd..8482eb20445 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/bytes/InternalByteTermsFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/bytes/InternalByteTermsFacet.java @@ -112,6 +112,8 @@ public class InternalByteTermsFacet extends InternalTermsFacet { int requiredSize; + long missing; + Collection entries = ImmutableList.of(); private ComparatorType comparatorType; @@ -119,12 +121,13 @@ public class InternalByteTermsFacet extends InternalTermsFacet { InternalByteTermsFacet() { } - public InternalByteTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries) { + public InternalByteTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries, long missing) { this.name = name; this.fieldName = fieldName; this.comparatorType = comparatorType; this.requiredSize = requiredSize; this.entries = entries; + this.missing = missing; } @Override public String name() { @@ -159,6 +162,14 @@ public class InternalByteTermsFacet extends InternalTermsFacet { return comparatorType(); } + @Override public long missingCount() { + return this.missing; + } + + @Override public long getMissingCount() { + return missingCount(); + } + @Override public List entries() { if (!(entries instanceof List)) { entries = ImmutableList.copyOf(entries); @@ -190,8 +201,10 @@ public class InternalByteTermsFacet extends InternalTermsFacet { TByteIntHashMap aggregated = aggregateCache.get().get(); aggregated.clear(); + long missing = 0; for (Facet facet : facets) { InternalByteTermsFacet mFacet = (InternalByteTermsFacet) facet; + missing += mFacet.missingCount(); for (ByteEntry entry : mFacet.entries) { aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count()); } @@ -203,12 +216,14 @@ public class InternalByteTermsFacet extends InternalTermsFacet { ordered.add(new ByteEntry(it.key(), it.value())); } first.entries = ordered; + first.missing = missing; return first; } static final class Fields { static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); static final XContentBuilderString _FIELD = new XContentBuilderString("_field"); + static final XContentBuilderString MISSING = new XContentBuilderString("missing"); static final XContentBuilderString TERMS = new XContentBuilderString("terms"); static final XContentBuilderString TERM = new XContentBuilderString("term"); static final XContentBuilderString COUNT = new XContentBuilderString("count"); @@ -218,6 +233,7 @@ public class InternalByteTermsFacet extends InternalTermsFacet { builder.startObject(name); builder.field(Fields._TYPE, TermsFacet.TYPE); builder.field(Fields._FIELD, fieldName); + builder.field(Fields.MISSING, missing); builder.startArray(Fields.TERMS); for (ByteEntry entry : entries) { builder.startObject(); @@ -241,6 +257,7 @@ public class InternalByteTermsFacet extends InternalTermsFacet { fieldName = in.readUTF(); comparatorType = ComparatorType.fromId(in.readByte()); requiredSize = in.readVInt(); + missing = in.readVLong(); int size = in.readVInt(); entries = new ArrayList(size); @@ -255,6 +272,7 @@ public class InternalByteTermsFacet extends InternalTermsFacet { out.writeByte(comparatorType.id()); out.writeVInt(requiredSize); + out.writeVLong(missing); out.writeVInt(entries.size()); for (ByteEntry entry : entries) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteFacetCollector.java index a9fb9f1ddcf..1a6d207d33d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteFacetCollector.java @@ -128,7 +128,7 @@ public class TermsByteFacetCollector extends AbstractFacetCollector { TByteIntHashMap facets = aggregator.facets(); if (facets.isEmpty()) { pushFacets(facets); - return new InternalByteTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of()); + return new InternalByteTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of(), aggregator.missing()); } else { // we need to fetch facets of "size * numberOfShards" because of problems in how they are distributed across shards BoundedTreeSet ordered = new BoundedTreeSet(comparatorType.comparator(), size * numberOfShards); @@ -137,7 +137,7 @@ public class TermsByteFacetCollector extends AbstractFacetCollector { ordered.add(new InternalByteTermsFacet.ByteEntry(it.key(), it.value())); } pushFacets(facets); - return new InternalByteTermsFacet(facetName, fieldName, comparatorType, size, ordered); + return new InternalByteTermsFacet(facetName, fieldName, comparatorType, size, ordered, aggregator.missing()); } } @@ -198,6 +198,8 @@ public class TermsByteFacetCollector extends AbstractFacetCollector { private final TByteIntHashMap facets; + private int missing; + public StaticAggregatorValueProc(TByteIntHashMap facets) { this.facets = facets; } @@ -206,8 +208,16 @@ public class TermsByteFacetCollector extends AbstractFacetCollector { facets.adjustOrPutValue(value, 1, 1); } + @Override public void onMissing(int docID) { + missing++; + } + public final TByteIntHashMap facets() { return facets; } + + public final int missing() { + return this.missing; + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/doubles/InternalDoubleTermsFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/doubles/InternalDoubleTermsFacet.java index ce2c98d1c2c..a8ce68c0e90 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/doubles/InternalDoubleTermsFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/doubles/InternalDoubleTermsFacet.java @@ -115,6 +115,8 @@ public class InternalDoubleTermsFacet extends InternalTermsFacet { int requiredSize; + long missing; + Collection entries = ImmutableList.of(); private ComparatorType comparatorType; @@ -122,12 +124,13 @@ public class InternalDoubleTermsFacet extends InternalTermsFacet { InternalDoubleTermsFacet() { } - public InternalDoubleTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries) { + public InternalDoubleTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries, long missing) { this.name = name; this.fieldName = fieldName; this.comparatorType = comparatorType; this.requiredSize = requiredSize; this.entries = entries; + this.missing = missing; } @Override public String name() { @@ -177,6 +180,13 @@ public class InternalDoubleTermsFacet extends InternalTermsFacet { return (Iterator) entries.iterator(); } + @Override public long missingCount() { + return this.missing; + } + + @Override public long getMissingCount() { + return missingCount(); + } private static ThreadLocal> aggregateCache = new ThreadLocal>() { @Override protected ThreadLocals.CleanableValue initialValue() { @@ -192,9 +202,10 @@ public class InternalDoubleTermsFacet extends InternalTermsFacet { InternalDoubleTermsFacet first = (InternalDoubleTermsFacet) facets.get(0); TDoubleIntHashMap aggregated = aggregateCache.get().get(); aggregated.clear(); - + long missing = 0; for (Facet facet : facets) { InternalDoubleTermsFacet mFacet = (InternalDoubleTermsFacet) facet; + missing += mFacet.missingCount(); for (DoubleEntry entry : mFacet.entries) { aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count()); } @@ -206,12 +217,14 @@ public class InternalDoubleTermsFacet extends InternalTermsFacet { ordered.add(new DoubleEntry(it.key(), it.value())); } first.entries = ordered; + first.missing = missing; return first; } static final class Fields { static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); static final XContentBuilderString _FIELD = new XContentBuilderString("_field"); + static final XContentBuilderString MISSING = new XContentBuilderString("missing"); static final XContentBuilderString TERMS = new XContentBuilderString("terms"); static final XContentBuilderString TERM = new XContentBuilderString("term"); static final XContentBuilderString COUNT = new XContentBuilderString("count"); @@ -221,6 +234,7 @@ public class InternalDoubleTermsFacet extends InternalTermsFacet { builder.startObject(name); builder.field(Fields._TYPE, TermsFacet.TYPE); builder.field(Fields._FIELD, fieldName); + builder.field(Fields.MISSING, missing); builder.startArray(Fields.TERMS); for (DoubleEntry entry : entries) { builder.startObject(); @@ -244,6 +258,7 @@ public class InternalDoubleTermsFacet extends InternalTermsFacet { fieldName = in.readUTF(); comparatorType = ComparatorType.fromId(in.readByte()); requiredSize = in.readVInt(); + missing = in.readVLong(); int size = in.readVInt(); entries = new ArrayList(size); @@ -256,8 +271,8 @@ public class InternalDoubleTermsFacet extends InternalTermsFacet { out.writeUTF(name); out.writeUTF(fieldName); out.writeByte(comparatorType.id()); - out.writeVInt(requiredSize); + out.writeVLong(missing); out.writeVInt(entries.size()); for (DoubleEntry entry : entries) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleFacetCollector.java index 450a5a3bd89..a8667bdbca5 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleFacetCollector.java @@ -128,7 +128,7 @@ public class TermsDoubleFacetCollector extends AbstractFacetCollector { TDoubleIntHashMap facets = aggregator.facets(); if (facets.isEmpty()) { pushFacets(facets); - return new InternalDoubleTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of()); + return new InternalDoubleTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of(), aggregator.missing()); } else { // we need to fetch facets of "size * numberOfShards" because of problems in how they are distributed across shards BoundedTreeSet ordered = new BoundedTreeSet(comparatorType.comparator(), size * numberOfShards); @@ -137,7 +137,7 @@ public class TermsDoubleFacetCollector extends AbstractFacetCollector { ordered.add(new InternalDoubleTermsFacet.DoubleEntry(it.key(), it.value())); } pushFacets(facets); - return new InternalDoubleTermsFacet(facetName, fieldName, comparatorType, size, ordered); + return new InternalDoubleTermsFacet(facetName, fieldName, comparatorType, size, ordered, aggregator.missing()); } } @@ -198,6 +198,8 @@ public class TermsDoubleFacetCollector extends AbstractFacetCollector { private final TDoubleIntHashMap facets; + private int missing; + public StaticAggregatorValueProc(TDoubleIntHashMap facets) { this.facets = facets; } @@ -206,8 +208,16 @@ public class TermsDoubleFacetCollector extends AbstractFacetCollector { facets.adjustOrPutValue(value, 1, 1); } + @Override public void onMissing(int docId) { + missing++; + } + public final TDoubleIntHashMap facets() { return facets; } + + public final int missing() { + return this.missing; + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/floats/InternalFloatTermsFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/floats/InternalFloatTermsFacet.java index 9df367ef4ea..1c29764d855 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/floats/InternalFloatTermsFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/floats/InternalFloatTermsFacet.java @@ -115,6 +115,8 @@ public class InternalFloatTermsFacet extends InternalTermsFacet { int requiredSize; + long missing; + Collection entries = ImmutableList.of(); private ComparatorType comparatorType; @@ -122,12 +124,13 @@ public class InternalFloatTermsFacet extends InternalTermsFacet { InternalFloatTermsFacet() { } - public InternalFloatTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries) { + public InternalFloatTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries, long missing) { this.name = name; this.fieldName = fieldName; this.comparatorType = comparatorType; this.requiredSize = requiredSize; this.entries = entries; + this.missing = missing; } @Override public String name() { @@ -177,6 +180,13 @@ public class InternalFloatTermsFacet extends InternalTermsFacet { return (Iterator) entries.iterator(); } + @Override public long missingCount() { + return this.missing; + } + + @Override public long getMissingCount() { + return missingCount(); + } private static ThreadLocal> aggregateCache = new ThreadLocal>() { @Override protected ThreadLocals.CleanableValue initialValue() { @@ -192,9 +202,11 @@ public class InternalFloatTermsFacet extends InternalTermsFacet { InternalFloatTermsFacet first = (InternalFloatTermsFacet) facets.get(0); TFloatIntHashMap aggregated = aggregateCache.get().get(); aggregated.clear(); + long missing = 0; for (Facet facet : facets) { InternalFloatTermsFacet mFacet = (InternalFloatTermsFacet) facet; + missing += mFacet.missingCount(); for (FloatEntry entry : mFacet.entries) { aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count()); } @@ -206,12 +218,14 @@ public class InternalFloatTermsFacet extends InternalTermsFacet { ordered.add(new FloatEntry(it.key(), it.value())); } first.entries = ordered; + first.missing = missing; return first; } static final class Fields { static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); static final XContentBuilderString _FIELD = new XContentBuilderString("_field"); + static final XContentBuilderString MISSING = new XContentBuilderString("missing"); static final XContentBuilderString TERMS = new XContentBuilderString("terms"); static final XContentBuilderString TERM = new XContentBuilderString("term"); static final XContentBuilderString COUNT = new XContentBuilderString("count"); @@ -221,6 +235,7 @@ public class InternalFloatTermsFacet extends InternalTermsFacet { builder.startObject(name); builder.field(Fields._TYPE, TermsFacet.TYPE); builder.field(Fields._FIELD, fieldName); + builder.field(Fields.MISSING, missing); builder.startArray(Fields.TERMS); for (FloatEntry entry : entries) { builder.startObject(); @@ -244,6 +259,7 @@ public class InternalFloatTermsFacet extends InternalTermsFacet { fieldName = in.readUTF(); comparatorType = ComparatorType.fromId(in.readByte()); requiredSize = in.readVInt(); + missing = in.readVLong(); int size = in.readVInt(); entries = new ArrayList(size); @@ -256,8 +272,8 @@ public class InternalFloatTermsFacet extends InternalTermsFacet { out.writeUTF(name); out.writeUTF(fieldName); out.writeByte(comparatorType.id()); - out.writeVInt(requiredSize); + out.writeVLong(missing); out.writeVInt(entries.size()); for (FloatEntry entry : entries) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatFacetCollector.java index 103a070248f..6ef0308160f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatFacetCollector.java @@ -128,7 +128,7 @@ public class TermsFloatFacetCollector extends AbstractFacetCollector { TFloatIntHashMap facets = aggregator.facets(); if (facets.isEmpty()) { pushFacets(facets); - return new InternalFloatTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of()); + return new InternalFloatTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of(), aggregator.missing()); } else { // we need to fetch facets of "size * numberOfShards" because of problems in how they are distributed across shards BoundedTreeSet ordered = new BoundedTreeSet(comparatorType.comparator(), size * numberOfShards); @@ -137,7 +137,7 @@ public class TermsFloatFacetCollector extends AbstractFacetCollector { ordered.add(new InternalFloatTermsFacet.FloatEntry(it.key(), it.value())); } pushFacets(facets); - return new InternalFloatTermsFacet(facetName, fieldName, comparatorType, size, ordered); + return new InternalFloatTermsFacet(facetName, fieldName, comparatorType, size, ordered, aggregator.missing()); } } @@ -198,6 +198,8 @@ public class TermsFloatFacetCollector extends AbstractFacetCollector { private final TFloatIntHashMap facets; + private int missing; + public StaticAggregatorValueProc(TFloatIntHashMap facets) { this.facets = facets; } @@ -206,8 +208,16 @@ public class TermsFloatFacetCollector extends AbstractFacetCollector { facets.adjustOrPutValue(value, 1, 1); } + @Override public void onMissing(int docId) { + missing++; + } + public final TFloatIntHashMap facets() { return facets; } + + public final int missing() { + return this.missing; + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/index/IndexNameFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/index/IndexNameFacetCollector.java index f872d7a8c8c..16696cbf5ce 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/index/IndexNameFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/index/IndexNameFacetCollector.java @@ -56,6 +56,6 @@ public class IndexNameFacetCollector extends AbstractFacetCollector { } @Override public Facet facet() { - return new InternalStringTermsFacet(facetName, "_index", comparatorType, size, Sets.newHashSet(new InternalStringTermsFacet.StringEntry(indexName, count))); + return new InternalStringTermsFacet(facetName, "_index", comparatorType, size, Sets.newHashSet(new InternalStringTermsFacet.StringEntry(indexName, count)), 0); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/ints/InternalIntTermsFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/ints/InternalIntTermsFacet.java index 8614a0e682a..3bd69d62f18 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/ints/InternalIntTermsFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/ints/InternalIntTermsFacet.java @@ -112,6 +112,8 @@ public class InternalIntTermsFacet extends InternalTermsFacet { int requiredSize; + long missing; + Collection entries = ImmutableList.of(); private ComparatorType comparatorType; @@ -119,12 +121,13 @@ public class InternalIntTermsFacet extends InternalTermsFacet { InternalIntTermsFacet() { } - public InternalIntTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries) { + public InternalIntTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries, long missing) { this.name = name; this.fieldName = fieldName; this.comparatorType = comparatorType; this.requiredSize = requiredSize; this.entries = entries; + this.missing = missing; } @Override public String name() { @@ -174,6 +177,13 @@ public class InternalIntTermsFacet extends InternalTermsFacet { return (Iterator) entries.iterator(); } + @Override public long missingCount() { + return this.missing; + } + + @Override public long getMissingCount() { + return missingCount(); + } private static ThreadLocal> aggregateCache = new ThreadLocal>() { @Override protected ThreadLocals.CleanableValue initialValue() { @@ -189,9 +199,11 @@ public class InternalIntTermsFacet extends InternalTermsFacet { InternalIntTermsFacet first = (InternalIntTermsFacet) facets.get(0); TIntIntHashMap aggregated = aggregateCache.get().get(); aggregated.clear(); + long missing = 0; for (Facet facet : facets) { InternalIntTermsFacet mFacet = (InternalIntTermsFacet) facet; + missing += mFacet.missingCount(); for (IntEntry entry : mFacet.entries) { aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count()); } @@ -203,12 +215,14 @@ public class InternalIntTermsFacet extends InternalTermsFacet { ordered.add(new IntEntry(it.key(), it.value())); } first.entries = ordered; + first.missing = missing; return first; } static final class Fields { static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); static final XContentBuilderString _FIELD = new XContentBuilderString("_field"); + static final XContentBuilderString MISSING = new XContentBuilderString("missing"); static final XContentBuilderString TERMS = new XContentBuilderString("terms"); static final XContentBuilderString TERM = new XContentBuilderString("term"); static final XContentBuilderString COUNT = new XContentBuilderString("count"); @@ -218,6 +232,7 @@ public class InternalIntTermsFacet extends InternalTermsFacet { builder.startObject(name); builder.field(Fields._TYPE, TermsFacet.TYPE); builder.field(Fields._FIELD, fieldName); + builder.field(Fields.MISSING, missing); builder.startArray(Fields.TERMS); for (IntEntry entry : entries) { builder.startObject(); @@ -241,6 +256,7 @@ public class InternalIntTermsFacet extends InternalTermsFacet { fieldName = in.readUTF(); comparatorType = ComparatorType.fromId(in.readByte()); requiredSize = in.readVInt(); + missing = in.readVLong(); int size = in.readVInt(); entries = new ArrayList(size); @@ -253,8 +269,8 @@ public class InternalIntTermsFacet extends InternalTermsFacet { out.writeUTF(name); out.writeUTF(fieldName); out.writeByte(comparatorType.id()); - out.writeVInt(requiredSize); + out.writeVLong(missing); out.writeVInt(entries.size()); for (IntEntry entry : entries) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntFacetCollector.java index 99722d30d34..e5f1901c4eb 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntFacetCollector.java @@ -128,7 +128,7 @@ public class TermsIntFacetCollector extends AbstractFacetCollector { TIntIntHashMap facets = aggregator.facets(); if (facets.isEmpty()) { pushFacets(facets); - return new InternalIntTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of()); + return new InternalIntTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of(), aggregator.missing()); } else { // we need to fetch facets of "size * numberOfShards" because of problems in how they are distributed across shards BoundedTreeSet ordered = new BoundedTreeSet(comparatorType.comparator(), size * numberOfShards); @@ -137,7 +137,7 @@ public class TermsIntFacetCollector extends AbstractFacetCollector { ordered.add(new InternalIntTermsFacet.IntEntry(it.key(), it.value())); } pushFacets(facets); - return new InternalIntTermsFacet(facetName, fieldName, comparatorType, size, ordered); + return new InternalIntTermsFacet(facetName, fieldName, comparatorType, size, ordered, aggregator.missing()); } } @@ -198,6 +198,8 @@ public class TermsIntFacetCollector extends AbstractFacetCollector { private final TIntIntHashMap facets; + private int missing; + public StaticAggregatorValueProc(TIntIntHashMap facets) { this.facets = facets; } @@ -206,8 +208,16 @@ public class TermsIntFacetCollector extends AbstractFacetCollector { facets.adjustOrPutValue(value, 1, 1); } + @Override public void onMissing(int docId) { + missing++; + } + public final TIntIntHashMap facets() { return facets; } + + public final int missing() { + return this.missing; + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/longs/InternalLongTermsFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/longs/InternalLongTermsFacet.java index e31e2e85c20..bf7348e4c65 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/longs/InternalLongTermsFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/longs/InternalLongTermsFacet.java @@ -115,6 +115,8 @@ public class InternalLongTermsFacet extends InternalTermsFacet { int requiredSize; + long missing; + Collection entries = ImmutableList.of(); private ComparatorType comparatorType; @@ -122,12 +124,13 @@ public class InternalLongTermsFacet extends InternalTermsFacet { InternalLongTermsFacet() { } - public InternalLongTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries) { + public InternalLongTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries, long missing) { this.name = name; this.fieldName = fieldName; this.comparatorType = comparatorType; this.requiredSize = requiredSize; this.entries = entries; + this.missing = missing; } @Override public String name() { @@ -177,6 +180,13 @@ public class InternalLongTermsFacet extends InternalTermsFacet { return (Iterator) entries.iterator(); } + @Override public long missingCount() { + return this.missing; + } + + @Override public long getMissingCount() { + return missingCount(); + } private static ThreadLocal> aggregateCache = new ThreadLocal>() { @Override protected ThreadLocals.CleanableValue initialValue() { @@ -192,9 +202,11 @@ public class InternalLongTermsFacet extends InternalTermsFacet { InternalLongTermsFacet first = (InternalLongTermsFacet) facets.get(0); TLongIntHashMap aggregated = aggregateCache.get().get(); aggregated.clear(); + long missing = 0; for (Facet facet : facets) { InternalLongTermsFacet mFacet = (InternalLongTermsFacet) facet; + missing += mFacet.missingCount(); for (LongEntry entry : mFacet.entries) { aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count()); } @@ -206,12 +218,14 @@ public class InternalLongTermsFacet extends InternalTermsFacet { ordered.add(new LongEntry(it.key(), it.value())); } first.entries = ordered; + first.missing = missing; return first; } static final class Fields { static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); static final XContentBuilderString _FIELD = new XContentBuilderString("_field"); + static final XContentBuilderString MISSING = new XContentBuilderString("missing"); static final XContentBuilderString TERMS = new XContentBuilderString("terms"); static final XContentBuilderString TERM = new XContentBuilderString("term"); static final XContentBuilderString COUNT = new XContentBuilderString("count"); @@ -221,6 +235,7 @@ public class InternalLongTermsFacet extends InternalTermsFacet { builder.startObject(name); builder.field(Fields._TYPE, TermsFacet.TYPE); builder.field(Fields._FIELD, fieldName); + builder.field(Fields.MISSING, missing); builder.startArray(Fields.TERMS); for (LongEntry entry : entries) { builder.startObject(); @@ -244,6 +259,7 @@ public class InternalLongTermsFacet extends InternalTermsFacet { fieldName = in.readUTF(); comparatorType = ComparatorType.fromId(in.readByte()); requiredSize = in.readVInt(); + missing = in.readVLong(); int size = in.readVInt(); entries = new ArrayList(size); @@ -256,8 +272,8 @@ public class InternalLongTermsFacet extends InternalTermsFacet { out.writeUTF(name); out.writeUTF(fieldName); out.writeByte(comparatorType.id()); - out.writeVInt(requiredSize); + out.writeVLong(missing); out.writeVInt(entries.size()); for (LongEntry entry : entries) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongFacetCollector.java index ebf3daeb90f..06315ea3ab3 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongFacetCollector.java @@ -129,7 +129,7 @@ public class TermsLongFacetCollector extends AbstractFacetCollector { TLongIntHashMap facets = aggregator.facets(); if (facets.isEmpty()) { pushFacets(facets); - return new InternalLongTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of()); + return new InternalLongTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of(), aggregator.missing()); } else { // we need to fetch facets of "size * numberOfShards" because of problems in how they are distributed across shards BoundedTreeSet ordered = new BoundedTreeSet(comparatorType.comparator(), size * numberOfShards); @@ -138,7 +138,7 @@ public class TermsLongFacetCollector extends AbstractFacetCollector { ordered.add(new InternalLongTermsFacet.LongEntry(it.key(), it.value())); } pushFacets(facets); - return new InternalLongTermsFacet(facetName, fieldName, comparatorType, size, ordered); + return new InternalLongTermsFacet(facetName, fieldName, comparatorType, size, ordered, aggregator.missing()); } } @@ -199,6 +199,8 @@ public class TermsLongFacetCollector extends AbstractFacetCollector { private final TLongIntHashMap facets; + private int missing; + public StaticAggregatorValueProc(TLongIntHashMap facets) { this.facets = facets; } @@ -207,8 +209,16 @@ public class TermsLongFacetCollector extends AbstractFacetCollector { facets.adjustOrPutValue(value, 1, 1); } + @Override public void onMissing(int docId) { + missing++; + } + public final TLongIntHashMap facets() { return facets; } + + public final int missing() { + return this.missing; + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/shorts/InternalShortTermsFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/shorts/InternalShortTermsFacet.java index f12e4b43123..a8eb90ed6d4 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/shorts/InternalShortTermsFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/shorts/InternalShortTermsFacet.java @@ -112,6 +112,8 @@ public class InternalShortTermsFacet extends InternalTermsFacet { int requiredSize; + long missing; + Collection entries = ImmutableList.of(); private ComparatorType comparatorType; @@ -119,12 +121,13 @@ public class InternalShortTermsFacet extends InternalTermsFacet { InternalShortTermsFacet() { } - public InternalShortTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries) { + public InternalShortTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries, long missing) { this.name = name; this.fieldName = fieldName; this.comparatorType = comparatorType; this.requiredSize = requiredSize; this.entries = entries; + this.missing = missing; } @Override public String name() { @@ -174,6 +177,13 @@ public class InternalShortTermsFacet extends InternalTermsFacet { return (Iterator) entries.iterator(); } + @Override public long missingCount() { + return this.missing; + } + + @Override public long getMissingCount() { + return missingCount(); + } private static ThreadLocal> aggregateCache = new ThreadLocal>() { @Override protected ThreadLocals.CleanableValue initialValue() { @@ -189,9 +199,10 @@ public class InternalShortTermsFacet extends InternalTermsFacet { InternalShortTermsFacet first = (InternalShortTermsFacet) facets.get(0); TShortIntHashMap aggregated = aggregateCache.get().get(); aggregated.clear(); - + long missing = 0; for (Facet facet : facets) { InternalShortTermsFacet mFacet = (InternalShortTermsFacet) facet; + missing += mFacet.missingCount(); for (ShortEntry entry : mFacet.entries) { aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count()); } @@ -203,12 +214,14 @@ public class InternalShortTermsFacet extends InternalTermsFacet { ordered.add(new ShortEntry(it.key(), it.value())); } first.entries = ordered; + first.missing = missing; return first; } static final class Fields { static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); static final XContentBuilderString _FIELD = new XContentBuilderString("_field"); + static final XContentBuilderString MISSING = new XContentBuilderString("missing"); static final XContentBuilderString TERMS = new XContentBuilderString("terms"); static final XContentBuilderString TERM = new XContentBuilderString("term"); static final XContentBuilderString COUNT = new XContentBuilderString("count"); @@ -218,6 +231,7 @@ public class InternalShortTermsFacet extends InternalTermsFacet { builder.startObject(name); builder.field(Fields._TYPE, TermsFacet.TYPE); builder.field(Fields._FIELD, fieldName); + builder.field(Fields.MISSING, missing); builder.startArray(Fields.TERMS); for (ShortEntry entry : entries) { builder.startObject(); @@ -241,6 +255,7 @@ public class InternalShortTermsFacet extends InternalTermsFacet { fieldName = in.readUTF(); comparatorType = ComparatorType.fromId(in.readByte()); requiredSize = in.readVInt(); + missing = in.readVLong(); int size = in.readVInt(); entries = new ArrayList(size); @@ -253,8 +268,8 @@ public class InternalShortTermsFacet extends InternalTermsFacet { out.writeUTF(name); out.writeUTF(fieldName); out.writeByte(comparatorType.id()); - out.writeVInt(requiredSize); + out.writeVLong(missing); out.writeVInt(entries.size()); for (ShortEntry entry : entries) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortFacetCollector.java index 834334b631b..44a54307ef7 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortFacetCollector.java @@ -128,7 +128,7 @@ public class TermsShortFacetCollector extends AbstractFacetCollector { TShortIntHashMap facets = aggregator.facets(); if (facets.isEmpty()) { pushFacets(facets); - return new InternalShortTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of()); + return new InternalShortTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of(), aggregator.missing()); } else { // we need to fetch facets of "size * numberOfShards" because of problems in how they are distributed across shards BoundedTreeSet ordered = new BoundedTreeSet(comparatorType.comparator(), size * numberOfShards); @@ -137,7 +137,7 @@ public class TermsShortFacetCollector extends AbstractFacetCollector { ordered.add(new InternalShortTermsFacet.ShortEntry(it.key(), it.value())); } pushFacets(facets); - return new InternalShortTermsFacet(facetName, fieldName, comparatorType, size, ordered); + return new InternalShortTermsFacet(facetName, fieldName, comparatorType, size, ordered, aggregator.missing()); } } @@ -198,6 +198,8 @@ public class TermsShortFacetCollector extends AbstractFacetCollector { private final TShortIntHashMap facets; + private int missing; + public StaticAggregatorValueProc(TShortIntHashMap facets) { this.facets = facets; } @@ -206,8 +208,16 @@ public class TermsShortFacetCollector extends AbstractFacetCollector { facets.adjustOrPutValue(value, 1, 1); } + @Override public void onMissing(int docId) { + missing++; + } + public final TShortIntHashMap facets() { return facets; } + + public final int missing() { + return this.missing; + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetCollector.java index 7230007bf2a..408c07c7e41 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetCollector.java @@ -125,7 +125,7 @@ public class FieldsTermsStringFacetCollector extends AbstractFacetCollector { TObjectIntHashMap facets = aggregator.facets(); if (facets.isEmpty()) { TermsStringFacetCollector.pushFacets(facets); - return new InternalStringTermsFacet(facetName, arrayToCommaDelimitedString(fieldsNames), comparatorType, size, ImmutableList.of()); + return new InternalStringTermsFacet(facetName, arrayToCommaDelimitedString(fieldsNames), comparatorType, size, ImmutableList.of(), aggregator.missing()); } else { // we need to fetch facets of "size * numberOfShards" because of problems in how they are distributed across shards BoundedTreeSet ordered = new BoundedTreeSet(comparatorType.comparator(), size * numberOfShards); @@ -134,7 +134,7 @@ public class FieldsTermsStringFacetCollector extends AbstractFacetCollector { ordered.add(new InternalStringTermsFacet.StringEntry(it.key(), it.value())); } TermsStringFacetCollector.pushFacets(facets); - return new InternalStringTermsFacet(facetName, arrayToCommaDelimitedString(fieldsNames), comparatorType, size, ordered); + return new InternalStringTermsFacet(facetName, arrayToCommaDelimitedString(fieldsNames), comparatorType, size, ordered, aggregator.missing()); } } @@ -189,6 +189,8 @@ public class FieldsTermsStringFacetCollector extends AbstractFacetCollector { private final TObjectIntHashMap facets; + private int missing; + public StaticAggregatorValueProc(TObjectIntHashMap facets) { this.facets = facets; } @@ -197,8 +199,16 @@ public class FieldsTermsStringFacetCollector extends AbstractFacetCollector { facets.adjustOrPutValue(value, 1, 1); } + @Override public void onMissing(int docId) { + missing++; + } + public final TObjectIntHashMap facets() { return facets; } + + public final int missing() { + return this.missing; + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/InternalStringTermsFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/InternalStringTermsFacet.java index 1647d493a0f..ec73d40ad27 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/InternalStringTermsFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/InternalStringTermsFacet.java @@ -111,6 +111,8 @@ public class InternalStringTermsFacet extends InternalTermsFacet { int requiredSize; + long missing; + Collection entries = ImmutableList.of(); private ComparatorType comparatorType; @@ -118,12 +120,13 @@ public class InternalStringTermsFacet extends InternalTermsFacet { InternalStringTermsFacet() { } - public InternalStringTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries) { + public InternalStringTermsFacet(String name, String fieldName, ComparatorType comparatorType, int requiredSize, Collection entries, long missing) { this.name = name; this.fieldName = fieldName; this.comparatorType = comparatorType; this.requiredSize = requiredSize; this.entries = entries; + this.missing = missing; } @Override public String name() { @@ -173,6 +176,13 @@ public class InternalStringTermsFacet extends InternalTermsFacet { return (Iterator) entries.iterator(); } + @Override public long missingCount() { + return this.missing; + } + + @Override public long getMissingCount() { + return missingCount(); + } private static ThreadLocal>> aggregateCache = new ThreadLocal>>() { @Override protected ThreadLocals.CleanableValue> initialValue() { @@ -188,9 +198,10 @@ public class InternalStringTermsFacet extends InternalTermsFacet { InternalStringTermsFacet first = (InternalStringTermsFacet) facets.get(0); TObjectIntHashMap aggregated = aggregateCache.get().get(); aggregated.clear(); - + long missing = 0; for (Facet facet : facets) { InternalStringTermsFacet mFacet = (InternalStringTermsFacet) facet; + missing += mFacet.missingCount(); for (InternalStringTermsFacet.StringEntry entry : mFacet.entries) { aggregated.adjustOrPutValue(entry.term(), entry.count(), entry.count()); } @@ -202,12 +213,14 @@ public class InternalStringTermsFacet extends InternalTermsFacet { ordered.add(new StringEntry(it.key(), it.value())); } first.entries = ordered; + first.missing = missing; return first; } static final class Fields { static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); static final XContentBuilderString _FIELD = new XContentBuilderString("_field"); + static final XContentBuilderString MISSING = new XContentBuilderString("missing"); static final XContentBuilderString TERMS = new XContentBuilderString("terms"); static final XContentBuilderString TERM = new XContentBuilderString("term"); static final XContentBuilderString COUNT = new XContentBuilderString("count"); @@ -217,6 +230,7 @@ public class InternalStringTermsFacet extends InternalTermsFacet { builder.startObject(name); builder.field(Fields._TYPE, TermsFacet.TYPE); builder.field(Fields._FIELD, fieldName); + builder.field(Fields.MISSING, missing); builder.startArray(Fields.TERMS); for (Entry entry : entries) { builder.startObject(); @@ -240,6 +254,7 @@ public class InternalStringTermsFacet extends InternalTermsFacet { fieldName = in.readUTF(); comparatorType = ComparatorType.fromId(in.readByte()); requiredSize = in.readVInt(); + missing = in.readVLong(); int size = in.readVInt(); entries = new ArrayList(size); @@ -252,8 +267,8 @@ public class InternalStringTermsFacet extends InternalTermsFacet { out.writeUTF(name); out.writeUTF(fieldName); out.writeByte(comparatorType.id()); - out.writeVInt(requiredSize); + out.writeVLong(missing); out.writeVInt(entries.size()); for (Entry entry : entries) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/ScriptTermsStringFieldFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/ScriptTermsStringFieldFacetCollector.java index 4d91af554a9..6bc6b575cc9 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/ScriptTermsStringFieldFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/ScriptTermsStringFieldFacetCollector.java @@ -56,6 +56,8 @@ public class ScriptTermsStringFieldFacetCollector extends AbstractFacetCollector private final TObjectIntHashMap facets; + private int missing; + public ScriptTermsStringFieldFacetCollector(String facetName, int size, InternalStringTermsFacet.ComparatorType comparatorType, SearchContext context, ImmutableSet excluded, Pattern pattern, String scriptLang, String script, Map params) { super(facetName); @@ -78,26 +80,39 @@ public class ScriptTermsStringFieldFacetCollector extends AbstractFacetCollector @Override protected void doCollect(int doc) throws IOException { Object o = script.execute(doc); if (o == null) { + missing++; return; } if (o instanceof Iterable) { + boolean found = false; for (Object o1 : ((Iterable) o)) { String value = o1.toString(); if (match(value)) { + found = true; facets.adjustOrPutValue(value, 1, 1); } } + if (!found) { + missing++; + } } else if (o instanceof Object[]) { + boolean found = false; for (Object o1 : ((Object[]) o)) { String value = o1.toString(); if (match(value)) { + found = true; facets.adjustOrPutValue(value, 1, 1); } } + if (!found) { + missing++; + } } else { String value = o.toString(); if (match(value)) { facets.adjustOrPutValue(value, 1, 1); + } else { + missing++; } } } @@ -115,7 +130,7 @@ public class ScriptTermsStringFieldFacetCollector extends AbstractFacetCollector @Override public Facet facet() { if (facets.isEmpty()) { TermsStringFacetCollector.pushFacets(facets); - return new InternalStringTermsFacet(facetName, sScript, comparatorType, size, ImmutableList.of()); + return new InternalStringTermsFacet(facetName, sScript, comparatorType, size, ImmutableList.of(), missing); } else { // we need to fetch facets of "size * numberOfShards" because of problems in how they are distributed across shards BoundedTreeSet ordered = new BoundedTreeSet(comparatorType.comparator(), size * numberOfShards); @@ -124,7 +139,7 @@ public class ScriptTermsStringFieldFacetCollector extends AbstractFacetCollector ordered.add(new InternalStringTermsFacet.StringEntry(it.key(), it.value())); } TermsStringFacetCollector.pushFacets(facets); - return new InternalStringTermsFacet(facetName, sScript, comparatorType, size, ordered); + return new InternalStringTermsFacet(facetName, sScript, comparatorType, size, ordered, missing); } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetCollector.java index f5466d19457..a17a60e89ff 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetCollector.java @@ -128,7 +128,7 @@ public class TermsStringFacetCollector extends AbstractFacetCollector { TObjectIntHashMap facets = aggregator.facets(); if (facets.isEmpty()) { pushFacets(facets); - return new InternalStringTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of()); + return new InternalStringTermsFacet(facetName, fieldName, comparatorType, size, ImmutableList.of(), aggregator.missing()); } else { // we need to fetch facets of "size * numberOfShards" because of problems in how they are distributed across shards BoundedTreeSet ordered = new BoundedTreeSet(comparatorType.comparator(), size * numberOfShards); @@ -137,7 +137,7 @@ public class TermsStringFacetCollector extends AbstractFacetCollector { ordered.add(new InternalStringTermsFacet.StringEntry(it.key(), it.value())); } pushFacets(facets); - return new InternalStringTermsFacet(facetName, fieldName, comparatorType, size, ordered); + return new InternalStringTermsFacet(facetName, fieldName, comparatorType, size, ordered, aggregator.missing()); } } @@ -210,6 +210,8 @@ public class TermsStringFacetCollector extends AbstractFacetCollector { private final TObjectIntHashMap facets; + private int missing = 0; + public StaticAggregatorValueProc(TObjectIntHashMap facets) { this.facets = facets; } @@ -218,8 +220,16 @@ public class TermsStringFacetCollector extends AbstractFacetCollector { facets.adjustOrPutValue(value, 1, 1); } + @Override public void onMissing(int docId) { + missing++; + } + public final TObjectIntHashMap facets() { return facets; } + + public final int missing() { + return this.missing; + } } } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java index 386e950adad..c052e9f6ea2 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java @@ -208,6 +208,47 @@ public class SimpleFacetsTests extends AbstractNodesTests { assertThat(facet.count(), equalTo(2l)); } + @Test public void testTermsFacetsMissing() throws Exception { + try { + client.admin().indices().prepareDelete("test").execute().actionGet(); + } catch (Exception e) { + // ignore + } + client.admin().indices().prepareCreate("test") + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("bstag").field("type", "byte").endObject() + .startObject("shstag").field("type", "short").endObject() + .startObject("istag").field("type", "integer").endObject() + .startObject("lstag").field("type", "long").endObject() + .startObject("fstag").field("type", "float").endObject() + .startObject("dstag").field("type", "double").endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + + client.prepareIndex("test", "type1").setSource(jsonBuilder().startObject() + .field("stag", "111") + .field("bstag", 111) + .field("shstag", 111) + .field("istag", 111) + .field("lstag", 111) + .field("fstag", 111.1f) + .field("dstag", 111.1) + .endObject()).execute().actionGet(); + client.prepareIndex("test", "type1").setSource(jsonBuilder().startObject() + .field("kuku", "kuku") + .endObject()).execute().actionGet(); + client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); + + SearchResponse searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addFacet(termsFacet("facet1").field("stag").size(10)) + .execute().actionGet(); + + TermsFacet facet = searchResponse.facets().facet("facet1"); + assertThat(facet.missingCount(), equalTo(1l)); + } + @Test public void testTermsFacets() throws Exception { try { client.admin().indices().prepareDelete("test").execute().actionGet();