From 700a2a9577ac8997684b76613d43f96eeb3f74ce Mon Sep 17 00:00:00 2001 From: kimchy Date: Sun, 13 Mar 2011 02:29:05 +0200 Subject: [PATCH] Sort: Support "missing" specific handling, include _last, _first, and custom value (for numeric values), closes #772. --- .../index/field/data/FieldDataType.java | 3 +- .../bytes/ByteFieldDataMissingComparator.java | 73 ++++++++++++++ .../field/data/bytes/ByteFieldDataType.java | 25 ++++- .../DoubleFieldDataMissingComparator.java | 87 +++++++++++++++++ .../data/doubles/DoubleFieldDataType.java | 25 ++++- .../FloatFieldDataMissingComparator.java | 92 ++++++++++++++++++ .../field/data/floats/FloatFieldDataType.java | 25 ++++- .../ints/IntFieldDataMissingComparator.java | 97 +++++++++++++++++++ .../field/data/ints/IntFieldDataType.java | 25 ++++- .../longs/LongFieldDataMissingComparator.java | 95 ++++++++++++++++++ .../field/data/longs/LongFieldDataType.java | 25 ++++- .../ShortFieldDataMissingComparator.java | 73 ++++++++++++++ .../field/data/shorts/ShortFieldDataType.java | 25 ++++- .../data/strings/StringFieldDataType.java | 6 +- .../xcontent/geo/GeoPointFieldDataType.java | 2 +- .../search/sort/FieldSortBuilder.java | 14 +++ .../search/sort/GeoDistanceSortBuilder.java | 7 ++ .../search/sort/ScoreSortBuilder.java | 4 + .../search/sort/ScriptSortBuilder.java | 7 ++ .../search/sort/SortBuilder.java | 9 ++ .../search/sort/SortParseElement.java | 14 ++- .../search/sort/SimpleSortTests.java | 62 ++++++++++++ 22 files changed, 775 insertions(+), 20 deletions(-) create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataMissingComparator.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataMissingComparator.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataMissingComparator.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataMissingComparator.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataMissingComparator.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataMissingComparator.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/FieldDataType.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/FieldDataType.java index 763dc9f2121..3e4602ac7d9 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/FieldDataType.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/FieldDataType.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.field.data; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.FieldComparatorSource; +import org.elasticsearch.common.Nullable; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.bytes.ByteFieldDataType; import org.elasticsearch.index.field.data.doubles.DoubleFieldDataType; @@ -47,7 +48,7 @@ public interface FieldDataType { public static final DoubleFieldDataType DOUBLE = new DoubleFieldDataType(); } - FieldComparatorSource newFieldComparatorSource(FieldDataCache cache); + FieldComparatorSource newFieldComparatorSource(FieldDataCache cache, @Nullable String missing); T load(IndexReader reader, String fieldName) throws IOException; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataMissingComparator.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataMissingComparator.java new file mode 100644 index 00000000000..ee06dfb74db --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataMissingComparator.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.field.data.bytes; + +import org.elasticsearch.index.cache.field.data.FieldDataCache; +import org.elasticsearch.index.field.data.FieldDataType; +import org.elasticsearch.index.field.data.support.NumericFieldDataComparator; + +/** + * @author kimchy (shay.banon) + */ +// LUCENE MONITOR: Monitor against FieldComparator.Short +public class ByteFieldDataMissingComparator extends NumericFieldDataComparator { + + private final byte[] values; + private short bottom; + private final byte missingValue; + + public ByteFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, byte missingValue) { + super(fieldName, fieldDataCache); + values = new byte[numHits]; + this.missingValue = missingValue; + } + + @Override public FieldDataType fieldDataType() { + return FieldDataType.DefaultTypes.BYTE; + } + + @Override public int compare(int slot1, int slot2) { + return values[slot1] - values[slot2]; + } + + @Override public int compareBottom(int doc) { + byte value = missingValue; + if (currentFieldData.hasValue(doc)) { + value = currentFieldData.byteValue(doc); + } + return bottom - value; + } + + @Override public void copy(int slot, int doc) { + byte value = missingValue; + if (currentFieldData.hasValue(doc)) { + value = currentFieldData.byteValue(doc); + } + values[slot] = value; + } + + @Override public void setBottom(final int bottom) { + this.bottom = values[bottom]; + } + + @Override public Comparable value(int slot) { + return Byte.valueOf(values[slot]); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataType.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataType.java index aa1dcfe92e2..939885af4ea 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataType.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataType.java @@ -32,10 +32,31 @@ import java.io.IOException; */ public class ByteFieldDataType implements FieldDataType { - @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache) { + @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) { + if (missing == null) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new ByteFieldDataComparator(numHits, fieldname, cache); + } + }; + } + if (missing.equals("_last")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new ByteFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Byte.MIN_VALUE : Byte.MAX_VALUE); + } + }; + } + if (missing.equals("_first")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new ByteFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Byte.MAX_VALUE : Byte.MIN_VALUE); + } + }; + } return new FieldComparatorSource() { @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new ByteFieldDataComparator(numHits, fieldname, cache); + return new ByteFieldDataMissingComparator(numHits, fieldname, cache, Byte.parseByte(missing)); } }; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataMissingComparator.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataMissingComparator.java new file mode 100644 index 00000000000..e9b5485d435 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataMissingComparator.java @@ -0,0 +1,87 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.field.data.doubles; + +import org.elasticsearch.index.cache.field.data.FieldDataCache; +import org.elasticsearch.index.field.data.FieldDataType; +import org.elasticsearch.index.field.data.support.NumericFieldDataComparator; + +/** + * @author kimchy (shay.banon) + */ +// LUCENE MONITOR: Monitor against FieldComparator.Double +public class DoubleFieldDataMissingComparator extends NumericFieldDataComparator { + + private final double[] values; + private double bottom; + private final double missingValue; + + public DoubleFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, double missingValue) { + super(fieldName, fieldDataCache); + values = new double[numHits]; + this.missingValue = missingValue; + } + + @Override public FieldDataType fieldDataType() { + return FieldDataType.DefaultTypes.DOUBLE; + } + + @Override public int compare(int slot1, int slot2) { + final double v1 = values[slot1]; + final double v2 = values[slot2]; + if (v1 > v2) { + return 1; + } else if (v1 < v2) { + return -1; + } else { + return 0; + } + } + + @Override public int compareBottom(int doc) { + double v2 = missingValue; + if (currentFieldData.hasValue(doc)) { + v2 = currentFieldData.doubleValue(doc); + } + if (bottom > v2) { + return 1; + } else if (bottom < v2) { + return -1; + } else { + return 0; + } + } + + @Override public void copy(int slot, int doc) { + double value = missingValue; + if (currentFieldData.hasValue(doc)) { + value = currentFieldData.doubleValue(doc); + } + values[slot] = value; + } + + @Override public void setBottom(final int bottom) { + this.bottom = values[bottom]; + } + + @Override public Comparable value(int slot) { + return Double.valueOf(values[slot]); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataType.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataType.java index 7abd9f9ae3a..c96eef52e01 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataType.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataType.java @@ -32,10 +32,31 @@ import java.io.IOException; */ public class DoubleFieldDataType implements FieldDataType { - @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache) { + @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) { + if (missing == null) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new DoubleFieldDataComparator(numHits, fieldname, cache); + } + }; + } + if (missing.equals("_last")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new DoubleFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Double.MIN_VALUE : Double.MAX_VALUE); + } + }; + } + if (missing.equals("_first")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new DoubleFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Double.MAX_VALUE : Double.MIN_VALUE); + } + }; + } return new FieldComparatorSource() { @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new DoubleFieldDataComparator(numHits, fieldname, cache); + return new DoubleFieldDataMissingComparator(numHits, fieldname, cache, Double.parseDouble(missing)); } }; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataMissingComparator.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataMissingComparator.java new file mode 100644 index 00000000000..f3644ad2f38 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataMissingComparator.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.field.data.floats; + +import org.elasticsearch.index.cache.field.data.FieldDataCache; +import org.elasticsearch.index.field.data.FieldDataType; +import org.elasticsearch.index.field.data.support.NumericFieldDataComparator; + +/** + * @author kimchy (shay.banon) + */ +// LUCENE MONITOR - Monitor against FieldComparator.Float +public class FloatFieldDataMissingComparator extends NumericFieldDataComparator { + + private final float[] values; + private float bottom; + private final float missingValue; + + public FloatFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, float missingValue) { + super(fieldName, fieldDataCache); + values = new float[numHits]; + this.missingValue = missingValue; + } + + @Override public FieldDataType fieldDataType() { + return FieldDataType.DefaultTypes.FLOAT; + } + + @Override public int compare(int slot1, int slot2) { + // TODO: are there sneaky non-branch ways to compute + // sign of float? + final float v1 = values[slot1]; + final float v2 = values[slot2]; + if (v1 > v2) { + return 1; + } else if (v1 < v2) { + return -1; + } else { + return 0; + } + } + + @Override public int compareBottom(int doc) { + // TODO: are there sneaky non-branch ways to compute + // sign of float? + float v2 = missingValue; + if (currentFieldData.hasValue(doc)) { + v2 = currentFieldData.floatValue(doc); + } + if (bottom > v2) { + return 1; + } else if (bottom < v2) { + return -1; + } else { + return 0; + } + } + + @Override + public void copy(int slot, int doc) { + float value = missingValue; + if (currentFieldData.hasValue(doc)) { + value = currentFieldData.floatValue(doc); + } + values[slot] = value; + } + + @Override public void setBottom(final int bottom) { + this.bottom = values[bottom]; + } + + @Override public Comparable value(int slot) { + return Float.valueOf(values[slot]); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataType.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataType.java index 240bbe6a187..f58362bf263 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataType.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataType.java @@ -32,10 +32,31 @@ import java.io.IOException; */ public class FloatFieldDataType implements FieldDataType { - @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache) { + @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) { + if (missing == null) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new FloatFieldDataComparator(numHits, fieldname, cache); + } + }; + } + if (missing.equals("_last")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new FloatFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Float.MIN_VALUE : Float.MAX_VALUE); + } + }; + } + if (missing.equals("_first")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new FloatFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Float.MAX_VALUE : Float.MIN_VALUE); + } + }; + } return new FieldComparatorSource() { @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new FloatFieldDataComparator(numHits, fieldname, cache); + return new FloatFieldDataMissingComparator(numHits, fieldname, cache, Float.parseFloat(missing)); } }; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataMissingComparator.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataMissingComparator.java new file mode 100644 index 00000000000..1721b525f17 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataMissingComparator.java @@ -0,0 +1,97 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.field.data.ints; + +import org.elasticsearch.index.cache.field.data.FieldDataCache; +import org.elasticsearch.index.field.data.FieldDataType; +import org.elasticsearch.index.field.data.support.NumericFieldDataComparator; + +/** + * @author kimchy (shay.banon) + */ +// LUCENE MONITOR - Monitor against FieldComparator.Int +public class IntFieldDataMissingComparator extends NumericFieldDataComparator { + + private final int[] values; + + private int bottom; // Value of bottom of queue + private final int missingValue; + + public IntFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, int missingValue) { + super(fieldName, fieldDataCache); + values = new int[numHits]; + this.missingValue = missingValue; + } + + @Override public FieldDataType fieldDataType() { + return FieldDataType.DefaultTypes.INT; + } + + @Override public int compare(int slot1, int slot2) { + // TODO: there are sneaky non-branch ways to compute + // -1/+1/0 sign + // Cannot return values[slot1] - values[slot2] because that + // may overflow + final int v1 = values[slot1]; + final int v2 = values[slot2]; + if (v1 > v2) { + return 1; + } else if (v1 < v2) { + return -1; + } else { + return 0; + } + } + + @Override public int compareBottom(int doc) { + // TODO: there are sneaky non-branch ways to compute + // -1/+1/0 sign + // Cannot return bottom - values[slot2] because that + // may overflow +// final int v2 = currentReaderValues[doc]; + int v2 = missingValue; + if (currentFieldData.hasValue(doc)) { + v2 = currentFieldData.intValue(doc); + } + if (bottom > v2) { + return 1; + } else if (bottom < v2) { + return -1; + } else { + return 0; + } + } + + @Override public void copy(int slot, int doc) { + int value = missingValue; + if (currentFieldData.hasValue(doc)) { + value = currentFieldData.intValue(doc); + } + values[slot] = value; + } + + @Override public void setBottom(final int bottom) { + this.bottom = values[bottom]; + } + + @Override public Comparable value(int slot) { + return Integer.valueOf(values[slot]); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataType.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataType.java index 9902ac92ec7..ac147bf95a3 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataType.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataType.java @@ -32,10 +32,31 @@ import java.io.IOException; */ public class IntFieldDataType implements FieldDataType { - @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache) { + @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) { + if (missing == null) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new IntFieldDataComparator(numHits, fieldname, cache); + } + }; + } + if (missing.equals("_last")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new IntFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Integer.MIN_VALUE : Integer.MAX_VALUE); + } + }; + } + if (missing.equals("_first")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new IntFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Integer.MAX_VALUE : Integer.MIN_VALUE); + } + }; + } return new FieldComparatorSource() { @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new IntFieldDataComparator(numHits, fieldname, cache); + return new IntFieldDataMissingComparator(numHits, fieldname, cache, Integer.parseInt(missing)); } }; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataMissingComparator.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataMissingComparator.java new file mode 100644 index 00000000000..b5c2051ba66 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataMissingComparator.java @@ -0,0 +1,95 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.field.data.longs; + +import org.elasticsearch.index.cache.field.data.FieldDataCache; +import org.elasticsearch.index.field.data.FieldDataType; +import org.elasticsearch.index.field.data.support.NumericFieldDataComparator; + +/** + * @author kimchy (shay.banon) + */ +// LUCENE MONITOR - Monitor against FieldComparator.Long +public class LongFieldDataMissingComparator extends NumericFieldDataComparator { + + private final long[] values; + private long bottom; + private final long missingValue; + + public LongFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, long missingValue) { + super(fieldName, fieldDataCache); + values = new long[numHits]; + this.missingValue = missingValue; + } + + @Override public FieldDataType fieldDataType() { + return FieldDataType.DefaultTypes.LONG; + } + + @Override public int compare(int slot1, int slot2) { + // TODO: there are sneaky non-branch ways to compute + // -1/+1/0 sign + final long v1 = values[slot1]; + final long v2 = values[slot2]; + if (v1 > v2) { + return 1; + } else if (v1 < v2) { + return -1; + } else { + return 0; + } + } + + @Override + public int compareBottom(int doc) { + // TODO: there are sneaky non-branch ways to compute + // -1/+1/0 sign +// final long v2 = currentReaderValues[doc]; + long v2 = missingValue; + if (currentFieldData.hasValue(doc)) { + v2 = currentFieldData.longValue(doc); + } + if (bottom > v2) { + return 1; + } else if (bottom < v2) { + return -1; + } else { + return 0; + } + } + + @Override + public void copy(int slot, int doc) { + long value = missingValue; + if (currentFieldData.hasValue(doc)) { + value = currentFieldData.longValue(doc); + } + values[slot] = value; + } + + @Override public void setBottom(final int bottom) { + this.bottom = values[bottom]; + } + + @Override public Comparable value(int slot) { + return Long.valueOf(values[slot]); + } + +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataType.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataType.java index d57fb2afee6..f381a742771 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataType.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataType.java @@ -32,10 +32,31 @@ import java.io.IOException; */ public class LongFieldDataType implements FieldDataType { - @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache) { + @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) { + if (missing == null) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new LongFieldDataComparator(numHits, fieldname, cache); + } + }; + } + if (missing.equals("_last")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new LongFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Long.MIN_VALUE : Long.MAX_VALUE); + } + }; + } + if (missing.equals("_first")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new LongFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Long.MAX_VALUE : Long.MIN_VALUE); + } + }; + } return new FieldComparatorSource() { @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new LongFieldDataComparator(numHits, fieldname, cache); + return new LongFieldDataMissingComparator(numHits, fieldname, cache, Long.parseLong(missing)); } }; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataMissingComparator.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataMissingComparator.java new file mode 100644 index 00000000000..ec77ce6b43c --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataMissingComparator.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.field.data.shorts; + +import org.elasticsearch.index.cache.field.data.FieldDataCache; +import org.elasticsearch.index.field.data.FieldDataType; +import org.elasticsearch.index.field.data.support.NumericFieldDataComparator; + +/** + * @author kimchy (shay.banon) + */ +// LUCENE MONITOR: Monitor against FieldComparator.Short +public class ShortFieldDataMissingComparator extends NumericFieldDataComparator { + + private final short[] values; + private short bottom; + private final short missingValue; + + public ShortFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, short missingValue) { + super(fieldName, fieldDataCache); + values = new short[numHits]; + this.missingValue = missingValue; + } + + @Override public FieldDataType fieldDataType() { + return FieldDataType.DefaultTypes.SHORT; + } + + @Override public int compare(int slot1, int slot2) { + return values[slot1] - values[slot2]; + } + + @Override public int compareBottom(int doc) { + short value = missingValue; + if (currentFieldData.hasValue(doc)) { + value = currentFieldData.shortValue(doc); + } + return bottom - value; + } + + @Override public void copy(int slot, int doc) { + short value = missingValue; + if (currentFieldData.hasValue(doc)) { + value = currentFieldData.shortValue(doc); + } + values[slot] = value; + } + + @Override public void setBottom(final int bottom) { + this.bottom = values[bottom]; + } + + @Override public Comparable value(int slot) { + return Short.valueOf(values[slot]); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataType.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataType.java index 1c8bcd1ba1e..78d9169c4c0 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataType.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataType.java @@ -32,10 +32,31 @@ import java.io.IOException; */ public class ShortFieldDataType implements FieldDataType { - @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache) { + @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) { + if (missing == null) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new ShortFieldDataComparator(numHits, fieldname, cache); + } + }; + } + if (missing.equals("_last")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new ShortFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Short.MIN_VALUE : Short.MAX_VALUE); + } + }; + } + if (missing.equals("_first")) { + return new FieldComparatorSource() { + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new ShortFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Short.MAX_VALUE : Short.MIN_VALUE); + } + }; + } return new FieldComparatorSource() { @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new ShortFieldDataComparator(numHits, fieldname, cache); + return new ShortFieldDataMissingComparator(numHits, fieldname, cache, Short.parseShort(missing)); } }; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/StringFieldDataType.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/StringFieldDataType.java index 529808dad02..5662cdb4a65 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/StringFieldDataType.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/StringFieldDataType.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.field.data.strings; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldComparatorSource; +import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.field.data.FieldDataType; @@ -32,7 +33,10 @@ import java.io.IOException; */ public class StringFieldDataType implements FieldDataType { - @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache) { + @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) { + if (missing != null) { + throw new ElasticSearchIllegalArgumentException("Sorting on string type field does not support missing parameter"); + } return new FieldComparatorSource() { @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { return new StringOrdValFieldDataComparator(numHits, fieldname, sortPos, reversed, cache); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/GeoPointFieldDataType.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/GeoPointFieldDataType.java index b2cabc6a441..c4b2e87edfe 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/GeoPointFieldDataType.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/GeoPointFieldDataType.java @@ -35,7 +35,7 @@ public class GeoPointFieldDataType implements FieldDataType { public static final GeoPointFieldDataType TYPE = new GeoPointFieldDataType(); - @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache) { + @Override public FieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) { return new FieldComparatorSource() { @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { return new StringOrdValFieldDataComparator(numHits, fieldname, sortPos, reversed, cache); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index cc825bcc76d..1ff6feeae64 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -34,6 +34,8 @@ public class FieldSortBuilder extends SortBuilder { private SortOrder order; + private Object missing; + /** * Constructs a new sort based on a document field. * @@ -51,11 +53,23 @@ public class FieldSortBuilder extends SortBuilder { return this; } + /** + * Sets the value when a field is missing in a doc. Can also be set to _last or + * _first to sort missing last or first respectively. + */ + public FieldSortBuilder missing(Object missing) { + this.missing = missing; + return this; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(fieldName); if (order == SortOrder.DESC) { builder.field("reverse", true); } + if (missing != null) { + builder.field("missing", missing); + } builder.endObject(); return builder; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index 3d9a5a56ec3..d465e5f8fed 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -95,6 +95,13 @@ public class GeoDistanceSortBuilder extends SortBuilder { return this; } + /** + * Not relevant. + */ + @Override public SortBuilder missing(Object missing) { + return this; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("_geo_distance"); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java index 66477ac9eb7..272dd4b2c3e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java @@ -40,6 +40,10 @@ public class ScoreSortBuilder extends SortBuilder { return this; } + @Override public SortBuilder missing(Object missing) { + return this; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("_score"); if (order == SortOrder.ASC) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index fd975b1f845..d840d50975a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -75,6 +75,13 @@ public class ScriptSortBuilder extends SortBuilder { return this; } + /** + * Not really relevant. + */ + @Override public SortBuilder missing(Object missing) { + return this; + } + /** * The language of the script. */ diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/SortBuilder.java index 95cc0be557d..f193615a3ff 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/SortBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/SortBuilder.java @@ -26,5 +26,14 @@ import org.elasticsearch.common.xcontent.ToXContent; */ public abstract class SortBuilder implements ToXContent { + /** + * The order of sorting. Defaults to {@link SortOrder#ASC}. + */ public abstract SortBuilder order(SortOrder order); + + /** + * Sets the value when a field is missing in a doc. Can also be set to _last or + * _first to sort missing last or first respectively. + */ + public abstract SortBuilder missing(Object missing); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/SortParseElement.java index 7f7a199f00b..f5e1cba9833 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/SortParseElement.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/sort/SortParseElement.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.sort; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableMap; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.xcontent.XContentParser; @@ -67,7 +68,7 @@ public class SortParseElement implements SearchParseElement { if (token == XContentParser.Token.START_OBJECT) { addCompoundSortField(parser, context, sortFields); } else if (token == XContentParser.Token.VALUE_STRING) { - addSortField(context, sortFields, parser.text(), false); + addSortField(context, sortFields, parser.text(), false, null); } } } else { @@ -84,6 +85,7 @@ public class SortParseElement implements SearchParseElement { if (token == XContentParser.Token.FIELD_NAME) { String fieldName = parser.currentName(); boolean reverse = false; + String missing = null; String innerJsonName = null; token = parser.nextToken(); if (token == XContentParser.Token.VALUE_STRING) { @@ -93,7 +95,7 @@ public class SortParseElement implements SearchParseElement { } else if (direction.equals("desc")) { reverse = !SCORE_FIELD_NAME.equals(fieldName); } - addSortField(context, sortFields, fieldName, reverse); + addSortField(context, sortFields, fieldName, reverse, missing); } else { if (parsers.containsKey(fieldName)) { sortFields.add(parsers.get(fieldName).parse(parser, context)); @@ -110,17 +112,19 @@ public class SortParseElement implements SearchParseElement { } else if ("desc".equals(parser.text())) { reverse = !SCORE_FIELD_NAME.equals(fieldName); } + } else if ("missing".equals(innerJsonName)) { + missing = parser.textOrNull(); } } } - addSortField(context, sortFields, fieldName, reverse); + addSortField(context, sortFields, fieldName, reverse, missing); } } } } } - private void addSortField(SearchContext context, List sortFields, String fieldName, boolean reverse) { + private void addSortField(SearchContext context, List sortFields, String fieldName, boolean reverse, @Nullable final String missing) { if (SCORE_FIELD_NAME.equals(fieldName)) { if (reverse) { sortFields.add(SORT_SCORE_REVERSE); @@ -138,7 +142,7 @@ public class SortParseElement implements SearchParseElement { if (fieldMapper == null) { throw new SearchParseException(context, "No mapping found for [" + fieldName + "]"); } - sortFields.add(new SortField(fieldName, fieldMapper.fieldDataType().newFieldComparatorSource(context.fieldDataCache()), reverse)); + sortFields.add(new SortField(fieldName, fieldMapper.fieldDataType().newFieldComparatorSource(context.fieldDataCache(), missing), reverse)); } } } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java index c4464f47fbd..230241cc006 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterClass; @@ -438,4 +439,65 @@ public class SimpleSortTests extends AbstractNodesTests { assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); assertThat((String) searchResponse.hits().getAt(0).field("id").value(), equalTo("2")); } + + @Test public void testSortMissing() throws Exception { + try { + client.admin().indices().prepareDelete("test").execute().actionGet(); + } catch (Exception e) { + // ignore + } + client.admin().indices().prepareCreate("test").execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + + client.prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() + .field("id", "1") + .field("i_value", -1) + .field("d_value", -1.1) + .endObject()).execute().actionGet(); + + client.prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject() + .field("id", "2") + .endObject()).execute().actionGet(); + + client.prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject() + .field("id", "1") + .field("i_value", 2) + .field("d_value", 2.2) + .endObject()).execute().actionGet(); + + client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet(); + + logger.info("--> sort with no missing"); + SearchResponse searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC)) + .execute().actionGet(); + + assertThat(searchResponse.hits().getTotalHits(), equalTo(3l)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.hits().getAt(1).id(), equalTo("2")); + assertThat(searchResponse.hits().getAt(2).id(), equalTo("3")); + + logger.info("--> sort with missing _last"); + searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_last")) + .execute().actionGet(); + + assertThat(searchResponse.hits().getTotalHits(), equalTo(3l)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.hits().getAt(1).id(), equalTo("3")); + assertThat(searchResponse.hits().getAt(2).id(), equalTo("2")); + + logger.info("--> sort with missing _first"); + searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_first")) + .execute().actionGet(); + + assertThat(searchResponse.hits().getTotalHits(), equalTo(3l)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("2")); + assertThat(searchResponse.hits().getAt(1).id(), equalTo("1")); + assertThat(searchResponse.hits().getAt(2).id(), equalTo("3")); + } }