diff --git a/lucene/common-build.xml b/lucene/common-build.xml index 87d2e0a7387..48cf4572669 100644 --- a/lucene/common-build.xml +++ b/lucene/common-build.xml @@ -1073,6 +1073,7 @@ + diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 205c7bc6b03..82c3d2b63de 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -74,6 +74,10 @@ Optimizations * SOLR-9584: Support Solr being proxied with another endpoint than default /solr, by using relative links in AdminUI javascripts (Yun Jie Zhou via janhoy) +Other Changes +---------------------- +* SOLR-8396: Add support for PointFields in Solr (Ishan Chattopadhyaya, Tomás Fernández Löbbe) + ================== 6.5.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java index 50f46ef8d02..7f08684a59e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java @@ -289,8 +289,6 @@ public class LukeRequestHandler extends RequestHandlerBase f.add( "schema", getFieldFlags( sfield ) ); f.add( "flags", getFieldFlags( field ) ); - Term t = new Term(field.name(), ftype!=null ? ftype.storedToIndexed(field) : field.stringValue()); - f.add( "value", (ftype==null)?null:ftype.toExternal( field ) ); // TODO: this really should be "stored" @@ -301,7 +299,10 @@ public class LukeRequestHandler extends RequestHandlerBase f.add( "binary", Base64.byteArrayToBase64(bytes.bytes, bytes.offset, bytes.length)); } f.add( "boost", field.boost() ); - f.add( "docFreq", t.text()==null ? 0 : reader.docFreq( t ) ); // this can be 0 for non-indexed fields + if (!ftype.isPointField()) { + Term t = new Term(field.name(), ftype!=null ? ftype.storedToIndexed(field) : field.stringValue()); + f.add( "docFreq", t.text()==null ? 0 : reader.docFreq( t ) ); // this can be 0 for non-indexed fields + }// TODO: Calculate docFreq for point fields // If we have a term vector, return that if( field.fieldType().storeTermVectors() ) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java index 1cc05ab78cc..bcff0c28644 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java @@ -33,6 +33,7 @@ import java.util.Map.Entry; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; @@ -47,6 +48,7 @@ import org.apache.solr.common.util.StrUtils; import org.apache.solr.request.SimpleFacets; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.PointField; import org.apache.solr.search.QueryParsing; import org.apache.solr.search.SyntaxError; import org.apache.solr.search.facet.FacetDebugInfo; @@ -1477,7 +1479,13 @@ public class FacetComponent extends SearchComponent { if (sfc == null) { sfc = new ShardFacetCount(); sfc.name = name; - sfc.indexed = ftype == null ? sfc.name : ftype.toInternal(sfc.name); + if (ftype == null) { + sfc.indexed = null; + } else if (ftype.isPointField()) { + sfc.indexed = ((PointField)ftype).toInternalByteRef(sfc.name); + } else { + sfc.indexed = new BytesRef(ftype.toInternal(sfc.name)); + } sfc.termNum = termNum++; counts.put(name, sfc); } @@ -1553,7 +1561,7 @@ public class FacetComponent extends SearchComponent { public static class ShardFacetCount { public String name; // the indexed form of the name... used for comparisons - public String indexed; + public BytesRef indexed; public long count; public int termNum; // term number starting at 0 (used in bit arrays) diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 88ff7310ae9..c357202b6a1 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -185,6 +185,11 @@ public class QueryComponent extends SearchComponent } rb.setSortSpec( parser.getSortSpec(true) ); + for (SchemaField sf:rb.getSortSpec().getSchemaFields()) { + if (sf != null && sf.getType().isPointField() && !sf.hasDocValues()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"Can't sort on a point field without docValues"); + } + } rb.setQparser(parser); final String cursorStr = rb.req.getParams().get(CursorMarkParams.CURSOR_MARK_PARAM); @@ -335,11 +340,21 @@ public class QueryComponent extends SearchComponent List idArr = StrUtils.splitSmart(ids, ",", true); int[] luceneIds = new int[idArr.size()]; int docs = 0; - for (int i=0; i= 0) - luceneIds[docs++] = id; + if (idField.getType().isPointField()) { + for (int i=0; i= 0) { + luceneIds[docs++] = id; + } + } + } else { + for (int i=0; i= 0) + luceneIds[docs++] = id; + } } DocListAndSet res = new DocListAndSet(); diff --git a/solr/core/src/java/org/apache/solr/handler/component/RangeFacetProcessor.java b/solr/core/src/java/org/apache/solr/handler/component/RangeFacetProcessor.java index 731d2240ec7..f8ab7b71811 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/RangeFacetProcessor.java +++ b/solr/core/src/java/org/apache/solr/handler/component/RangeFacetProcessor.java @@ -55,7 +55,6 @@ public class RangeFacetProcessor extends SimpleFacets { * * @see org.apache.solr.common.params.FacetParams#FACET_RANGE */ - @SuppressWarnings("unchecked") public NamedList getFacetRangeCounts() throws IOException, SyntaxError { final NamedList resOuter = new SimpleOrderedMap<>(); @@ -92,7 +91,7 @@ public class RangeFacetProcessor extends SimpleFacets { final FieldType ft = sf.getType(); if (method.equals(FacetRangeMethod.DV)) { - assert ft instanceof TrieField; + assert ft instanceof TrieField || ft.isPointField(); resOuter.add(key, getFacetRangeCountsDocValues(rangeFacetRequest)); } else { resOuter.add(key, getFacetRangeCounts(rangeFacetRequest)); diff --git a/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java b/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java index 8c0c3812b45..f129e739a5d 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java +++ b/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java @@ -34,6 +34,7 @@ import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.schema.DateRangeField; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.PointField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TrieDateField; import org.apache.solr.schema.TrieField; @@ -91,6 +92,11 @@ public class RangeFacetRequest extends FacetComponent.FacetBase { DateRangeField.class + "'. Will use method '" + FacetParams.FacetRangeMethod.FILTER + "' instead"); method = FacetParams.FacetRangeMethod.FILTER; } + if (method.equals(FacetParams.FacetRangeMethod.DV) && !schemaField.hasDocValues() && (schemaField.getType().isPointField())) { + log.warn("Range facet method '" + FacetParams.FacetRangeMethod.DV + "' is not supported on PointFields without docValues." + + "Will use method '" + FacetParams.FacetRangeMethod.FILTER + "' instead"); + method = FacetParams.FacetRangeMethod.FILTER; + } this.start = required.getFieldParam(facetOn, FacetParams.FACET_RANGE_START); this.end = required.getFieldParam(facetOn, FacetParams.FACET_RANGE_END); @@ -159,10 +165,33 @@ public class RangeFacetRequest extends FacetComponent.FacetBase { default: throw new SolrException (SolrException.ErrorCode.BAD_REQUEST, - "Unable to range facet on tried field of unexpected type:" + this.facetOn); + "Unable to range facet on Trie field of unexpected type:" + this.facetOn); } } else if (ft instanceof DateRangeField) { calc = new DateRangeEndpointCalculator(this, null); + } else if (ft.isPointField()) { + final PointField pointField = (PointField) ft; + switch (pointField.getType()) { + case FLOAT: + calc = new FloatRangeEndpointCalculator(this); + break; + case DOUBLE: + calc = new DoubleRangeEndpointCalculator(this); + break; + case INTEGER: + calc = new IntegerRangeEndpointCalculator(this); + break; + case LONG: + calc = new LongRangeEndpointCalculator(this); + break; + case DATE: + calc = new DateRangeEndpointCalculator(this, null); + break; + default: + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "Unable to range facet on Point field of unexpected type:" + this.facetOn); + } } else { throw new SolrException (SolrException.ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java b/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java index 68284c7bcb2..6a6e9bef0d6 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java @@ -45,6 +45,12 @@ public class StatsComponent extends SearchComponent { rb.setNeedDocSet( true ); rb.doStats = true; rb._statsInfo = new StatsInfo(rb); + for (StatsField statsField : rb._statsInfo.getStatsFields()) { + if (statsField.getSchemaField() != null && statsField.getSchemaField().getType().isPointField() && !statsField.getSchemaField().hasDocValues()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Can't calculate stats on a PointField without docValues"); + } + } } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java index 8a35ee0d404..7605f73902a 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -65,7 +65,7 @@ public class StatsValuesFactory { if (TrieDateField.class.isInstance(fieldType)) { return new DateStatsValues(statsField); - } else if (TrieField.class.isInstance(fieldType)) { + } else if (TrieField.class.isInstance(fieldType) || PointField.class.isInstance(fieldType)) { return new NumericStatsValues(statsField); } else if (StrField.class.isInstance(fieldType)) { return new StringStatsValues(statsField); diff --git a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java index 5031faf0125..12f5bd112ea 100644 --- a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java +++ b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java @@ -65,9 +65,6 @@ public final class SlowCompositeReaderWrapper extends LeafReader { SlowCompositeReaderWrapper(CompositeReader reader, boolean merging) throws IOException { super(); in = reader; - if (getFieldInfos().hasPointValues()) { - throw new IllegalArgumentException("cannot wrap points"); - } fields = MultiFields.getFields(in); in.registerParentReader(this); this.merging = merging; diff --git a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java index 14bf700bee7..88e39fcf5ac 100644 --- a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java +++ b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java @@ -42,6 +42,7 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.request.IntervalFacets.FacetInterval; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.PointField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TrieDateField; import org.apache.solr.search.DocIterator; @@ -625,6 +626,9 @@ public class IntervalFacets implements Iterable { if ("*".equals(value)) { return null; } + if (schemaField.getType().isPointField()) { + return ((PointField)schemaField.getType()).toInternalByteRef(value); + } return new BytesRef(schemaField.getType().toInternal(value)); } diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java index 641b1f39402..0d9cb29e533 100644 --- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java +++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java @@ -411,6 +411,10 @@ public class SimpleFacets { NamedList counts; SchemaField sf = searcher.getSchema().getField(field); + if (sf.getType().isPointField() && !sf.hasDocValues()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Can't facet on a PointField without docValues"); + } FieldType ft = sf.getType(); // determine what type of faceting method to use @@ -579,6 +583,10 @@ public class SimpleFacets { static FacetMethod selectFacetMethod(SchemaField field, FacetMethod method, Integer mincount) { FieldType type = field.getType(); + if (type.isPointField()) { + // Only FCS is supported for PointFields for now + return FacetMethod.FCS; + } /*The user did not specify any preference*/ if (method == null) { @@ -810,12 +818,20 @@ public class SimpleFacets { * @param terms a list of term values (in the specified field) to compute the counts for */ protected NamedList getListedTermCounts(String field, final ParsedParams parsed, List terms) throws IOException { - FieldType ft = searcher.getSchema().getFieldType(field); + SchemaField sf = searcher.getSchema().getField(field); + FieldType ft = sf.getType(); NamedList res = new NamedList<>(); - for (String term : terms) { - String internal = ft.toInternal(term); - int count = searcher.numDocs(new TermQuery(new Term(field, internal)), parsed.docs); - res.add(term, count); + if (ft.isPointField()) { + for (String term : terms) { + int count = searcher.numDocs(ft.getFieldQuery(null, sf, term), parsed.docs); + res.add(term, count); + } + } else { + for (String term : terms) { + String internal = ft.toInternal(term); + int count = searcher.numDocs(new TermQuery(new Term(field, internal)), parsed.docs); + res.add(term, count); + } } return res; } @@ -848,7 +864,7 @@ public class SimpleFacets { public NamedList getFacetTermEnumCounts(SolrIndexSearcher searcher, DocSet docs, String field, int offset, int limit, int mincount, boolean missing, String sort, String prefix, String contains, boolean ignoreCase, boolean intersectsCheck) throws IOException { - + /* :TODO: potential optimization... * cache the Terms with the highest docFreq and try them first * don't enum if we get our max from them @@ -864,10 +880,12 @@ public class SimpleFacets { fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size()); } - IndexSchema schema = searcher.getSchema(); - LeafReader r = searcher.getSlowAtomicReader(); FieldType ft = schema.getFieldType(field); + assert !ft.isPointField(): "Point Fields don't support enum method"; + + LeafReader r = searcher.getSlowAtomicReader(); + boolean sortByCount = sort.equals("count") || sort.equals("true"); final int maxsize = limit>=0 ? offset+limit : Integer.MAX_VALUE-1; @@ -1082,6 +1100,9 @@ public class SimpleFacets { if (parsed.params.getBool(GroupParams.GROUP_FACET, false)) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Interval Faceting can't be used with " + GroupParams.GROUP_FACET); } + if (schemaField.getType().isPointField() && !schemaField.hasDocValues()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can't use interval faceting on a PointField without docValues"); + } SimpleOrderedMap fieldResults = new SimpleOrderedMap(); res.add(parsed.key, fieldResults); diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java index bee699ca749..ef0b0c71960 100644 --- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java +++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java @@ -31,8 +31,12 @@ import org.apache.solr.common.SolrException; import org.apache.solr.response.transform.DocTransformer; import org.apache.solr.schema.BinaryField; import org.apache.solr.schema.BoolField; +import org.apache.solr.schema.DoublePointField; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.FloatPointField; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.IntPointField; +import org.apache.solr.schema.LongPointField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.StrField; import org.apache.solr.schema.TextField; @@ -213,6 +217,10 @@ public class DocsStreamer implements Iterator { KNOWN_TYPES.add(TrieDoubleField.class); KNOWN_TYPES.add(TrieDateField.class); KNOWN_TYPES.add(BinaryField.class); + KNOWN_TYPES.add(IntPointField.class); + KNOWN_TYPES.add(LongPointField.class); + KNOWN_TYPES.add(DoublePointField.class); + KNOWN_TYPES.add(FloatPointField.class); // We do not add UUIDField because UUID object is not a supported type in JavaBinCodec // and if we write UUIDField.toObject, we wouldn't know how to handle it in the client side } diff --git a/solr/core/src/java/org/apache/solr/schema/DoublePointField.java b/solr/core/src/java/org/apache/solr/schema/DoublePointField.java new file mode 100644 index 00000000000..c393dfef4c0 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/DoublePointField.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.lang.invoke.MethodHandles; +import java.util.Collection; + +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.legacy.LegacyNumericType; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.DoubleFieldSource; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * {@code PointField} implementation for {@code Double} values. + * @see PointField + * @see DoublePoint + */ +public class DoublePointField extends PointField implements DoubleValueFieldType { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Override + public Object toNativeType(Object val) { + if (val == null) return null; + if (val instanceof Number) return ((Number) val).doubleValue(); + if (val instanceof String) return Double.parseDouble((String) val); + return super.toNativeType(val); + } + + @Override + public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, + boolean maxInclusive) { + double actualMin, actualMax; + if (min == null) { + actualMin = Double.NEGATIVE_INFINITY; + } else { + actualMin = Double.parseDouble(min); + if (!minInclusive) { + actualMin = DoublePoint.nextUp(actualMin); + } + } + if (max == null) { + actualMax = Double.POSITIVE_INFINITY; + } else { + actualMax = Double.parseDouble(max); + if (!maxInclusive) { + actualMax = DoublePoint.nextDown(actualMax); + } + } + return DoublePoint.newRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Object toObject(SchemaField sf, BytesRef term) { + return DoublePoint.decodeDimension(term.bytes, term.offset); + } + + @Override + public Object toObject(IndexableField f) { + final Number val = f.numericValue(); + if (val != null) { + if (f.fieldType().stored() == false && f.fieldType().docValuesType() == DocValuesType.NUMERIC) { + return Double.longBitsToDouble(val.longValue()); + } else { + return val; + } + } else { + throw new AssertionError("Unexpected state. Field: '" + f + "'"); + } + } + + @Override + protected Query getExactQuery(SchemaField field, String externalVal) { + return DoublePoint.newExactQuery(field.getName(), Double.parseDouble(externalVal)); + } + + @Override + public Query getSetQuery(QParser parser, SchemaField field, Collection externalVal) { + assert externalVal.size() > 0; + double[] values = new double[externalVal.size()]; + int i = 0; + for (String val:externalVal) { + values[i] = Double.parseDouble(val); + i++; + } + return DoublePoint.newSetQuery(field.getName(), values); + } + + @Override + protected String indexedToReadable(BytesRef indexedForm) { + return Double.toString(DoublePoint.decodeDimension(indexedForm.bytes, indexedForm.offset)); + } + + @Override + public void readableToIndexed(CharSequence val, BytesRefBuilder result) { + result.grow(Double.BYTES); + result.setLength(Double.BYTES); + DoublePoint.encodeDimension(Double.parseDouble(val.toString()), result.bytes(), 0); + } + + @Override + public SortField getSortField(SchemaField field, boolean top) { + field.checkSortability(); + + Object missingValue = null; + boolean sortMissingLast = field.sortMissingLast(); + boolean sortMissingFirst = field.sortMissingFirst(); + + if (sortMissingLast) { + missingValue = top ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY; + } else if (sortMissingFirst) { + missingValue = top ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; + } + SortField sf = new SortField(field.getName(), SortField.Type.DOUBLE, top); + sf.setMissingValue(missingValue); + return sf; + } + + @Override + public Type getUninversionType(SchemaField sf) { + if (sf.multiValued()) { + throw new UnsupportedOperationException("MultiValued Point fields with DocValues is not currently supported"); +// return Type.SORTED_DOUBLE; + } else { + return Type.DOUBLE_POINT; + } + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser qparser) { + field.checkFieldCacheSource(); + return new DoubleFieldSource(field.getName()); + } + + @Override + public LegacyNumericType getNumericType() { + // TODO: refactor this to not use LegacyNumericType + return LegacyNumericType.DOUBLE; + } + + @Override + public IndexableField createField(SchemaField field, Object value, float boost) { + if (!isFieldUsed(field)) return null; + + if (boost != 1.0 && log.isTraceEnabled()) { + log.trace("Can't use document/field boost for PointField. Field: " + field.getName() + ", boost: " + boost); + } + double doubleValue = (value instanceof Number) ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()); + return new DoublePoint(field.getName(), doubleValue); + } + + @Override + protected StoredField getStoredField(SchemaField sf, Object value) { + return new StoredField(sf.getName(), (Double) this.toNativeType(value)); + } + + @Override + public PointTypes getType() { + return PointTypes.DOUBLE; + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index a5c898a40a5..3922edc1060 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -126,6 +126,10 @@ public abstract class FieldType extends FieldProperties { public boolean isPolyField(){ return false; } + + public boolean isPointField() { + return false; + } /** * Returns true if the fields' docValues should be used for obtaining stored value @@ -395,7 +399,10 @@ public abstract class FieldType extends FieldProperties { return toInternal(val); } - /** Given the readable value, return the term value that will match it. */ + /** Given the readable value, return the term value that will match it. + * This method will modify the size and length of the {@code result} + * parameter and write from offset 0 + */ public void readableToIndexed(CharSequence val, BytesRefBuilder result) { final String internal = readableToIndexed(val.toString()); result.copyChars(internal); diff --git a/solr/core/src/java/org/apache/solr/schema/FloatPointField.java b/solr/core/src/java/org/apache/solr/schema/FloatPointField.java new file mode 100644 index 00000000000..766c6e9f441 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/FloatPointField.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.lang.invoke.MethodHandles; +import java.util.Collection; + +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.legacy.LegacyNumericType; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.FloatFieldSource; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * {@code PointField} implementation for {@code Float} values. + * @see PointField + * @see FloatPoint + */ +public class FloatPointField extends PointField implements FloatValueFieldType { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Override + public Object toNativeType(Object val) { + if (val == null) return null; + if (val instanceof Number) return ((Number) val).floatValue(); + if (val instanceof String) return Float.parseFloat((String) val); + return super.toNativeType(val); + } + + @Override + public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, + boolean maxInclusive) { + float actualMin, actualMax; + if (min == null) { + actualMin = Float.NEGATIVE_INFINITY; + } else { + actualMin = Float.parseFloat(min); + if (!minInclusive) { + actualMin = FloatPoint.nextUp(actualMin); + } + } + if (max == null) { + actualMax = Float.POSITIVE_INFINITY; + } else { + actualMax = Float.parseFloat(max); + if (!maxInclusive) { + actualMax = FloatPoint.nextDown(actualMax); + } + } + return FloatPoint.newRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Object toObject(SchemaField sf, BytesRef term) { + return FloatPoint.decodeDimension(term.bytes, term.offset); + } + + @Override + public Object toObject(IndexableField f) { + final Number val = f.numericValue(); + if (val != null) { + if (f.fieldType().stored() == false && f.fieldType().docValuesType() == DocValuesType.NUMERIC) { + return Float.intBitsToFloat(val.intValue()); + } else { + return val; + } + } else { + throw new AssertionError("Unexpected state. Field: '" + f + "'"); + } + } + + @Override + protected Query getExactQuery(SchemaField field, String externalVal) { + return FloatPoint.newExactQuery(field.getName(), Float.parseFloat(externalVal)); + } + + @Override + public Query getSetQuery(QParser parser, SchemaField field, Collection externalVal) { + assert externalVal.size() > 0; + float[] values = new float[externalVal.size()]; + int i = 0; + for (String val:externalVal) { + values[i] = Float.parseFloat(val); + i++; + } + return FloatPoint.newSetQuery(field.getName(), values); + } + + @Override + protected String indexedToReadable(BytesRef indexedForm) { + return Float.toString(FloatPoint.decodeDimension(indexedForm.bytes, indexedForm.offset)); + } + + @Override + public void readableToIndexed(CharSequence val, BytesRefBuilder result) { + result.grow(Float.BYTES); + result.setLength(Float.BYTES); + FloatPoint.encodeDimension(Float.parseFloat(val.toString()), result.bytes(), 0); + } + + @Override + public SortField getSortField(SchemaField field, boolean top) { + field.checkSortability(); + + Object missingValue = null; + boolean sortMissingLast = field.sortMissingLast(); + boolean sortMissingFirst = field.sortMissingFirst(); + + if (sortMissingLast) { + missingValue = top ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY; + } else if (sortMissingFirst) { + missingValue = top ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY; + } + SortField sf = new SortField(field.getName(), SortField.Type.FLOAT, top); + sf.setMissingValue(missingValue); + return sf; + } + + @Override + public Type getUninversionType(SchemaField sf) { + if (sf.multiValued()) { + throw new UnsupportedOperationException("MultiValued Point fields with DocValues is not currently supported"); +// return Type.SORTED_FLOAT; + } else { + return Type.FLOAT_POINT; + } + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser qparser) { + field.checkFieldCacheSource(); + return new FloatFieldSource(field.getName()); + } + + @Override + public LegacyNumericType getNumericType() { + // TODO: refactor this to not use LegacyNumericType + return LegacyNumericType.FLOAT; + } + + @Override + public IndexableField createField(SchemaField field, Object value, float boost) { + if (!isFieldUsed(field)) return null; + + if (boost != 1.0 && log.isTraceEnabled()) { + log.trace("Can't use document/field boost for PointField. Field: " + field.getName() + ", boost: " + boost); + } + float floatValue = (value instanceof Number) ? ((Number) value).floatValue() : Float.parseFloat(value.toString()); + return new FloatPoint(field.getName(), floatValue); + } + + @Override + protected StoredField getStoredField(SchemaField sf, Object value) { + return new StoredField(sf.getName(), (Float) this.toNativeType(value)); + } + + @Override + public PointTypes getType() { + return PointTypes.FLOAT; + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/IntPointField.java b/solr/core/src/java/org/apache/solr/schema/IntPointField.java new file mode 100644 index 00000000000..a7bab071ec7 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/IntPointField.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.lang.invoke.MethodHandles; +import java.util.Collection; + +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.legacy.LegacyNumericType; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.IntFieldSource; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * {@code PointField} implementation for {@code Integer} values. + * @see PointField + * @see IntPoint + */ +public class IntPointField extends PointField implements IntValueFieldType { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Override + public Object toNativeType(Object val) { + if (val == null) return null; + if (val instanceof Number) return ((Number) val).intValue(); + try { + if (val instanceof String) return Integer.parseInt((String) val); + } catch (NumberFormatException e) { + Float v = Float.parseFloat((String) val); + return v.intValue(); + } + return super.toNativeType(val); + } + + @Override + public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, + boolean maxInclusive) { + int actualMin, actualMax; + if (min == null) { + actualMin = Integer.MIN_VALUE; + } else { + actualMin = Integer.parseInt(min); + if (!minInclusive) { + actualMin++; + } + } + if (max == null) { + actualMax = Integer.MAX_VALUE; + } else { + actualMax = Integer.parseInt(max); + if (!maxInclusive) { + actualMax--; + } + } + return IntPoint.newRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Object toObject(SchemaField sf, BytesRef term) { + return IntPoint.decodeDimension(term.bytes, term.offset); + } + + @Override + public Object toObject(IndexableField f) { + final Number val = f.numericValue(); + if (val != null) { + return val; + } else { + throw new AssertionError("Unexpected state. Field: '" + f + "'"); + } + } + + @Override + protected Query getExactQuery(SchemaField field, String externalVal) { + return IntPoint.newExactQuery(field.getName(), Integer.parseInt(externalVal)); + } + + @Override + public Query getSetQuery(QParser parser, SchemaField field, Collection externalVal) { + assert externalVal.size() > 0; + int[] values = new int[externalVal.size()]; + int i = 0; + for (String val:externalVal) { + values[i] = Integer.parseInt(val); + i++; + } + return IntPoint.newSetQuery(field.getName(), values); + } + + @Override + protected String indexedToReadable(BytesRef indexedForm) { + return Integer.toString(IntPoint.decodeDimension(indexedForm.bytes, indexedForm.offset)); + } + + @Override + public void readableToIndexed(CharSequence val, BytesRefBuilder result) { + result.grow(Integer.BYTES); + result.setLength(Integer.BYTES); + IntPoint.encodeDimension(Integer.parseInt(val.toString()), result.bytes(), 0); + } + + @Override + public SortField getSortField(SchemaField field, boolean top) { + field.checkSortability(); + + Object missingValue = null; + boolean sortMissingLast = field.sortMissingLast(); + boolean sortMissingFirst = field.sortMissingFirst(); + + if (sortMissingLast) { + missingValue = top ? Integer.MIN_VALUE : Integer.MAX_VALUE; + } else if (sortMissingFirst) { + missingValue = top ? Integer.MAX_VALUE : Integer.MIN_VALUE; + } + SortField sf = new SortField(field.getName(), SortField.Type.INT, top); + sf.setMissingValue(missingValue); + return sf; + } + + @Override + public Type getUninversionType(SchemaField sf) { + if (sf.multiValued()) { + throw new UnsupportedOperationException("MultiValued Point fields with DocValues is not currently supported"); +// return Type.SORTED_INTEGER; + } else { + return Type.INTEGER_POINT; + } + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser qparser) { + field.checkFieldCacheSource(); + return new IntFieldSource(field.getName()); + } + + @Override + public LegacyNumericType getNumericType() { + return LegacyNumericType.INT; + } + + @Override + public IndexableField createField(SchemaField field, Object value, float boost) { + if (!isFieldUsed(field)) return null; + + if (boost != 1.0 && log.isTraceEnabled()) { + log.trace("Can't use document/field boost for PointField. Field: " + field.getName() + ", boost: " + boost); + } + int intValue = (value instanceof Number) ? ((Number) value).intValue() : Integer.parseInt(value.toString()); + return new IntPoint(field.getName(), intValue); + } + + @Override + protected StoredField getStoredField(SchemaField sf, Object value) { + return new StoredField(sf.getName(), (Integer) this.toNativeType(value)); + } + + @Override + public PointTypes getType() { + return PointTypes.INTEGER; + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/LongPointField.java b/solr/core/src/java/org/apache/solr/schema/LongPointField.java new file mode 100644 index 00000000000..f3fca3c4bca --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/LongPointField.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.lang.invoke.MethodHandles; +import java.util.Collection; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.legacy.LegacyNumericType; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.LongFieldSource; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * {@code PointField} implementation for {@code Long} values. + * @see PointField + * @see LongPoint + */ +public class LongPointField extends PointField implements LongValueFieldType { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Override + public Object toNativeType(Object val) { + if (val == null) return null; + if (val instanceof Number) return ((Number) val).longValue(); + try { + if (val instanceof String) return Long.parseLong((String) val); + } catch (NumberFormatException e) { + Double v = Double.parseDouble((String) val); + return v.longValue(); + } + return super.toNativeType(val); + } + + @Override + public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, + boolean maxInclusive) { + long actualMin, actualMax; + if (min == null) { + actualMin = Long.MIN_VALUE; + } else { + actualMin = Long.parseLong(min); + if (!minInclusive) { + actualMin++; + } + } + if (max == null) { + actualMax = Long.MAX_VALUE; + } else { + actualMax = Long.parseLong(max); + if (!maxInclusive) { + actualMax--; + } + } + return LongPoint.newRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Object toObject(SchemaField sf, BytesRef term) { + return LongPoint.decodeDimension(term.bytes, term.offset); + } + + @Override + public Object toObject(IndexableField f) { + final Number val = f.numericValue(); + if (val != null) { + return val; + } else { + throw new AssertionError("Unexpected state. Field: '" + f + "'"); + } + } + + @Override + protected Query getExactQuery(SchemaField field, String externalVal) { + return LongPoint.newExactQuery(field.getName(), Long.parseLong(externalVal)); + } + + @Override + public Query getSetQuery(QParser parser, SchemaField field, Collection externalVal) { + assert externalVal.size() > 0; + long[] values = new long[externalVal.size()]; + int i = 0; + for (String val:externalVal) { + values[i] = Long.parseLong(val); + i++; + } + return LongPoint.newSetQuery(field.getName(), values); + } + + @Override + protected String indexedToReadable(BytesRef indexedForm) { + return Long.toString(LongPoint.decodeDimension(indexedForm.bytes, indexedForm.offset)); + } + + @Override + public void readableToIndexed(CharSequence val, BytesRefBuilder result) { + result.grow(Long.BYTES); + result.setLength(Long.BYTES); + LongPoint.encodeDimension(Long.parseLong(val.toString()), result.bytes(), 0); + } + + @Override + public SortField getSortField(SchemaField field, boolean top) { + field.checkSortability(); + + Object missingValue = null; + boolean sortMissingLast = field.sortMissingLast(); + boolean sortMissingFirst = field.sortMissingFirst(); + + if (sortMissingLast) { + missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE; + } else if (sortMissingFirst) { + missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE; + } + SortField sf = new SortField(field.getName(), SortField.Type.LONG, top); + sf.setMissingValue(missingValue); + return sf; + } + + @Override + public Type getUninversionType(SchemaField sf) { + if (sf.multiValued()) { + throw new UnsupportedOperationException("MultiValued Point fields with DocValues is not currently supported"); +// return Type.SORTED_LONG; + } else { + return Type.LONG_POINT; + } + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser qparser) { + field.checkFieldCacheSource(); + return new LongFieldSource(field.getName()); + } + + @Override + public LegacyNumericType getNumericType() { + return LegacyNumericType.LONG; + } + + @Override + public IndexableField createField(SchemaField field, Object value, float boost) { + if (!isFieldUsed(field)) return null; + + if (boost != 1.0 && log.isTraceEnabled()) { + log.trace("Can't use document/field boost for PointField. Field: " + field.getName() + ", boost: " + boost); + } + long longValue = (value instanceof Number) ? ((Number) value).longValue() : Long.parseLong(value.toString()); + return new LongPoint(field.getName(), longValue); + } + + @Override + protected StoredField getStoredField(SchemaField sf, Object value) { + return new StoredField(sf.getName(), (Long) this.toNativeType(value)); + } + + @Override + public PointTypes getType() { + return PointTypes.LONG; + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/PointField.java b/solr/core/src/java/org/apache/solr/schema/PointField.java new file mode 100644 index 00000000000..a2dd8a8aaf6 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/PointField.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortedSetSelector; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.CharsRefBuilder; +import org.apache.solr.common.SolrException; +import org.apache.solr.response.TextResponseWriter; +import org.apache.solr.search.QParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides field types to support for Lucene's {@link + * org.apache.lucene.document.IntPoint}, {@link org.apache.lucene.document.LongPoint}, {@link org.apache.lucene.document.FloatPoint} and + * {@link org.apache.lucene.document.DoublePoint}. + * See {@link org.apache.lucene.search.PointRangeQuery} for more details. + * It supports integer, float, long and double types. See subclasses for details. + *
+ * {@code DocValues} are supported for single-value cases ({@code NumericDocValues}). + * {@code FieldCache} is not supported for {@code PointField}s, so sorting, faceting, etc on these fields require the use of {@code docValues="true"} in the schema. + */ +public abstract class PointField extends PrimitiveFieldType { + + public enum PointTypes { + INTEGER, + LONG, + FLOAT, + DOUBLE, + DATE + } + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Override + public boolean isPointField() { + return true; + } + + @Override + public final ValueSource getSingleValueSource(MultiValueSelector choice, SchemaField field, QParser parser) { + // trivial base case + if (!field.multiValued()) { + // single value matches any selector + return getValueSource(field, parser); + } + + // Point fields don't support UninvertingReader. See SOLR-9202 + if (!field.hasDocValues()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "docValues='true' is required to select '" + choice.toString() + + "' value from multivalued field ("+ field.getName() +") at query time"); + } + + // multivalued Point fields all use SortedSetDocValues, so we give a clean error if that's + // not supported by the specified choice, else we delegate to a helper + SortedSetSelector.Type selectorType = choice.getSortedSetSelectorType(); + if (null == selectorType) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + choice.toString() + " is not a supported option for picking a single value" + + " from the multivalued field: " + field.getName() + + " (type: " + this.getTypeName() + ")"); + } + + return getSingleValueSource(selectorType, field); + } + + /** + * Helper method that will only be called for multivalued Point fields that have doc values. + * Default impl throws an error indicating that selecting a single value from this multivalued + * field is not supported for this field type + * + * @param choice the selector Type to use, will never be null + * @param field the field to use, guaranteed to be multivalued. + * @see #getSingleValueSource(MultiValueSelector,SchemaField,QParser) + */ + protected ValueSource getSingleValueSource(SortedSetSelector.Type choice, SchemaField field) { + throw new UnsupportedOperationException("MultiValued Point fields with DocValues is not currently supported"); + } + + @Override + public boolean isTokenized() { + return false; + } + + @Override + public boolean multiValuedFieldCache() { + return false; + } + + /** + * @return the type of this field + */ + public abstract PointTypes getType(); + + @Override + public abstract Query getSetQuery(QParser parser, SchemaField field, Collection externalVals); + + @Override + public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) { + if (!field.indexed() && field.hasDocValues()) { + // currently implemented as singleton range + return getRangeQuery(parser, field, externalVal, externalVal, true, true); + } else { + return getExactQuery(field, externalVal); + } + } + + protected abstract Query getExactQuery(SchemaField field, String externalVal); + + @Override + public String storedToReadable(IndexableField f) { + return toExternal(f); + } + + @Override + public String toInternal(String val) { + throw new UnsupportedOperationException("Can't generate internal string in PointField. use PointField.toInternalByteRef"); + } + + public BytesRef toInternalByteRef(String val) { + final BytesRefBuilder bytes = new BytesRefBuilder(); + readableToIndexed(val, bytes); + return bytes.get(); + } + + @Override + public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException { + writer.writeVal(name, toObject(f)); + } + + @Override + public String storedToIndexed(IndexableField f) { + throw new UnsupportedOperationException("Not supported with PointFields"); + } + + @Override + public CharsRef indexedToReadable(BytesRef indexedForm, CharsRefBuilder charsRef) { + final String value = indexedToReadable(indexedForm); + charsRef.grow(value.length()); + charsRef.setLength(value.length()); + value.getChars(0, charsRef.length(), charsRef.chars(), 0); + return charsRef.get(); + } + + @Override + public String indexedToReadable(String indexedForm) { + return indexedToReadable(new BytesRef(indexedForm)); + } + + protected abstract String indexedToReadable(BytesRef indexedForm); + + protected boolean isFieldUsed(SchemaField field) { + boolean indexed = field.indexed(); + boolean stored = field.stored(); + boolean docValues = field.hasDocValues(); + + if (!indexed && !stored && !docValues) { + if (log.isTraceEnabled()) { + log.trace("Ignoring unindexed/unstored field: " + field); + } + return false; + } + return true; + } + + @Override + public List createFields(SchemaField sf, Object value, float boost) { + if (!(sf.hasDocValues() || sf.stored())) { + return Collections.singletonList(createField(sf, value, boost)); + } + List fields = new ArrayList<>(); + final IndexableField field = createField(sf, value, boost); + fields.add(field); + + if (sf.hasDocValues()) { + if (sf.multiValued()) { + throw new UnsupportedOperationException("MultiValued Point fields with DocValues is not currently supported. Field: '" + sf.getName() + "'"); + } else { + final long bits; + if (field.numericValue() instanceof Integer || field.numericValue() instanceof Long) { + bits = field.numericValue().longValue(); + } else if (field.numericValue() instanceof Float) { + bits = Float.floatToIntBits(field.numericValue().floatValue()); + } else { + assert field.numericValue() instanceof Double; + bits = Double.doubleToLongBits(field.numericValue().doubleValue()); + } + fields.add(new NumericDocValuesField(sf.getName(), bits)); + } + } + if (sf.stored()) { + fields.add(getStoredField(sf, value)); + } + return fields; + } + + protected abstract StoredField getStoredField(SchemaField sf, Object value); + + @Override + public void checkSchemaField(final SchemaField field) { + // PointFields support DocValues + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaField.java b/solr/core/src/java/org/apache/solr/schema/SchemaField.java index bcd68c28285..009e5fc0deb 100644 --- a/solr/core/src/java/org/apache/solr/schema/SchemaField.java +++ b/solr/core/src/java/org/apache/solr/schema/SchemaField.java @@ -170,6 +170,11 @@ public final class SchemaField extends FieldProperties implements IndexableField "can not sort on multivalued field: " + getName()); } + if (this.type.isPointField() && !hasDocValues()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "can not sort on a PointField without doc values: " + + getName()); + } } /** @@ -191,6 +196,11 @@ public final class SchemaField extends FieldProperties implements IndexableField "can not use FieldCache on multivalued field: " + getName()); } + if (this.type.isPointField() && !hasDocValues()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Point fields can't use FieldCache. Use docValues=true for field: " + + getName()); + } } diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 6d13b515076..7c56311da88 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -98,6 +98,7 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.BoolField; import org.apache.solr.schema.EnumField; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.PointField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TrieDateField; import org.apache.solr.schema.TrieDoubleField; @@ -821,16 +822,39 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI continue; } Object newVal = val; - if (schemaField.getType() instanceof TrieIntField) { - newVal = val.intValue(); - } else if (schemaField.getType() instanceof TrieFloatField) { - newVal = Float.intBitsToFloat(val.intValue()); - } else if (schemaField.getType() instanceof TrieDoubleField) { - newVal = Double.longBitsToDouble(val); - } else if (schemaField.getType() instanceof TrieDateField) { - newVal = new Date(val); - } else if (schemaField.getType() instanceof EnumField) { - newVal = ((EnumField) schemaField.getType()).intValueToStringValue(val.intValue()); + if (schemaField.getType().isPointField()) { + PointField.PointTypes type = ((PointField)schemaField.getType()).getType(); + switch (type) { + case INTEGER: + newVal = val.intValue(); + break; + case LONG: + newVal = val.longValue(); + break; + case FLOAT: + newVal = Float.intBitsToFloat(val.intValue()); + break; + case DOUBLE: + newVal = Double.longBitsToDouble(val); + break; + case DATE: + newVal = new Date(val); + break; + default: + throw new AssertionError("Unexpected PointType: " + type); + } + } else { + if (schemaField.getType() instanceof TrieIntField) { + newVal = val.intValue(); + } else if (schemaField.getType() instanceof TrieFloatField) { + newVal = Float.intBitsToFloat(val.intValue()); + } else if (schemaField.getType() instanceof TrieDoubleField) { + newVal = Double.longBitsToDouble(val); + } else if (schemaField.getType() instanceof TrieDateField) { + newVal = new Date(val); + } else if (schemaField.getType() instanceof EnumField) { + newVal = ((EnumField) schemaField.getType()).intValueToStringValue(val.intValue()); + } } doc.addField(fieldName, newVal); break; diff --git a/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java index 99ef4c4be50..89b3d283e8e 100644 --- a/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java @@ -50,10 +50,16 @@ public class TermQParserPlugin extends QParserPlugin { String fname = localParams.get(QueryParsing.F); FieldType ft = req.getSchema().getFieldTypeNoEx(fname); String val = localParams.get(QueryParsing.V); - BytesRefBuilder term = new BytesRefBuilder(); + BytesRefBuilder term; if (ft != null) { - ft.readableToIndexed(val, term); + if (ft.isPointField()) { + return ft.getFieldQuery(this, req.getSchema().getField(fname), val); + } else { + term = new BytesRefBuilder(); + ft.readableToIndexed(val, term); + } } else { + term = new BytesRefBuilder(); term.copyChars(val); } return new TermQuery(new Term(fname, term.get())); diff --git a/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java index 3a60149ce44..c4073539cc4 100644 --- a/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java @@ -17,6 +17,7 @@ package org.apache.solr.search; import java.util.Arrays; +import java.util.Locale; import java.util.regex.Pattern; import org.apache.lucene.index.Term; @@ -35,6 +36,7 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.solr.common.params.SolrParams; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.PointField; /** * Finds documents whose specified field has any of the specified values. It's like @@ -110,6 +112,14 @@ public class TermsQParserPlugin extends QParserPlugin { return new MatchNoDocsQuery(); final String[] splitVals = sepIsSpace ? qstr.split("\\s+") : qstr.split(Pattern.quote(separator), -1); assert splitVals.length > 0; + + if (ft.isPointField()) { + if (localParams.get(METHOD) != null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Method '%s' not supported in TermsQParser when using PointFields", localParams.get(METHOD))); + } + return ((PointField)ft).getSetQuery(this, req.getSchema().getField(fname), Arrays.asList(splitVals)); + } BytesRef[] bytesRefs = new BytesRef[splitVals.length]; BytesRefBuilder term = new BytesRefBuilder(); diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java index 99f6fceb7e0..900bbf7bd3d 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java @@ -30,6 +30,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.params.FacetParams; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.PointField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TrieDateField; import org.apache.solr.schema.TrieField; @@ -141,7 +142,32 @@ class FacetRangeProcessor extends FacetProcessor { (SolrException.ErrorCode.BAD_REQUEST, "Expected numeric field type :" + sf); } - } else { + } else if (ft instanceof PointField) { + final PointField pfield = (PointField)ft; + + switch (pfield.getType()) { + case FLOAT: + calc = new FloatCalc(sf); + break; + case DOUBLE: + calc = new DoubleCalc(sf); + break; + case INTEGER: + calc = new IntCalc(sf); + break; + case LONG: + calc = new LongCalc(sf); + break; + case DATE: + calc = new DateCalc(sf, null); + break; + default: + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "Expected numeric field type :" + sf); + } + } + else { throw new SolrException (SolrException.ErrorCode.BAD_REQUEST, "Expected numeric field type :" + sf); diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/DocumentExpressionDictionaryFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/DocumentExpressionDictionaryFactory.java index b0d7007f88a..3b7abdf7b84 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/DocumentExpressionDictionaryFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/DocumentExpressionDictionaryFactory.java @@ -28,7 +28,11 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.spell.Dictionary; import org.apache.lucene.search.suggest.DocumentValueSourceDictionary; import org.apache.solr.core.SolrCore; +import org.apache.solr.schema.DoublePointField; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.FloatPointField; +import org.apache.solr.schema.IntPointField; +import org.apache.solr.schema.LongPointField; import org.apache.solr.schema.TrieDoubleField; import org.apache.solr.schema.TrieFloatField; import org.apache.solr.schema.TrieIntField; @@ -111,13 +115,13 @@ public class DocumentExpressionDictionaryFactory extends DictionaryFactory { SortField.Type type = null; String fieldTypeName = core.getLatestSchema().getField(sortFieldName).getType().getTypeName(); FieldType ft = core.getLatestSchema().getFieldTypes().get(fieldTypeName); - if (ft instanceof TrieFloatField) { + if (ft instanceof TrieFloatField || ft instanceof FloatPointField) { type = SortField.Type.FLOAT; - } else if (ft instanceof TrieIntField) { + } else if (ft instanceof TrieIntField || ft instanceof IntPointField) { type = SortField.Type.INT; - } else if (ft instanceof TrieLongField) { + } else if (ft instanceof TrieLongField || ft instanceof LongPointField) { type = SortField.Type.LONG; - } else if (ft instanceof TrieDoubleField) { + } else if (ft instanceof TrieDoubleField || ft instanceof DoublePointField) { type = SortField.Type.DOUBLE; } return type; diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-distrib-interval-faceting.xml b/solr/core/src/test-files/solr/collection1/conf/schema-distrib-interval-faceting.xml index 79d200d240a..ff73fdcd300 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-distrib-interval-faceting.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-distrib-interval-faceting.xml @@ -24,12 +24,18 @@ + + + + + + - + @@ -37,13 +43,13 @@ - + - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml index 113e8682a28..673e7dd1968 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml @@ -23,12 +23,17 @@ + + + + + @@ -37,14 +42,17 @@ + + + @@ -55,11 +63,15 @@ id + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-point.xml b/solr/core/src/test-files/solr/collection1/conf/schema-point.xml new file mode 100644 index 00000000000..ca37ff5d579 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-point.xml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml index c9180172ccd..f68841c78ff 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml @@ -45,30 +45,30 @@ NOTE: Tests expect every field in this schema to be sortable. - - - + + + - - - + + + - - - + + + - - - + + + @@ -220,6 +220,11 @@ NOTE: Tests expect every field in this schema to be sortable. sortMissingLast="true"/> + + + @@ -229,6 +234,11 @@ NOTE: Tests expect every field in this schema to be sortable. sortMissingLast="true"/> + + + @@ -238,6 +248,11 @@ NOTE: Tests expect every field in this schema to be sortable. sortMissingLast="true"/> + + + @@ -247,6 +262,11 @@ NOTE: Tests expect every field in this schema to be sortable. sortMissingLast="true"/> + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml index 35de1660b47..be1b6f55e9b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml @@ -43,6 +43,12 @@ + + + + + + @@ -550,7 +556,7 @@ - + @@ -599,7 +605,7 @@ --> - + @@ -610,9 +616,9 @@ - + - + @@ -662,7 +668,7 @@ - + - + - - @@ -691,8 +697,8 @@ useDocValuesAsStored="true"/> - - + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema11.xml b/solr/core/src/test-files/solr/collection1/conf/schema11.xml index e5b92338309..370f32159b3 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema11.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema11.xml @@ -77,7 +77,12 @@ - + + + + + + - + - - - + + + - - + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema12.xml b/solr/core/src/test-files/solr/collection1/conf/schema12.xml index 0de219a2b38..206cd9ebacf 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema12.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema12.xml @@ -42,6 +42,12 @@ + + + + + + @@ -432,7 +438,7 @@ - + @@ -550,7 +556,8 @@ - + + @@ -559,9 +566,11 @@ - + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml index 280fd34cf6a..c6491eb4173 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml @@ -202,7 +202,7 @@ - + @@ -215,11 +215,11 @@ - + - + @@ -227,15 +227,15 @@ - + - + - + - + @@ -394,6 +394,13 @@ + + + + + + +