diff --git a/dev-tools/idea/lucene/join/join.iml b/dev-tools/idea/lucene/join/join.iml index 1f9e80b8adb..6de5e90a06e 100644 --- a/dev-tools/idea/lucene/join/join.iml +++ b/dev-tools/idea/lucene/join/join.iml @@ -14,6 +14,7 @@ + diff --git a/dev-tools/idea/lucene/queryparser/queryparser.iml b/dev-tools/idea/lucene/queryparser/queryparser.iml index cd2915fe3a9..86a50a58aab 100644 --- a/dev-tools/idea/lucene/queryparser/queryparser.iml +++ b/dev-tools/idea/lucene/queryparser/queryparser.iml @@ -17,5 +17,6 @@ + diff --git a/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml b/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml index 5694371fd6f..6285d261ee4 100644 --- a/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml +++ b/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml @@ -27,6 +27,7 @@ + \ No newline at end of file diff --git a/dev-tools/idea/solr/contrib/analytics/analytics.iml b/dev-tools/idea/solr/contrib/analytics/analytics.iml index 2ff93365d37..10f51a7ef25 100644 --- a/dev-tools/idea/solr/contrib/analytics/analytics.iml +++ b/dev-tools/idea/solr/contrib/analytics/analytics.iml @@ -20,6 +20,7 @@ + diff --git a/dev-tools/idea/solr/core/src/java/solr-core.iml b/dev-tools/idea/solr/core/src/java/solr-core.iml index 822b24f6cab..6cf1ab175f4 100644 --- a/dev-tools/idea/solr/core/src/java/solr-core.iml +++ b/dev-tools/idea/solr/core/src/java/solr-core.iml @@ -31,5 +31,6 @@ + diff --git a/dev-tools/idea/solr/core/src/solr-core-tests.iml b/dev-tools/idea/solr/core/src/solr-core-tests.iml index 56f768b49f3..99297d0a70d 100644 --- a/dev-tools/idea/solr/core/src/solr-core-tests.iml +++ b/dev-tools/idea/solr/core/src/solr-core-tests.iml @@ -32,5 +32,6 @@ + diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index c5db1438d8b..3f7f7c3f6f3 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -38,6 +38,9 @@ API Changes New Features +* LUCENE-7388: Add point based IntRangeField, FloatRangeField, LongRangeField along with + supporting queries and tests (Nick Knize) + * LUCENE-7381: Add point based DoubleRangeField and RangeFieldQuery for indexing and querying on Ranges up to 4 dimensions (Nick Knize) @@ -85,6 +88,9 @@ Bug Fixes * LUCENE-7391: Fix performance regression in MemoryIndex's fields() introduced in Lucene 6. (Steve Mason via David Smiley) +* SOLR-9413: Fix analysis/kuromoji's CSVUtil.quoteEscape logic, add TestCSVUtil test. + (AppChecker, Christine Poerschke) + Improvements * LUCENE-7323: Compound file writing now verifies the incoming @@ -142,6 +148,13 @@ Improvements because the ICU word-breaking algorithm has some issues. This allows for the previous tokenization used before Lucene 5. (AM, Robert Muir) +* LUCENE-7409: Changed MMapDirectory's unmapping to work safer, but still with + no guarantees. This uses a store-store barrier and yields the current thread + before unmapping to allow in-flight requests to finish. The new code no longer + uses WeakIdentityMap as it delegates all ByteBuffer reads throgh a new + ByteBufferGuard wrapper that is shared between all ByteBufferIndexInput clones. + (Robert Muir, Uwe Schindler) + Optimizations * LUCENE-7330, LUCENE-7339: Speed up conjunction queries. (Adrien Grand) diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java index 6301d2c05ca..04f86038d4a 100644 --- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java +++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java @@ -101,7 +101,7 @@ public final class CSVUtil { String result = original; if (result.indexOf('\"') >= 0) { - result.replace("\"", ESCAPED_QUOTE); + result = result.replace("\"", ESCAPED_QUOTE); } if(result.indexOf(COMMA) >= 0) { result = "\"" + result + "\""; diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestCSVUtil.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestCSVUtil.java new file mode 100644 index 00000000000..01545dbfe36 --- /dev/null +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestCSVUtil.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.analysis.ja; + +import java.io.IOException; + +import org.apache.lucene.analysis.ja.util.CSVUtil; +import org.apache.lucene.util.LuceneTestCase; + +/* + * Tests for the CSVUtil class. + */ +public class TestCSVUtil extends LuceneTestCase { + + public void testQuoteEscapeQuotes() throws IOException { + final String input = "\"Let It Be\" is a song and album by the The Beatles."; + final String expectedOutput = input.replace("\"", "\"\""); + implTestQuoteEscape(input, expectedOutput); + } + + public void testQuoteEscapeComma() throws IOException { + final String input = "To be, or not to be ..."; + final String expectedOutput = '"'+input+'"'; + implTestQuoteEscape(input, expectedOutput); + } + + public void testQuoteEscapeQuotesAndComma() throws IOException { + final String input = "\"To be, or not to be ...\" is a well-known phrase from Shakespeare's Hamlet."; + final String expectedOutput = '"'+input.replace("\"", "\"\"")+'"'; + implTestQuoteEscape(input, expectedOutput); + } + + private void implTestQuoteEscape(String input, String expectedOutput) throws IOException { + final String actualOutput = CSVUtil.quoteEscape(input); + assertEquals(expectedOutput, actualOutput); + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/document/LegacyDoubleField.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyDoubleField.java similarity index 81% rename from lucene/core/src/java/org/apache/lucene/document/LegacyDoubleField.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyDoubleField.java index 55ba81cb120..e98a4f0f567 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LegacyDoubleField.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyDoubleField.java @@ -14,9 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.document; +package org.apache.lucene.legacy; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.index.IndexOptions; @@ -49,7 +51,7 @@ import org.apache.lucene.index.IndexOptions; * LegacyFloatField}. * *

To perform range querying or filtering against a - * LegacyDoubleField, use {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + * LegacyDoubleField, use {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. * To sort according to a * LegacyDoubleField, use the normal numeric sort types, eg * {@link org.apache.lucene.search.SortField.Type#DOUBLE}. LegacyDoubleField @@ -79,11 +81,11 @@ import org.apache.lucene.index.IndexOptions; * but may result in faster range search performance. The * default value, 16, was selected for a reasonable tradeoff * of disk space consumption versus performance. You can - * create a custom {@link FieldType} and invoke the {@link - * FieldType#setNumericPrecisionStep} method if you'd + * create a custom {@link LegacyFieldType} and invoke the {@link + * LegacyFieldType#setNumericPrecisionStep} method if you'd * like to change the value. Note that you must also * specify a congruent value when creating {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. * For low cardinality fields larger precision steps are good. * If the cardinality is < 100, it is fair * to use {@link Integer#MAX_VALUE}, which produces one @@ -91,9 +93,9 @@ import org.apache.lucene.index.IndexOptions; * *

For more information on the internals of numeric trie * indexing, including the precisionStep - * configuration, see {@link org.apache.lucene.search.LegacyNumericRangeQuery}. The format of - * indexed values is described in {@link org.apache.lucene.util.LegacyNumericUtils}. + * href="LegacyNumericRangeQuery.html#precisionStepDesc">precisionStep + * configuration, see {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. The format of + * indexed values is described in {@link org.apache.lucene.legacy.LegacyNumericUtils}. * *

If you only need to sort by numeric value, and never * run range querying/filtering, you can index using a @@ -101,7 +103,7 @@ import org.apache.lucene.index.IndexOptions; * This will minimize disk space consumed.

* *

More advanced users can instead use {@link - * org.apache.lucene.analysis.LegacyNumericTokenStream} directly, when indexing numbers. This + * org.apache.lucene.legacy.LegacyNumericTokenStream} directly, when indexing numbers. This * class is a wrapper around this token stream type for * easier, more intuitive usage.

* @@ -111,18 +113,18 @@ import org.apache.lucene.index.IndexOptions; */ @Deprecated -public final class LegacyDoubleField extends Field { +public final class LegacyDoubleField extends LegacyField { /** * Type for a LegacyDoubleField that is not stored: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_NOT_STORED = new FieldType(); + public static final LegacyFieldType TYPE_NOT_STORED = new LegacyFieldType(); static { TYPE_NOT_STORED.setTokenized(true); TYPE_NOT_STORED.setOmitNorms(true); TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_NOT_STORED.setNumericType(FieldType.LegacyNumericType.DOUBLE); + TYPE_NOT_STORED.setNumericType(LegacyNumericType.DOUBLE); TYPE_NOT_STORED.freeze(); } @@ -130,19 +132,19 @@ public final class LegacyDoubleField extends Field { * Type for a stored LegacyDoubleField: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_STORED = new FieldType(); + public static final LegacyFieldType TYPE_STORED = new LegacyFieldType(); static { TYPE_STORED.setTokenized(true); TYPE_STORED.setOmitNorms(true); TYPE_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_STORED.setNumericType(FieldType.LegacyNumericType.DOUBLE); + TYPE_STORED.setNumericType(LegacyNumericType.DOUBLE); TYPE_STORED.setStored(true); TYPE_STORED.freeze(); } /** Creates a stored or un-stored LegacyDoubleField with the provided value * and default precisionStep {@link - * org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). + * org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). * @param name field name * @param value 64-bit double value * @param stored Store.YES if the content should also be stored @@ -154,17 +156,17 @@ public final class LegacyDoubleField extends Field { } /** Expert: allows you to customize the {@link - * FieldType}. + * LegacyFieldType}. * @param name field name * @param value 64-bit double value - * @param type customized field type: must have {@link FieldType#numericType()} - * of {@link org.apache.lucene.document.FieldType.LegacyNumericType#DOUBLE}. + * @param type customized field type: must have {@link LegacyFieldType#numericType()} + * of {@link LegacyNumericType#DOUBLE}. * @throws IllegalArgumentException if the field name or type is null, or * if the field type does not have a DOUBLE numericType() */ - public LegacyDoubleField(String name, double value, FieldType type) { + public LegacyDoubleField(String name, double value, LegacyFieldType type) { super(name, type); - if (type.numericType() != FieldType.LegacyNumericType.DOUBLE) { + if (type.numericType() != LegacyNumericType.DOUBLE) { throw new IllegalArgumentException("type.numericType() must be DOUBLE but got " + type.numericType()); } fieldsData = Double.valueOf(value); diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyField.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyField.java new file mode 100644 index 00000000000..87ac0e566cf --- /dev/null +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyField.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexOptions; + +/** + * Field extension with support for legacy numerics + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ +@Deprecated +public class LegacyField extends Field { + + /** + * Expert: creates a field with no initial value. + * Intended only for custom LegacyField subclasses. + * @param name field name + * @param type field type + * @throws IllegalArgumentException if either the name or type + * is null. + */ + public LegacyField(String name, LegacyFieldType type) { + super(name, type); + } + + @Override + public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) { + if (fieldType().indexOptions() == IndexOptions.NONE) { + // Not indexed + return null; + } + final LegacyFieldType fieldType = (LegacyFieldType) fieldType(); + final LegacyNumericType numericType = fieldType.numericType(); + if (numericType != null) { + if (!(reuse instanceof LegacyNumericTokenStream && ((LegacyNumericTokenStream)reuse).getPrecisionStep() == fieldType.numericPrecisionStep())) { + // lazy init the TokenStream as it is heavy to instantiate + // (attributes,...) if not needed (stored field loading) + reuse = new LegacyNumericTokenStream(fieldType.numericPrecisionStep()); + } + final LegacyNumericTokenStream nts = (LegacyNumericTokenStream) reuse; + // initialize value in TokenStream + final Number val = (Number) fieldsData; + switch (numericType) { + case INT: + nts.setIntValue(val.intValue()); + break; + case LONG: + nts.setLongValue(val.longValue()); + break; + case FLOAT: + nts.setFloatValue(val.floatValue()); + break; + case DOUBLE: + nts.setDoubleValue(val.doubleValue()); + break; + default: + throw new AssertionError("Should never get here"); + } + return reuse; + } + return super.tokenStream(analyzer, reuse); + } + + @Override + public void setTokenStream(TokenStream tokenStream) { + final LegacyFieldType fieldType = (LegacyFieldType) fieldType(); + if (fieldType.numericType() != null) { + throw new IllegalArgumentException("cannot set private TokenStream on numeric fields"); + } + super.setTokenStream(tokenStream); + } + +} diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFieldType.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFieldType.java new file mode 100644 index 00000000000..1f4b0af4768 --- /dev/null +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFieldType.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; + +/** + * FieldType extension with support for legacy numerics + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ +@Deprecated +public final class LegacyFieldType extends FieldType { + private LegacyNumericType numericType; + private int numericPrecisionStep = LegacyNumericUtils.PRECISION_STEP_DEFAULT; + + /** + * Create a new mutable LegacyFieldType with all of the properties from ref + */ + public LegacyFieldType(LegacyFieldType ref) { + super(ref); + this.numericType = ref.numericType; + this.numericPrecisionStep = ref.numericPrecisionStep; + } + + /** + * Create a new FieldType with default properties. + */ + public LegacyFieldType() { + } + + /** + * Specifies the field's numeric type. + * @param type numeric type, or null if the field has no numeric type. + * @throws IllegalStateException if this FieldType is frozen against + * future modifications. + * @see #numericType() + * + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ + @Deprecated + public void setNumericType(LegacyNumericType type) { + checkIfFrozen(); + numericType = type; + } + + /** + * LegacyNumericType: if non-null then the field's value will be indexed + * numerically so that {@link org.apache.lucene.legacy.LegacyNumericRangeQuery} can be used at + * search time. + *

+ * The default is null (no numeric type) + * @see #setNumericType(LegacyNumericType) + * + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ + @Deprecated + public LegacyNumericType numericType() { + return numericType; + } + + /** + * Sets the numeric precision step for the field. + * @param precisionStep numeric precision step for the field + * @throws IllegalArgumentException if precisionStep is less than 1. + * @throws IllegalStateException if this FieldType is frozen against + * future modifications. + * @see #numericPrecisionStep() + * + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ + @Deprecated + public void setNumericPrecisionStep(int precisionStep) { + checkIfFrozen(); + if (precisionStep < 1) { + throw new IllegalArgumentException("precisionStep must be >= 1 (got " + precisionStep + ")"); + } + this.numericPrecisionStep = precisionStep; + } + + /** + * Precision step for numeric field. + *

+ * This has no effect if {@link #numericType()} returns null. + *

+ * The default is {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} + * @see #setNumericPrecisionStep(int) + * + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ + @Deprecated + public int numericPrecisionStep() { + return numericPrecisionStep; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + numericPrecisionStep; + result = prime * result + ((numericType == null) ? 0 : numericType.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (!super.equals(obj)) { + return false; + } + if (getClass() != obj.getClass()) return false; + LegacyFieldType other = (LegacyFieldType) obj; + if (numericPrecisionStep != other.numericPrecisionStep) return false; + if (numericType != other.numericType) return false; + return true; + } + + /** Prints a Field for human consumption. */ + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append(super.toString()); + if (indexOptions() != IndexOptions.NONE) { + if (result.length() > 0) { + result.append(","); + } + if (numericType != null) { + result.append(",numericType="); + result.append(numericType); + result.append(",numericPrecisionStep="); + result.append(numericPrecisionStep); + } + } + return result.toString(); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/document/LegacyFloatField.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFloatField.java similarity index 80% rename from lucene/core/src/java/org/apache/lucene/document/LegacyFloatField.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFloatField.java index e24bf30fa54..ea3b84ab65f 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LegacyFloatField.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFloatField.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.document; - +package org.apache.lucene.legacy; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FloatPoint; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.util.LegacyNumericUtils; /** *

@@ -49,7 +49,7 @@ import org.apache.lucene.util.LegacyNumericUtils; * LegacyDoubleField}. * *

To perform range querying or filtering against a - * LegacyFloatField, use {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + * LegacyFloatField, use {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. * To sort according to a * LegacyFloatField, use the normal numeric sort types, eg * {@link org.apache.lucene.search.SortField.Type#FLOAT}. LegacyFloatField @@ -79,11 +79,11 @@ import org.apache.lucene.util.LegacyNumericUtils; * but may result in faster range search performance. The * default value, 8, was selected for a reasonable tradeoff * of disk space consumption versus performance. You can - * create a custom {@link FieldType} and invoke the {@link - * FieldType#setNumericPrecisionStep} method if you'd + * create a custom {@link LegacyFieldType} and invoke the {@link + * LegacyFieldType#setNumericPrecisionStep} method if you'd * like to change the value. Note that you must also * specify a congruent value when creating {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. * For low cardinality fields larger precision steps are good. * If the cardinality is < 100, it is fair * to use {@link Integer#MAX_VALUE}, which produces one @@ -91,9 +91,9 @@ import org.apache.lucene.util.LegacyNumericUtils; * *

For more information on the internals of numeric trie * indexing, including the precisionStep - * configuration, see {@link org.apache.lucene.search.LegacyNumericRangeQuery}. The format of - * indexed values is described in {@link org.apache.lucene.util.LegacyNumericUtils}. + * href="LegacyNumericRangeQuery.html#precisionStepDesc">precisionStep + * configuration, see {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. The format of + * indexed values is described in {@link org.apache.lucene.legacy.LegacyNumericUtils}. * *

If you only need to sort by numeric value, and never * run range querying/filtering, you can index using a @@ -101,7 +101,7 @@ import org.apache.lucene.util.LegacyNumericUtils; * This will minimize disk space consumed.

* *

More advanced users can instead use {@link - * org.apache.lucene.analysis.LegacyNumericTokenStream} directly, when indexing numbers. This + * org.apache.lucene.legacy.LegacyNumericTokenStream} directly, when indexing numbers. This * class is a wrapper around this token stream type for * easier, more intuitive usage.

* @@ -111,18 +111,18 @@ import org.apache.lucene.util.LegacyNumericUtils; */ @Deprecated -public final class LegacyFloatField extends Field { +public final class LegacyFloatField extends LegacyField { /** * Type for a LegacyFloatField that is not stored: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_NOT_STORED = new FieldType(); + public static final LegacyFieldType TYPE_NOT_STORED = new LegacyFieldType(); static { TYPE_NOT_STORED.setTokenized(true); TYPE_NOT_STORED.setOmitNorms(true); TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_NOT_STORED.setNumericType(FieldType.LegacyNumericType.FLOAT); + TYPE_NOT_STORED.setNumericType(LegacyNumericType.FLOAT); TYPE_NOT_STORED.setNumericPrecisionStep(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32); TYPE_NOT_STORED.freeze(); } @@ -131,12 +131,12 @@ public final class LegacyFloatField extends Field { * Type for a stored LegacyFloatField: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_STORED = new FieldType(); + public static final LegacyFieldType TYPE_STORED = new LegacyFieldType(); static { TYPE_STORED.setTokenized(true); TYPE_STORED.setOmitNorms(true); TYPE_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_STORED.setNumericType(FieldType.LegacyNumericType.FLOAT); + TYPE_STORED.setNumericType(LegacyNumericType.FLOAT); TYPE_STORED.setNumericPrecisionStep(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32); TYPE_STORED.setStored(true); TYPE_STORED.freeze(); @@ -144,7 +144,7 @@ public final class LegacyFloatField extends Field { /** Creates a stored or un-stored LegacyFloatField with the provided value * and default precisionStep {@link - * org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). + * org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). * @param name field name * @param value 32-bit double value * @param stored Store.YES if the content should also be stored @@ -156,17 +156,17 @@ public final class LegacyFloatField extends Field { } /** Expert: allows you to customize the {@link - * FieldType}. + * LegacyFieldType}. * @param name field name * @param value 32-bit float value - * @param type customized field type: must have {@link FieldType#numericType()} - * of {@link org.apache.lucene.document.FieldType.LegacyNumericType#FLOAT}. + * @param type customized field type: must have {@link LegacyFieldType#numericType()} + * of {@link LegacyNumericType#FLOAT}. * @throws IllegalArgumentException if the field name or type is null, or * if the field type does not have a FLOAT numericType() */ - public LegacyFloatField(String name, float value, FieldType type) { + public LegacyFloatField(String name, float value, LegacyFieldType type) { super(name, type); - if (type.numericType() != FieldType.LegacyNumericType.FLOAT) { + if (type.numericType() != LegacyNumericType.FLOAT) { throw new IllegalArgumentException("type.numericType() must be FLOAT but got " + type.numericType()); } fieldsData = Float.valueOf(value); diff --git a/lucene/core/src/java/org/apache/lucene/document/LegacyIntField.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyIntField.java similarity index 80% rename from lucene/core/src/java/org/apache/lucene/document/LegacyIntField.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyIntField.java index 6eb0376ee64..e3ae9658b1d 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LegacyIntField.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyIntField.java @@ -14,11 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.document; +package org.apache.lucene.legacy; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.util.LegacyNumericUtils; /** *

@@ -49,7 +50,7 @@ import org.apache.lucene.util.LegacyNumericUtils; * LegacyDoubleField}. * *

To perform range querying or filtering against a - * LegacyIntField, use {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + * LegacyIntField, use {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. * To sort according to a * LegacyIntField, use the normal numeric sort types, eg * {@link org.apache.lucene.search.SortField.Type#INT}. LegacyIntField @@ -79,11 +80,11 @@ import org.apache.lucene.util.LegacyNumericUtils; * but may result in faster range search performance. The * default value, 8, was selected for a reasonable tradeoff * of disk space consumption versus performance. You can - * create a custom {@link FieldType} and invoke the {@link - * FieldType#setNumericPrecisionStep} method if you'd + * create a custom {@link LegacyFieldType} and invoke the {@link + * LegacyFieldType#setNumericPrecisionStep} method if you'd * like to change the value. Note that you must also * specify a congruent value when creating {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. * For low cardinality fields larger precision steps are good. * If the cardinality is < 100, it is fair * to use {@link Integer#MAX_VALUE}, which produces one @@ -91,9 +92,9 @@ import org.apache.lucene.util.LegacyNumericUtils; * *

For more information on the internals of numeric trie * indexing, including the precisionStep - * configuration, see {@link org.apache.lucene.search.LegacyNumericRangeQuery}. The format of - * indexed values is described in {@link org.apache.lucene.util.LegacyNumericUtils}. + * href="LegacyNumericRangeQuery.html#precisionStepDesc">precisionStep + * configuration, see {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. The format of + * indexed values is described in {@link org.apache.lucene.legacy.LegacyNumericUtils}. * *

If you only need to sort by numeric value, and never * run range querying/filtering, you can index using a @@ -101,7 +102,7 @@ import org.apache.lucene.util.LegacyNumericUtils; * This will minimize disk space consumed.

* *

More advanced users can instead use {@link - * org.apache.lucene.analysis.LegacyNumericTokenStream} directly, when indexing numbers. This + * org.apache.lucene.legacy.LegacyNumericTokenStream} directly, when indexing numbers. This * class is a wrapper around this token stream type for * easier, more intuitive usage.

* @@ -111,18 +112,18 @@ import org.apache.lucene.util.LegacyNumericUtils; */ @Deprecated -public final class LegacyIntField extends Field { +public final class LegacyIntField extends LegacyField { /** * Type for an LegacyIntField that is not stored: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_NOT_STORED = new FieldType(); + public static final LegacyFieldType TYPE_NOT_STORED = new LegacyFieldType(); static { TYPE_NOT_STORED.setTokenized(true); TYPE_NOT_STORED.setOmitNorms(true); TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_NOT_STORED.setNumericType(FieldType.LegacyNumericType.INT); + TYPE_NOT_STORED.setNumericType(LegacyNumericType.INT); TYPE_NOT_STORED.setNumericPrecisionStep(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32); TYPE_NOT_STORED.freeze(); } @@ -131,12 +132,12 @@ public final class LegacyIntField extends Field { * Type for a stored LegacyIntField: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_STORED = new FieldType(); + public static final LegacyFieldType TYPE_STORED = new LegacyFieldType(); static { TYPE_STORED.setTokenized(true); TYPE_STORED.setOmitNorms(true); TYPE_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_STORED.setNumericType(FieldType.LegacyNumericType.INT); + TYPE_STORED.setNumericType(LegacyNumericType.INT); TYPE_STORED.setNumericPrecisionStep(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32); TYPE_STORED.setStored(true); TYPE_STORED.freeze(); @@ -144,7 +145,7 @@ public final class LegacyIntField extends Field { /** Creates a stored or un-stored LegacyIntField with the provided value * and default precisionStep {@link - * org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). + * org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). * @param name field name * @param value 32-bit integer value * @param stored Store.YES if the content should also be stored @@ -156,17 +157,17 @@ public final class LegacyIntField extends Field { } /** Expert: allows you to customize the {@link - * FieldType}. + * LegacyFieldType}. * @param name field name * @param value 32-bit integer value - * @param type customized field type: must have {@link FieldType#numericType()} - * of {@link org.apache.lucene.document.FieldType.LegacyNumericType#INT}. + * @param type customized field type: must have {@link LegacyFieldType#numericType()} + * of {@link LegacyNumericType#INT}. * @throws IllegalArgumentException if the field name or type is null, or * if the field type does not have a INT numericType() */ - public LegacyIntField(String name, int value, FieldType type) { + public LegacyIntField(String name, int value, LegacyFieldType type) { super(name, type); - if (type.numericType() != FieldType.LegacyNumericType.INT) { + if (type.numericType() != LegacyNumericType.INT) { throw new IllegalArgumentException("type.numericType() must be INT but got " + type.numericType()); } fieldsData = Integer.valueOf(value); diff --git a/lucene/core/src/java/org/apache/lucene/document/LegacyLongField.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyLongField.java similarity index 81% rename from lucene/core/src/java/org/apache/lucene/document/LegacyLongField.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyLongField.java index fa1851fe7e6..3e20b448b96 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LegacyLongField.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyLongField.java @@ -14,9 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.document; +package org.apache.lucene.legacy; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.IndexOptions; @@ -59,7 +61,7 @@ import org.apache.lucene.index.IndexOptions; * long value. * *

To perform range querying or filtering against a - * LegacyLongField, use {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + * LegacyLongField, use {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. * To sort according to a * LegacyLongField, use the normal numeric sort types, eg * {@link org.apache.lucene.search.SortField.Type#LONG}. LegacyLongField @@ -89,11 +91,11 @@ import org.apache.lucene.index.IndexOptions; * but may result in faster range search performance. The * default value, 16, was selected for a reasonable tradeoff * of disk space consumption versus performance. You can - * create a custom {@link FieldType} and invoke the {@link - * FieldType#setNumericPrecisionStep} method if you'd + * create a custom {@link LegacyFieldType} and invoke the {@link + * LegacyFieldType#setNumericPrecisionStep} method if you'd * like to change the value. Note that you must also * specify a congruent value when creating {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. * For low cardinality fields larger precision steps are good. * If the cardinality is < 100, it is fair * to use {@link Integer#MAX_VALUE}, which produces one @@ -101,9 +103,9 @@ import org.apache.lucene.index.IndexOptions; * *

For more information on the internals of numeric trie * indexing, including the precisionStep - * configuration, see {@link org.apache.lucene.search.LegacyNumericRangeQuery}. The format of - * indexed values is described in {@link org.apache.lucene.util.LegacyNumericUtils}. + * href="LegacyNumericRangeQuery.html#precisionStepDesc">precisionStep + * configuration, see {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. The format of + * indexed values is described in {@link org.apache.lucene.legacy.LegacyNumericUtils}. * *

If you only need to sort by numeric value, and never * run range querying/filtering, you can index using a @@ -111,7 +113,7 @@ import org.apache.lucene.index.IndexOptions; * This will minimize disk space consumed. * *

More advanced users can instead use {@link - * org.apache.lucene.analysis.LegacyNumericTokenStream} directly, when indexing numbers. This + * org.apache.lucene.legacy.LegacyNumericTokenStream} directly, when indexing numbers. This * class is a wrapper around this token stream type for * easier, more intuitive usage.

* @@ -121,18 +123,18 @@ import org.apache.lucene.index.IndexOptions; */ @Deprecated -public final class LegacyLongField extends Field { +public final class LegacyLongField extends LegacyField { /** * Type for a LegacyLongField that is not stored: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_NOT_STORED = new FieldType(); + public static final LegacyFieldType TYPE_NOT_STORED = new LegacyFieldType(); static { TYPE_NOT_STORED.setTokenized(true); TYPE_NOT_STORED.setOmitNorms(true); TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_NOT_STORED.setNumericType(FieldType.LegacyNumericType.LONG); + TYPE_NOT_STORED.setNumericType(LegacyNumericType.LONG); TYPE_NOT_STORED.freeze(); } @@ -140,19 +142,19 @@ public final class LegacyLongField extends Field { * Type for a stored LegacyLongField: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_STORED = new FieldType(); + public static final LegacyFieldType TYPE_STORED = new LegacyFieldType(); static { TYPE_STORED.setTokenized(true); TYPE_STORED.setOmitNorms(true); TYPE_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_STORED.setNumericType(FieldType.LegacyNumericType.LONG); + TYPE_STORED.setNumericType(LegacyNumericType.LONG); TYPE_STORED.setStored(true); TYPE_STORED.freeze(); } /** Creates a stored or un-stored LegacyLongField with the provided value * and default precisionStep {@link - * org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). + * org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). * @param name field name * @param value 64-bit long value * @param stored Store.YES if the content should also be stored @@ -164,17 +166,17 @@ public final class LegacyLongField extends Field { } /** Expert: allows you to customize the {@link - * FieldType}. + * LegacyFieldType}. * @param name field name * @param value 64-bit long value - * @param type customized field type: must have {@link FieldType#numericType()} - * of {@link org.apache.lucene.document.FieldType.LegacyNumericType#LONG}. + * @param type customized field type: must have {@link LegacyFieldType#numericType()} + * of {@link LegacyNumericType#LONG}. * @throws IllegalArgumentException if the field name or type is null, or * if the field type does not have a LONG numericType() */ - public LegacyLongField(String name, long value, FieldType type) { + public LegacyLongField(String name, long value, LegacyFieldType type) { super(name, type); - if (type.numericType() != FieldType.LegacyNumericType.LONG) { + if (type.numericType() != LegacyNumericType.LONG) { throw new IllegalArgumentException("type.numericType() must be LONG but got " + type.numericType()); } fieldsData = Long.valueOf(value); diff --git a/lucene/core/src/java/org/apache/lucene/search/LegacyNumericRangeQuery.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericRangeQuery.java similarity index 89% rename from lucene/core/src/java/org/apache/lucene/search/LegacyNumericRangeQuery.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericRangeQuery.java index fe6c9e24864..f172a200779 100644 --- a/lucene/core/src/java/org/apache/lucene/search/LegacyNumericRangeQuery.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericRangeQuery.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.search; +package org.apache.lucene.legacy; import java.io.IOException; @@ -22,8 +22,6 @@ import java.util.LinkedList; import java.util.Objects; import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FieldType.LegacyNumericType; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; @@ -31,18 +29,21 @@ import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.PointValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.index.Term; // for javadocs /** *

A {@link Query} that matches numeric values within a * specified range. To use this, you must first index the - * numeric values using {@link org.apache.lucene.document.LegacyIntField}, {@link - * org.apache.lucene.document.LegacyFloatField}, {@link org.apache.lucene.document.LegacyLongField} or {@link org.apache.lucene.document.LegacyDoubleField} (expert: {@link - * org.apache.lucene.analysis.LegacyNumericTokenStream}). If your terms are instead textual, + * numeric values using {@link org.apache.lucene.legacy.LegacyIntField}, {@link + * org.apache.lucene.legacy.LegacyFloatField}, {@link org.apache.lucene.legacy.LegacyLongField} or {@link org.apache.lucene.legacy.LegacyDoubleField} (expert: {@link + * org.apache.lucene.legacy.LegacyNumericTokenStream}). If your terms are instead textual, * you should use {@link TermRangeQuery}.

* *

You create a new LegacyNumericRangeQuery with the static @@ -96,7 +97,7 @@ import org.apache.lucene.index.Term; // for javadocs * (all numerical values like doubles, longs, floats, and ints are converted to * lexicographic sortable string representations and stored with different precisions * (for a more detailed description of how the values are stored, - * see {@link org.apache.lucene.util.LegacyNumericUtils}). A range is then divided recursively into multiple intervals for searching: + * see {@link org.apache.lucene.legacy.LegacyNumericUtils}). A range is then divided recursively into multiple intervals for searching: * The center of the range is searched only with the lowest possible precision in the trie, * while the boundaries are matched more exactly. This reduces the number of terms dramatically.

* @@ -112,7 +113,7 @@ import org.apache.lucene.index.Term; // for javadocs *

Precision Step

*

You can choose any precisionStep when encoding values. * Lower step values mean more precisions and so more terms in index (and index gets larger). The number - * of indexed terms per value is (those are generated by {@link org.apache.lucene.analysis.LegacyNumericTokenStream}): + * of indexed terms per value is (those are generated by {@link org.apache.lucene.legacy.LegacyNumericTokenStream}): *

*   indexedTermsPerValue = ceil(bitsPerValue / precisionStep) *

@@ -148,8 +149,8 @@ import org.apache.lucene.index.Term; // for javadocs *
  • Steps ≥64 for long/double and ≥32 for int/float produces one token * per value in the index and querying is as slow as a conventional {@link TermRangeQuery}. But it can be used * to produce fields, that are solely used for sorting (in this case simply use {@link Integer#MAX_VALUE} as - * precisionStep). Using {@link org.apache.lucene.document.LegacyIntField}, - * {@link org.apache.lucene.document.LegacyLongField}, {@link org.apache.lucene.document.LegacyFloatField} or {@link org.apache.lucene.document.LegacyDoubleField} for sorting + * precisionStep). Using {@link org.apache.lucene.legacy.LegacyIntField}, + * {@link org.apache.lucene.legacy.LegacyLongField}, {@link org.apache.lucene.legacy.LegacyFloatField} or {@link org.apache.lucene.legacy.LegacyDoubleField} for sorting * is ideal, because building the field cache is much faster than with text-only numbers. * These fields have one term per value and therefore also work with term enumeration for building distinct lists * (e.g. facets / preselected values to search for). @@ -199,12 +200,12 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newLongRange(final String field, final int precisionStep, Long min, Long max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, precisionStep, FieldType.LegacyNumericType.LONG, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, precisionStep, LegacyNumericType.LONG, min, max, minInclusive, maxInclusive); } /** * Factory that creates a LegacyNumericRangeQuery, that queries a long - * range using the default precisionStep {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). + * range using the default precisionStep {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). * You can have half-open ranges (which are in fact </≤ or >/≥ queries) * by setting the min or max value to null. By setting inclusive to false, it will * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. @@ -212,7 +213,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newLongRange(final String field, Long min, Long max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT, FieldType.LegacyNumericType.LONG, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT, LegacyNumericType.LONG, min, max, minInclusive, maxInclusive); } /** @@ -225,12 +226,12 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newIntRange(final String field, final int precisionStep, Integer min, Integer max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, precisionStep, FieldType.LegacyNumericType.INT, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, precisionStep, LegacyNumericType.INT, min, max, minInclusive, maxInclusive); } /** * Factory that creates a LegacyNumericRangeQuery, that queries a int - * range using the default precisionStep {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). + * range using the default precisionStep {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). * You can have half-open ranges (which are in fact </≤ or >/≥ queries) * by setting the min or max value to null. By setting inclusive to false, it will * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. @@ -238,7 +239,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newIntRange(final String field, Integer min, Integer max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, FieldType.LegacyNumericType.INT, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, LegacyNumericType.INT, min, max, minInclusive, maxInclusive); } /** @@ -253,12 +254,12 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newDoubleRange(final String field, final int precisionStep, Double min, Double max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, precisionStep, FieldType.LegacyNumericType.DOUBLE, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, precisionStep, LegacyNumericType.DOUBLE, min, max, minInclusive, maxInclusive); } /** * Factory that creates a LegacyNumericRangeQuery, that queries a double - * range using the default precisionStep {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). + * range using the default precisionStep {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). * You can have half-open ranges (which are in fact </≤ or >/≥ queries) * by setting the min or max value to null. * {@link Double#NaN} will never match a half-open range, to hit {@code NaN} use a query @@ -268,7 +269,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newDoubleRange(final String field, Double min, Double max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT, FieldType.LegacyNumericType.DOUBLE, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT, LegacyNumericType.DOUBLE, min, max, minInclusive, maxInclusive); } /** @@ -283,12 +284,12 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newFloatRange(final String field, final int precisionStep, Float min, Float max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, precisionStep, FieldType.LegacyNumericType.FLOAT, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, precisionStep, LegacyNumericType.FLOAT, min, max, minInclusive, maxInclusive); } /** * Factory that creates a LegacyNumericRangeQuery, that queries a float - * range using the default precisionStep {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). + * range using the default precisionStep {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). * You can have half-open ranges (which are in fact </≤ or >/≥ queries) * by setting the min or max value to null. * {@link Float#NaN} will never match a half-open range, to hit {@code NaN} use a query @@ -298,7 +299,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newFloatRange(final String field, Float min, Float max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, FieldType.LegacyNumericType.FLOAT, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, LegacyNumericType.FLOAT, min, max, minInclusive, maxInclusive); } @Override @SuppressWarnings("unchecked") @@ -369,7 +370,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu // members (package private, to be also fast accessible by NumericRangeTermEnum) final int precisionStep; - final FieldType.LegacyNumericType dataType; + final LegacyNumericType dataType; final T min, max; final boolean minInclusive,maxInclusive; @@ -389,8 +390,8 @@ public final class LegacyNumericRangeQuery extends MultiTermQu *

    * WARNING: This term enumeration is not guaranteed to be always ordered by * {@link Term#compareTo}. - * The ordering depends on how {@link org.apache.lucene.util.LegacyNumericUtils#splitLongRange} and - * {@link org.apache.lucene.util.LegacyNumericUtils#splitIntRange} generates the sub-ranges. For + * The ordering depends on how {@link org.apache.lucene.legacy.LegacyNumericUtils#splitLongRange} and + * {@link org.apache.lucene.legacy.LegacyNumericUtils#splitIntRange} generates the sub-ranges. For * {@link MultiTermQuery} ordering is not relevant. */ private final class NumericRangeTermsEnum extends FilteredTermsEnum { @@ -406,10 +407,10 @@ public final class LegacyNumericRangeQuery extends MultiTermQu case DOUBLE: { // lower long minBound; - if (dataType == FieldType.LegacyNumericType.LONG) { + if (dataType == LegacyNumericType.LONG) { minBound = (min == null) ? Long.MIN_VALUE : min.longValue(); } else { - assert dataType == FieldType.LegacyNumericType.DOUBLE; + assert dataType == LegacyNumericType.DOUBLE; minBound = (min == null) ? LONG_NEGATIVE_INFINITY : NumericUtils.doubleToSortableLong(min.doubleValue()); } @@ -420,10 +421,10 @@ public final class LegacyNumericRangeQuery extends MultiTermQu // upper long maxBound; - if (dataType == FieldType.LegacyNumericType.LONG) { + if (dataType == LegacyNumericType.LONG) { maxBound = (max == null) ? Long.MAX_VALUE : max.longValue(); } else { - assert dataType == FieldType.LegacyNumericType.DOUBLE; + assert dataType == LegacyNumericType.DOUBLE; maxBound = (max == null) ? LONG_POSITIVE_INFINITY : NumericUtils.doubleToSortableLong(max.doubleValue()); } @@ -446,10 +447,10 @@ public final class LegacyNumericRangeQuery extends MultiTermQu case FLOAT: { // lower int minBound; - if (dataType == FieldType.LegacyNumericType.INT) { + if (dataType == LegacyNumericType.INT) { minBound = (min == null) ? Integer.MIN_VALUE : min.intValue(); } else { - assert dataType == FieldType.LegacyNumericType.FLOAT; + assert dataType == LegacyNumericType.FLOAT; minBound = (min == null) ? INT_NEGATIVE_INFINITY : NumericUtils.floatToSortableInt(min.floatValue()); } @@ -463,7 +464,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu if (dataType == LegacyNumericType.INT) { maxBound = (max == null) ? Integer.MAX_VALUE : max.intValue(); } else { - assert dataType == FieldType.LegacyNumericType.FLOAT; + assert dataType == LegacyNumericType.FLOAT; maxBound = (max == null) ? INT_POSITIVE_INFINITY : NumericUtils.floatToSortableInt(max.floatValue()); } diff --git a/lucene/core/src/java/org/apache/lucene/analysis/LegacyNumericTokenStream.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericTokenStream.java similarity index 94% rename from lucene/core/src/java/org/apache/lucene/analysis/LegacyNumericTokenStream.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericTokenStream.java index 19f7d37e31f..a2aba19e2ac 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/LegacyNumericTokenStream.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericTokenStream.java @@ -14,11 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.analysis; +package org.apache.lucene.legacy; import java.util.Objects; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; @@ -29,16 +30,15 @@ import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.AttributeReflector; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; /** * Expert: This class provides a {@link TokenStream} * for indexing numeric values that can be used by {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. * - *

    Note that for simple usage, {@link org.apache.lucene.document.LegacyIntField}, {@link - * org.apache.lucene.document.LegacyLongField}, {@link org.apache.lucene.document.LegacyFloatField} or {@link org.apache.lucene.document.LegacyDoubleField} is + *

    Note that for simple usage, {@link org.apache.lucene.legacy.LegacyIntField}, {@link + * org.apache.lucene.legacy.LegacyLongField}, {@link org.apache.lucene.legacy.LegacyFloatField} or {@link org.apache.lucene.legacy.LegacyDoubleField} is * recommended. These fields disable norms and * term freqs, as they are not usually needed during * searching. If you need to change these settings, you @@ -81,9 +81,9 @@ import org.apache.lucene.util.NumericUtils; * than one numeric field, use a separate LegacyNumericTokenStream * instance for each.

    * - *

    See {@link org.apache.lucene.search.LegacyNumericRangeQuery} for more details on the + *

    See {@link org.apache.lucene.legacy.LegacyNumericRangeQuery} for more details on the * precisionStep + * href="LegacyNumericRangeQuery.html#precisionStepDesc">precisionStep * parameter as well as how numeric fields work under the hood.

    * * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead @@ -140,7 +140,7 @@ public final class LegacyNumericTokenStream extends TokenStream { } } - /** Implementation of {@link org.apache.lucene.analysis.LegacyNumericTokenStream.LegacyNumericTermAttribute}. + /** Implementation of {@link org.apache.lucene.legacy.LegacyNumericTokenStream.LegacyNumericTermAttribute}. * @lucene.internal * @since 4.0 */ @@ -240,7 +240,7 @@ public final class LegacyNumericTokenStream extends TokenStream { /** * Creates a token stream for numeric values using the default precisionStep - * {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). The stream is not yet initialized, + * {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). The stream is not yet initialized, * before using set a value using the various set???Value() methods. */ public LegacyNumericTokenStream() { diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericType.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericType.java new file mode 100644 index 00000000000..345b4974b02 --- /dev/null +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericType.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + +/** Data type of the numeric value + * @since 3.2 + * + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ +@Deprecated +public enum LegacyNumericType { + /** 32-bit integer numeric type */ + INT, + /** 64-bit long numeric type */ + LONG, + /** 32-bit float numeric type */ + FLOAT, + /** 64-bit double numeric type */ + DOUBLE +} diff --git a/lucene/core/src/java/org/apache/lucene/util/LegacyNumericUtils.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericUtils.java similarity index 95% rename from lucene/core/src/java/org/apache/lucene/util/LegacyNumericUtils.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericUtils.java index 9a26bfa3f2b..e6659d7e102 100644 --- a/lucene/core/src/java/org/apache/lucene/util/LegacyNumericUtils.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericUtils.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.util; +package org.apache.lucene.legacy; import java.io.IOException; @@ -23,6 +23,8 @@ import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; /** * This is a helper class to generate prefix-encoded representations for numerical values @@ -41,9 +43,9 @@ import org.apache.lucene.index.TermsEnum; * during encoding. * *

    For easy usage, the trie algorithm is implemented for indexing inside - * {@link org.apache.lucene.analysis.LegacyNumericTokenStream} that can index int, long, + * {@link org.apache.lucene.legacy.LegacyNumericTokenStream} that can index int, long, * float, and double. For querying, - * {@link org.apache.lucene.search.LegacyNumericRangeQuery} implements the query part + * {@link org.apache.lucene.legacy.LegacyNumericRangeQuery} implements the query part * for the same data types. * * @lucene.internal @@ -59,15 +61,15 @@ public final class LegacyNumericUtils { private LegacyNumericUtils() {} // no instance! /** - * The default precision step used by {@link org.apache.lucene.document.LegacyLongField}, - * {@link org.apache.lucene.document.LegacyDoubleField}, {@link org.apache.lucene.analysis.LegacyNumericTokenStream}, {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * The default precision step used by {@link org.apache.lucene.legacy.LegacyLongField}, + * {@link org.apache.lucene.legacy.LegacyDoubleField}, {@link org.apache.lucene.legacy.LegacyNumericTokenStream}, {@link + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. */ public static final int PRECISION_STEP_DEFAULT = 16; /** - * The default precision step used by {@link org.apache.lucene.document.LegacyIntField} and - * {@link org.apache.lucene.document.LegacyFloatField}. + * The default precision step used by {@link org.apache.lucene.legacy.LegacyIntField} and + * {@link org.apache.lucene.legacy.LegacyFloatField}. */ public static final int PRECISION_STEP_DEFAULT_32 = 8; @@ -99,7 +101,7 @@ public final class LegacyNumericUtils { /** * Returns prefix coded bits after reducing the precision by shift bits. - * This is method is used by {@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * This is method is used by {@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * After encoding, {@code bytes.offset} will always be 0. * @param val the numeric value * @param shift how many bits to strip from the right @@ -126,7 +128,7 @@ public final class LegacyNumericUtils { /** * Returns prefix coded bits after reducing the precision by shift bits. - * This is method is used by {@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * This is method is used by {@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * After encoding, {@code bytes.offset} will always be 0. * @param val the numeric value * @param shift how many bits to strip from the right @@ -230,7 +232,7 @@ public final class LegacyNumericUtils { * {@link org.apache.lucene.search.BooleanQuery} for each call to its * {@link LongRangeBuilder#addRange(BytesRef,BytesRef)} * method. - *

    This method is used by {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + *

    This method is used by {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. */ public static void splitLongRange(final LongRangeBuilder builder, final int precisionStep, final long minBound, final long maxBound @@ -244,7 +246,7 @@ public final class LegacyNumericUtils { * {@link org.apache.lucene.search.BooleanQuery} for each call to its * {@link IntRangeBuilder#addRange(BytesRef,BytesRef)} * method. - *

    This method is used by {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + *

    This method is used by {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. */ public static void splitIntRange(final IntRangeBuilder builder, final int precisionStep, final int minBound, final int maxBound diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/legacy/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/package-info.java new file mode 100644 index 00000000000..d0167f80023 --- /dev/null +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Deprecated stuff! + */ +package org.apache.lucene.legacy; diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index 8226022e6d8..03480d779ed 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -47,8 +47,6 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FloatDocValuesField; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; @@ -57,9 +55,12 @@ import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.BaseDirectoryWrapper; @@ -72,7 +73,6 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.InfoStream; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java new file mode 100644 index 00000000000..65ff0969d9d --- /dev/null +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + +import java.io.StringReader; + +import org.apache.lucene.analysis.CannedTokenStream; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.document.Field; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; + +public class TestLegacyField extends LuceneTestCase { + + public void testLegacyDoubleField() throws Exception { + Field fields[] = new Field[] { + new LegacyDoubleField("foo", 5d, Field.Store.NO), + new LegacyDoubleField("foo", 5d, Field.Store.YES) + }; + + for (Field field : fields) { + trySetBoost(field); + trySetByteValue(field); + trySetBytesValue(field); + trySetBytesRefValue(field); + field.setDoubleValue(6d); // ok + trySetIntValue(field); + trySetFloatValue(field); + trySetLongValue(field); + trySetReaderValue(field); + trySetShortValue(field); + trySetStringValue(field); + trySetTokenStreamValue(field); + + assertEquals(6d, field.numericValue().doubleValue(), 0.0d); + } + } + + public void testLegacyFloatField() throws Exception { + Field fields[] = new Field[] { + new LegacyFloatField("foo", 5f, Field.Store.NO), + new LegacyFloatField("foo", 5f, Field.Store.YES) + }; + + for (Field field : fields) { + trySetBoost(field); + trySetByteValue(field); + trySetBytesValue(field); + trySetBytesRefValue(field); + trySetDoubleValue(field); + trySetIntValue(field); + field.setFloatValue(6f); // ok + trySetLongValue(field); + trySetReaderValue(field); + trySetShortValue(field); + trySetStringValue(field); + trySetTokenStreamValue(field); + + assertEquals(6f, field.numericValue().floatValue(), 0.0f); + } + } + + public void testLegacyIntField() throws Exception { + Field fields[] = new Field[] { + new LegacyIntField("foo", 5, Field.Store.NO), + new LegacyIntField("foo", 5, Field.Store.YES) + }; + + for (Field field : fields) { + trySetBoost(field); + trySetByteValue(field); + trySetBytesValue(field); + trySetBytesRefValue(field); + trySetDoubleValue(field); + field.setIntValue(6); // ok + trySetFloatValue(field); + trySetLongValue(field); + trySetReaderValue(field); + trySetShortValue(field); + trySetStringValue(field); + trySetTokenStreamValue(field); + + assertEquals(6, field.numericValue().intValue()); + } + } + + public void testLegacyLongField() throws Exception { + Field fields[] = new Field[] { + new LegacyLongField("foo", 5L, Field.Store.NO), + new LegacyLongField("foo", 5L, Field.Store.YES) + }; + + for (Field field : fields) { + trySetBoost(field); + trySetByteValue(field); + trySetBytesValue(field); + trySetBytesRefValue(field); + trySetDoubleValue(field); + trySetIntValue(field); + trySetFloatValue(field); + field.setLongValue(6); // ok + trySetReaderValue(field); + trySetShortValue(field); + trySetStringValue(field); + trySetTokenStreamValue(field); + + assertEquals(6L, field.numericValue().longValue()); + } + } + + private void trySetByteValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setByteValue((byte) 10); + }); + } + + private void trySetBytesValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setBytesValue(new byte[] { 5, 5 }); + }); + } + + private void trySetBytesRefValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setBytesValue(new BytesRef("bogus")); + }); + } + + private void trySetDoubleValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setDoubleValue(Double.MAX_VALUE); + }); + } + + private void trySetIntValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setIntValue(Integer.MAX_VALUE); + }); + } + + private void trySetLongValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setLongValue(Long.MAX_VALUE); + }); + } + + private void trySetFloatValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setFloatValue(Float.MAX_VALUE); + }); + } + + private void trySetReaderValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setReaderValue(new StringReader("BOO!")); + }); + } + + private void trySetShortValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setShortValue(Short.MAX_VALUE); + }); + } + + private void trySetStringValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setStringValue("BOO!"); + }); + } + + private void trySetTokenStreamValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setTokenStream(new CannedTokenStream(new Token("foo", 0, 3))); + }); + } + + private void trySetBoost(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setBoost(5.0f); + }); + } +} diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyFieldReuse.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyFieldReuse.java new file mode 100644 index 00000000000..9335290247d --- /dev/null +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyFieldReuse.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + + +import java.io.IOException; + +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.CannedTokenStream; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Field; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyNumericTokenStream; +import org.apache.lucene.legacy.LegacyNumericUtils; +import org.apache.lucene.legacy.LegacyNumericTokenStream.LegacyNumericTermAttribute; + +/** test tokenstream reuse by DefaultIndexingChain */ +public class TestLegacyFieldReuse extends BaseTokenStreamTestCase { + + public void testNumericReuse() throws IOException { + LegacyIntField legacyIntField = new LegacyIntField("foo", 5, Field.Store.NO); + + // passing null + TokenStream ts = legacyIntField.tokenStream(null, null); + assertTrue(ts instanceof LegacyNumericTokenStream); + assertEquals(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, ((LegacyNumericTokenStream)ts).getPrecisionStep()); + assertNumericContents(5, ts); + + // now reuse previous stream + legacyIntField = new LegacyIntField("foo", 20, Field.Store.NO); + TokenStream ts2 = legacyIntField.tokenStream(null, ts); + assertSame(ts, ts2); + assertNumericContents(20, ts); + + // pass a bogus stream and ensure it's still ok + legacyIntField = new LegacyIntField("foo", 2343, Field.Store.NO); + TokenStream bogus = new CannedTokenStream(new Token("bogus", 0, 5)); + ts = legacyIntField.tokenStream(null, bogus); + assertNotSame(bogus, ts); + assertNumericContents(2343, ts); + + // pass another bogus stream (numeric, but different precision step!) + legacyIntField = new LegacyIntField("foo", 42, Field.Store.NO); + assert 3 != LegacyNumericUtils.PRECISION_STEP_DEFAULT; + bogus = new LegacyNumericTokenStream(3); + ts = legacyIntField.tokenStream(null, bogus); + assertNotSame(bogus, ts); + assertNumericContents(42, ts); + } + + private void assertNumericContents(int value, TokenStream ts) throws IOException { + assertTrue(ts instanceof LegacyNumericTokenStream); + LegacyNumericTermAttribute numericAtt = ts.getAttribute(LegacyNumericTermAttribute.class); + ts.reset(); + boolean seen = false; + while (ts.incrementToken()) { + if (numericAtt.getShift() == 0) { + assertEquals(value, numericAtt.getRawValue()); + seen = true; + } + } + ts.end(); + ts.close(); + assertTrue(seen); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/util/TestLegacyNumericUtils.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyNumericUtils.java similarity index 98% rename from lucene/core/src/test/org/apache/lucene/util/TestLegacyNumericUtils.java rename to lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyNumericUtils.java index 2fb20d11db7..8607efdc893 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestLegacyNumericUtils.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyNumericUtils.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.util; +package org.apache.lucene.legacy; import java.util.Arrays; @@ -22,6 +22,13 @@ import java.util.Collections; import java.util.Iterator; import java.util.Random; +import org.apache.lucene.legacy.LegacyNumericUtils; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.LongBitSet; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.NumericUtils; + public class TestLegacyNumericUtils extends LuceneTestCase { public void testLongConversionAndOrdering() throws Exception { diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyTerms.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyTerms.java new file mode 100644 index 00000000000..27fae15e916 --- /dev/null +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyTerms.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericUtils; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.NumericUtils; + +public class TestLegacyTerms extends LuceneTestCase { + + public void testEmptyIntFieldMinMax() throws Exception { + assertNull(LegacyNumericUtils.getMinInt(EMPTY_TERMS)); + assertNull(LegacyNumericUtils.getMaxInt(EMPTY_TERMS)); + } + + public void testIntFieldMinMax() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir); + int numDocs = atLeast(100); + int minValue = Integer.MAX_VALUE; + int maxValue = Integer.MIN_VALUE; + for(int i=0;iupper) { - int a=lower; lower=upper; upper=a; - } - final BytesRef lowerBytes, upperBytes; - BytesRefBuilder b = new BytesRefBuilder(); - LegacyNumericUtils.intToPrefixCoded(lower, 0, b); - lowerBytes = b.toBytesRef(); - LegacyNumericUtils.intToPrefixCoded(upper, 0, b); - upperBytes = b.toBytesRef(); - - // test inclusive range - LegacyNumericRangeQuery tq= LegacyNumericRangeQuery.newIntRange(field, precisionStep, lower, upper, true, true); - TermRangeQuery cq=new TermRangeQuery(field, lowerBytes, upperBytes, true, true); - TopDocs tTopDocs = searcher.search(tq, 1); - TopDocs cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test exclusive range - tq= LegacyNumericRangeQuery.newIntRange(field, precisionStep, lower, upper, false, false); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, false, false); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test left exclusive range - tq= LegacyNumericRangeQuery.newIntRange(field, precisionStep, lower, upper, false, true); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, false, true); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test right exclusive range - tq= LegacyNumericRangeQuery.newIntRange(field, precisionStep, lower, upper, true, false); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, true, false); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - } - - checkTermCounts(precisionStep, totalTermCountT, totalTermCountC); - if (VERBOSE && precisionStep != Integer.MAX_VALUE) { - System.out.println("Average number of terms during random search on '" + field + "':"); - System.out.println(" Numeric query: " + (((double)totalTermCountT)/(num * 4))); - System.out.println(" Classical query: " + (((double)totalTermCountC)/(num * 4))); - } - } - - @Test - public void testEmptyEnums() throws Exception { - int count=3000; - int lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3); - // test empty enum - assert lower < upper; - assertTrue(0 < countTerms(LegacyNumericRangeQuery.newIntRange("field4", 4, lower, upper, true, true))); - assertEquals(0, countTerms(LegacyNumericRangeQuery.newIntRange("field4", 4, upper, lower, true, true))); - // test empty enum outside of bounds - lower = distance*noDocs+startOffset; - upper = 2 * lower; - assert lower < upper; - assertEquals(0, countTerms(LegacyNumericRangeQuery.newIntRange("field4", 4, lower, upper, true, true))); - } - - private int countTerms(MultiTermQuery q) throws Exception { - final Terms terms = MultiFields.getTerms(reader, q.getField()); - if (terms == null) - return 0; - final TermsEnum termEnum = q.getTermsEnum(terms); - assertNotNull(termEnum); - int count = 0; - BytesRef cur, last = null; - while ((cur = termEnum.next()) != null) { - count++; - if (last != null) { - assertTrue(last.compareTo(cur) < 0); - } - last = BytesRef.deepCopyOf(cur); - } - // LUCENE-3314: the results after next() already returned null are undefined, - // assertNull(termEnum.next()); - return count; - } - - private void checkTermCounts(int precisionStep, int termCountT, int termCountC) { - if (precisionStep == Integer.MAX_VALUE) { - assertEquals("Number of terms should be equal for unlimited precStep", termCountC, termCountT); - } else { - assertTrue("Number of terms for NRQ should be <= compared to classical TRQ", termCountT <= termCountC); - } - } - - @Test - public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception { - testRandomTrieAndClassicRangeQuery(8); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_4bit() throws Exception { - testRandomTrieAndClassicRangeQuery(4); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_2bit() throws Exception { - testRandomTrieAndClassicRangeQuery(2); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_NoTrie() throws Exception { - testRandomTrieAndClassicRangeQuery(Integer.MAX_VALUE); - } - private void testRangeSplit(int precisionStep) throws Exception { String field="ascfield"+precisionStep; // 10 random tests diff --git a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericRangeQuery64.java similarity index 72% rename from lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java rename to lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericRangeQuery64.java index 7f63fbc0136..b3ce55aa66d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericRangeQuery64.java @@ -14,28 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.search; +package org.apache.lucene.legacy; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyDoubleField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryUtils; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.TestLegacyNumericUtils; import org.apache.lucene.util.TestUtil; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -63,37 +61,37 @@ public class TestNumericRangeQuery64 extends LuceneTestCase { .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) .setMergePolicy(newLogMergePolicy())); - final FieldType storedLong = new FieldType(LegacyLongField.TYPE_NOT_STORED); + final LegacyFieldType storedLong = new LegacyFieldType(LegacyLongField.TYPE_NOT_STORED); storedLong.setStored(true); storedLong.freeze(); - final FieldType storedLong8 = new FieldType(storedLong); + final LegacyFieldType storedLong8 = new LegacyFieldType(storedLong); storedLong8.setNumericPrecisionStep(8); - final FieldType storedLong4 = new FieldType(storedLong); + final LegacyFieldType storedLong4 = new LegacyFieldType(storedLong); storedLong4.setNumericPrecisionStep(4); - final FieldType storedLong6 = new FieldType(storedLong); + final LegacyFieldType storedLong6 = new LegacyFieldType(storedLong); storedLong6.setNumericPrecisionStep(6); - final FieldType storedLong2 = new FieldType(storedLong); + final LegacyFieldType storedLong2 = new LegacyFieldType(storedLong); storedLong2.setNumericPrecisionStep(2); - final FieldType storedLongNone = new FieldType(storedLong); + final LegacyFieldType storedLongNone = new LegacyFieldType(storedLong); storedLongNone.setNumericPrecisionStep(Integer.MAX_VALUE); - final FieldType unstoredLong = LegacyLongField.TYPE_NOT_STORED; + final LegacyFieldType unstoredLong = LegacyLongField.TYPE_NOT_STORED; - final FieldType unstoredLong8 = new FieldType(unstoredLong); + final LegacyFieldType unstoredLong8 = new LegacyFieldType(unstoredLong); unstoredLong8.setNumericPrecisionStep(8); - final FieldType unstoredLong6 = new FieldType(unstoredLong); + final LegacyFieldType unstoredLong6 = new LegacyFieldType(unstoredLong); unstoredLong6.setNumericPrecisionStep(6); - final FieldType unstoredLong4 = new FieldType(unstoredLong); + final LegacyFieldType unstoredLong4 = new LegacyFieldType(unstoredLong); unstoredLong4.setNumericPrecisionStep(4); - final FieldType unstoredLong2 = new FieldType(unstoredLong); + final LegacyFieldType unstoredLong2 = new LegacyFieldType(unstoredLong); unstoredLong2.setNumericPrecisionStep(2); LegacyLongField @@ -374,137 +372,6 @@ public class TestNumericRangeQuery64 extends LuceneTestCase { dir.close(); } - private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception { - String field="field"+precisionStep; - int totalTermCountT=0,totalTermCountC=0,termCountT,termCountC; - int num = TestUtil.nextInt(random(), 10, 20); - for (int i = 0; i < num; i++) { - long lower=(long)(random().nextDouble()*noDocs*distance)+startOffset; - long upper=(long)(random().nextDouble()*noDocs*distance)+startOffset; - if (lower>upper) { - long a=lower; lower=upper; upper=a; - } - final BytesRef lowerBytes, upperBytes; - BytesRefBuilder b = new BytesRefBuilder(); - LegacyNumericUtils.longToPrefixCoded(lower, 0, b); - lowerBytes = b.toBytesRef(); - LegacyNumericUtils.longToPrefixCoded(upper, 0, b); - upperBytes = b.toBytesRef(); - - // test inclusive range - LegacyNumericRangeQuery tq= LegacyNumericRangeQuery.newLongRange(field, precisionStep, lower, upper, true, true); - TermRangeQuery cq=new TermRangeQuery(field, lowerBytes, upperBytes, true, true); - TopDocs tTopDocs = searcher.search(tq, 1); - TopDocs cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test exclusive range - tq= LegacyNumericRangeQuery.newLongRange(field, precisionStep, lower, upper, false, false); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, false, false); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test left exclusive range - tq= LegacyNumericRangeQuery.newLongRange(field, precisionStep, lower, upper, false, true); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, false, true); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test right exclusive range - tq= LegacyNumericRangeQuery.newLongRange(field, precisionStep, lower, upper, true, false); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, true, false); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - } - - checkTermCounts(precisionStep, totalTermCountT, totalTermCountC); - if (VERBOSE && precisionStep != Integer.MAX_VALUE) { - System.out.println("Average number of terms during random search on '" + field + "':"); - System.out.println(" Numeric query: " + (((double)totalTermCountT)/(num * 4))); - System.out.println(" Classical query: " + (((double)totalTermCountC)/(num * 4))); - } - } - - @Test - public void testEmptyEnums() throws Exception { - int count=3000; - long lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3); - // test empty enum - assert lower < upper; - assertTrue(0 < countTerms(LegacyNumericRangeQuery.newLongRange("field4", 4, lower, upper, true, true))); - assertEquals(0, countTerms(LegacyNumericRangeQuery.newLongRange("field4", 4, upper, lower, true, true))); - // test empty enum outside of bounds - lower = distance*noDocs+startOffset; - upper = 2L * lower; - assert lower < upper; - assertEquals(0, countTerms(LegacyNumericRangeQuery.newLongRange("field4", 4, lower, upper, true, true))); - } - - private int countTerms(MultiTermQuery q) throws Exception { - final Terms terms = MultiFields.getTerms(reader, q.getField()); - if (terms == null) - return 0; - final TermsEnum termEnum = q.getTermsEnum(terms); - assertNotNull(termEnum); - int count = 0; - BytesRef cur, last = null; - while ((cur = termEnum.next()) != null) { - count++; - if (last != null) { - assertTrue(last.compareTo(cur) < 0); - } - last = BytesRef.deepCopyOf(cur); - } - // LUCENE-3314: the results after next() already returned null are undefined, - // assertNull(termEnum.next()); - return count; - } - - private void checkTermCounts(int precisionStep, int termCountT, int termCountC) { - if (precisionStep == Integer.MAX_VALUE) { - assertEquals("Number of terms should be equal for unlimited precStep", termCountC, termCountT); - } else { - assertTrue("Number of terms for NRQ should be <= compared to classical TRQ", termCountT <= termCountC); - } - } - - @Test - public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception { - testRandomTrieAndClassicRangeQuery(8); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_6bit() throws Exception { - testRandomTrieAndClassicRangeQuery(6); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_4bit() throws Exception { - testRandomTrieAndClassicRangeQuery(4); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_2bit() throws Exception { - testRandomTrieAndClassicRangeQuery(2); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_NoTrie() throws Exception { - testRandomTrieAndClassicRangeQuery(Integer.MAX_VALUE); - } - private void testRangeSplit(int precisionStep) throws Exception { String field="ascfield"+precisionStep; // 10 random tests diff --git a/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericTokenStream.java similarity index 85% rename from lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java rename to lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericTokenStream.java index dfaa20e5a2e..a507af09e0d 100644 --- a/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericTokenStream.java @@ -14,15 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.analysis; +package org.apache.lucene.legacy; +import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; -import org.apache.lucene.analysis.LegacyNumericTokenStream.LegacyNumericTermAttributeImpl; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; -import org.apache.lucene.analysis.tokenattributes.TestCharTermAttributeImpl; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.apache.lucene.legacy.LegacyNumericTokenStream; +import org.apache.lucene.legacy.LegacyNumericUtils; +import org.apache.lucene.legacy.LegacyNumericTokenStream.LegacyNumericTermAttributeImpl; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl; @@ -150,20 +152,37 @@ public class TestNumericTokenStream extends BaseTokenStreamTestCase { public void testAttributeClone() throws Exception { LegacyNumericTermAttributeImpl att = new LegacyNumericTermAttributeImpl(); att.init(lvalue, 64, 8, 0); // set some value, to make getBytesRef() work - LegacyNumericTermAttributeImpl copy = TestCharTermAttributeImpl.assertCloneIsEqual(att); + LegacyNumericTermAttributeImpl copy = assertCloneIsEqual(att); assertNotSame(att.getBytesRef(), copy.getBytesRef()); - LegacyNumericTermAttributeImpl copy2 = TestCharTermAttributeImpl.assertCopyIsEqual(att); + LegacyNumericTermAttributeImpl copy2 = assertCopyIsEqual(att); assertNotSame(att.getBytesRef(), copy2.getBytesRef()); // LUCENE-7027 test att.init(lvalue, 64, 8, 64); // Exhausted TokenStream -> should return empty BytesRef assertEquals(new BytesRef(), att.getBytesRef()); - copy = TestCharTermAttributeImpl.assertCloneIsEqual(att); + copy = assertCloneIsEqual(att); assertEquals(new BytesRef(), copy.getBytesRef()); assertNotSame(att.getBytesRef(), copy.getBytesRef()); - copy2 = TestCharTermAttributeImpl.assertCopyIsEqual(att); + copy2 = assertCopyIsEqual(att); assertEquals(new BytesRef(), copy2.getBytesRef()); assertNotSame(att.getBytesRef(), copy2.getBytesRef()); } + public static T assertCloneIsEqual(T att) { + @SuppressWarnings("unchecked") + T clone = (T) att.clone(); + assertEquals("Clone must be equal", att, clone); + assertEquals("Clone's hashcode must be equal", att.hashCode(), clone.hashCode()); + return clone; + } + + public static T assertCopyIsEqual(T att) throws Exception { + @SuppressWarnings("unchecked") + T copy = (T) att.getClass().newInstance(); + att.copyTo(copy); + assertEquals("Copied instance must be equal", att, copy); + assertEquals("Copied instance's hashcode must be equal", att.hashCode(), copy.hashCode()); + return copy; + } + } diff --git a/lucene/core/src/java/org/apache/lucene/document/Field.java b/lucene/core/src/java/org/apache/lucene/document/Field.java index 87986101f3b..8f5f8692d66 100644 --- a/lucene/core/src/java/org/apache/lucene/document/Field.java +++ b/lucene/core/src/java/org/apache/lucene/document/Field.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.io.Reader; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.LegacyNumericTokenStream; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.BytesTermAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -426,9 +425,6 @@ public class Field implements IndexableField { if (type.indexOptions() == IndexOptions.NONE || !type.tokenized()) { throw new IllegalArgumentException("TokenStream fields must be indexed and tokenized"); } - if (type.numericType() != null) { - throw new IllegalArgumentException("cannot set private TokenStream on numeric fields"); - } this.tokenStream = tokenStream; } @@ -511,35 +507,6 @@ public class Field implements IndexableField { return null; } - final FieldType.LegacyNumericType numericType = fieldType().numericType(); - if (numericType != null) { - if (!(reuse instanceof LegacyNumericTokenStream && ((LegacyNumericTokenStream)reuse).getPrecisionStep() == type.numericPrecisionStep())) { - // lazy init the TokenStream as it is heavy to instantiate - // (attributes,...) if not needed (stored field loading) - reuse = new LegacyNumericTokenStream(type.numericPrecisionStep()); - } - final LegacyNumericTokenStream nts = (LegacyNumericTokenStream) reuse; - // initialize value in TokenStream - final Number val = (Number) fieldsData; - switch (numericType) { - case INT: - nts.setIntValue(val.intValue()); - break; - case LONG: - nts.setLongValue(val.longValue()); - break; - case FLOAT: - nts.setFloatValue(val.floatValue()); - break; - case DOUBLE: - nts.setDoubleValue(val.doubleValue()); - break; - default: - throw new AssertionError("Should never get here"); - } - return reuse; - } - if (!fieldType().tokenized()) { if (stringValue() != null) { if (!(reuse instanceof StringTokenStream)) { diff --git a/lucene/core/src/java/org/apache/lucene/document/FieldType.java b/lucene/core/src/java/org/apache/lucene/document/FieldType.java index e0f058f520e..6f206a49ca7 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FieldType.java +++ b/lucene/core/src/java/org/apache/lucene/document/FieldType.java @@ -22,30 +22,12 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.PointValues; -import org.apache.lucene.util.LegacyNumericUtils; /** * Describes the properties of a field. */ public class FieldType implements IndexableFieldType { - /** Data type of the numeric value - * @since 3.2 - * - * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead - */ - @Deprecated - public enum LegacyNumericType { - /** 32-bit integer numeric type */ - INT, - /** 64-bit long numeric type */ - LONG, - /** 32-bit float numeric type */ - FLOAT, - /** 64-bit double numeric type */ - DOUBLE - } - private boolean stored; private boolean tokenized = true; private boolean storeTermVectors; @@ -54,9 +36,7 @@ public class FieldType implements IndexableFieldType { private boolean storeTermVectorPayloads; private boolean omitNorms; private IndexOptions indexOptions = IndexOptions.NONE; - private LegacyNumericType numericType; private boolean frozen; - private int numericPrecisionStep = LegacyNumericUtils.PRECISION_STEP_DEFAULT; private DocValuesType docValuesType = DocValuesType.NONE; private int dimensionCount; private int dimensionNumBytes; @@ -73,8 +53,6 @@ public class FieldType implements IndexableFieldType { this.storeTermVectorPayloads = ref.storeTermVectorPayloads(); this.omitNorms = ref.omitNorms(); this.indexOptions = ref.indexOptions(); - this.numericType = ref.numericType(); - this.numericPrecisionStep = ref.numericPrecisionStep(); this.docValuesType = ref.docValuesType(); this.dimensionCount = ref.dimensionCount; this.dimensionNumBytes = ref.dimensionNumBytes; @@ -297,70 +275,6 @@ public class FieldType implements IndexableFieldType { this.indexOptions = value; } - /** - * Specifies the field's numeric type. - * @param type numeric type, or null if the field has no numeric type. - * @throws IllegalStateException if this FieldType is frozen against - * future modifications. - * @see #numericType() - * - * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead - */ - @Deprecated - public void setNumericType(LegacyNumericType type) { - checkIfFrozen(); - numericType = type; - } - - /** - * LegacyNumericType: if non-null then the field's value will be indexed - * numerically so that {@link org.apache.lucene.search.LegacyNumericRangeQuery} can be used at - * search time. - *

    - * The default is null (no numeric type) - * @see #setNumericType(org.apache.lucene.document.FieldType.LegacyNumericType) - * - * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead - */ - @Deprecated - public LegacyNumericType numericType() { - return numericType; - } - - /** - * Sets the numeric precision step for the field. - * @param precisionStep numeric precision step for the field - * @throws IllegalArgumentException if precisionStep is less than 1. - * @throws IllegalStateException if this FieldType is frozen against - * future modifications. - * @see #numericPrecisionStep() - * - * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead - */ - @Deprecated - public void setNumericPrecisionStep(int precisionStep) { - checkIfFrozen(); - if (precisionStep < 1) { - throw new IllegalArgumentException("precisionStep must be >= 1 (got " + precisionStep + ")"); - } - this.numericPrecisionStep = precisionStep; - } - - /** - * Precision step for numeric field. - *

    - * This has no effect if {@link #numericType()} returns null. - *

    - * The default is {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} - * @see #setNumericPrecisionStep(int) - * - * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead - */ - @Deprecated - public int numericPrecisionStep() { - return numericPrecisionStep; - } - /** * Enables points indexing. */ @@ -403,7 +317,7 @@ public class FieldType implements IndexableFieldType { /** Prints a Field for human consumption. */ @Override - public final String toString() { + public String toString() { StringBuilder result = new StringBuilder(); if (stored()) { result.append("stored"); @@ -434,12 +348,6 @@ public class FieldType implements IndexableFieldType { result.append(",indexOptions="); result.append(indexOptions); } - if (numericType != null) { - result.append(",numericType="); - result.append(numericType); - result.append(",numericPrecisionStep="); - result.append(numericPrecisionStep); - } } if (dimensionCount != 0) { if (result.length() > 0) { @@ -495,8 +403,6 @@ public class FieldType implements IndexableFieldType { result = prime * result + dimensionNumBytes; result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode()); result = prime * result + indexOptions.hashCode(); - result = prime * result + numericPrecisionStep; - result = prime * result + ((numericType == null) ? 0 : numericType.hashCode()); result = prime * result + (omitNorms ? 1231 : 1237); result = prime * result + (storeTermVectorOffsets ? 1231 : 1237); result = prime * result + (storeTermVectorPayloads ? 1231 : 1237); @@ -517,8 +423,6 @@ public class FieldType implements IndexableFieldType { if (dimensionNumBytes != other.dimensionNumBytes) return false; if (docValuesType != other.docValuesType) return false; if (indexOptions != other.indexOptions) return false; - if (numericPrecisionStep != other.numericPrecisionStep) return false; - if (numericType != other.numericType) return false; if (omitNorms != other.omitNorms) return false; if (storeTermVectorOffsets != other.storeTermVectorOffsets) return false; if (storeTermVectorPayloads != other.storeTermVectorPayloads) return false; diff --git a/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java b/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java index c8fddf728c7..a8200c6edd3 100644 --- a/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java +++ b/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java @@ -186,4 +186,33 @@ public class Rectangle { return new Rectangle(minLat, maxLat, minLon, maxLon); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Rectangle rectangle = (Rectangle) o; + + if (Double.compare(rectangle.minLat, minLat) != 0) return false; + if (Double.compare(rectangle.minLon, minLon) != 0) return false; + if (Double.compare(rectangle.maxLat, maxLat) != 0) return false; + return Double.compare(rectangle.maxLon, maxLon) == 0; + + } + + @Override + public int hashCode() { + int result; + long temp; + temp = Double.doubleToLongBits(minLat); + result = (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(minLon); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(maxLat); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(maxLon); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + return result; + } } diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBufferGuard.java b/lucene/core/src/java/org/apache/lucene/store/ByteBufferGuard.java new file mode 100644 index 00000000000..95fa17d5ea0 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/store/ByteBufferGuard.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A guard that is created for every {@link ByteBufferIndexInput} that tries on best effort + * to reject any access to the {@link ByteBuffer} behind, once it is unmapped. A single instance + * of this is used for the original and all clones, so once the original is closed and unmapped + * all clones also throw {@link AlreadyClosedException}, triggered by a {@link NullPointerException}. + *

    + * This code tries to hopefully flush any CPU caches using a store-store barrier. It also yields the + * current thread to give other threads a chance to finish in-flight requests... + */ +final class ByteBufferGuard { + + /** + * Pass in an implementation of this interface to cleanup ByteBuffers. + * MMapDirectory implements this to allow unmapping of bytebuffers with private Java APIs. + */ + @FunctionalInterface + static interface BufferCleaner { + void freeBuffer(String resourceDescription, ByteBuffer b) throws IOException; + } + + private final String resourceDescription; + private final BufferCleaner cleaner; + + /** Not volatile; see comments on visibility below! */ + private boolean invalidated = false; + + /** Used as a store-store barrier; see comments below! */ + private final AtomicInteger barrier = new AtomicInteger(); + + /** + * Creates an instance to be used for a single {@link ByteBufferIndexInput} which + * must be shared by all of its clones. + */ + public ByteBufferGuard(String resourceDescription, BufferCleaner cleaner) { + this.resourceDescription = resourceDescription; + this.cleaner = cleaner; + } + + /** + * Invalidates this guard and unmaps (if supported). + */ + public void invalidateAndUnmap(ByteBuffer... bufs) throws IOException { + if (cleaner != null) { + invalidated = true; + // This call should hopefully flush any CPU caches and as a result make + // the "invalidated" field update visible to other threads. We specifically + // don't make "invalidated" field volatile for performance reasons, hoping the + // JVM won't optimize away reads of that field and hardware should ensure + // caches are in sync after this call. This isn't entirely "fool-proof" + // (see LUCENE-7409 discussion), but it has been shown to work in practice + // and we count on this behavior. + barrier.lazySet(0); + // we give other threads a bit of time to finish reads on their ByteBuffer...: + Thread.yield(); + // finally unmap the ByteBuffers: + for (ByteBuffer b : bufs) { + cleaner.freeBuffer(resourceDescription, b); + } + } + } + + private void ensureValid() { + if (invalidated) { + // this triggers an AlreadyClosedException in ByteBufferIndexInput: + throw new NullPointerException(); + } + } + + public void getBytes(ByteBuffer receiver, byte[] dst, int offset, int length) { + ensureValid(); + receiver.get(dst, offset, length); + } + + public byte getByte(ByteBuffer receiver) { + ensureValid(); + return receiver.get(); + } + + public short getShort(ByteBuffer receiver) { + ensureValid(); + return receiver.getShort(); + } + + public int getInt(ByteBuffer receiver) { + ensureValid(); + return receiver.getInt(); + } + + public long getLong(ByteBuffer receiver) { + ensureValid(); + return receiver.getLong(); + } + + public byte getByte(ByteBuffer receiver, int pos) { + ensureValid(); + return receiver.get(pos); + } + + public short getShort(ByteBuffer receiver, int pos) { + ensureValid(); + return receiver.getShort(pos); + } + + public int getInt(ByteBuffer receiver, int pos) { + ensureValid(); + return receiver.getInt(pos); + } + + public long getLong(ByteBuffer receiver, int pos) { + ensureValid(); + return receiver.getLong(pos); + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java index 8e8ef90655a..0f6c733410b 100644 --- a/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java @@ -21,9 +21,6 @@ import java.io.EOFException; import java.io.IOException; import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; -import java.util.Iterator; - -import org.apache.lucene.util.WeakIdentityMap; /** * Base IndexInput implementation that uses an array @@ -37,35 +34,32 @@ import org.apache.lucene.util.WeakIdentityMap; * are a power-of-two (chunkSizePower). */ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessInput { - protected final BufferCleaner cleaner; protected final long length; protected final long chunkSizeMask; protected final int chunkSizePower; + protected final ByteBufferGuard guard; protected ByteBuffer[] buffers; protected int curBufIndex = -1; protected ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex] protected boolean isClone = false; - protected final WeakIdentityMap clones; - public static ByteBufferIndexInput newInstance(String resourceDescription, ByteBuffer[] buffers, long length, int chunkSizePower, BufferCleaner cleaner, boolean trackClones) { - final WeakIdentityMap clones = trackClones ? WeakIdentityMap.newConcurrentHashMap() : null; + public static ByteBufferIndexInput newInstance(String resourceDescription, ByteBuffer[] buffers, long length, int chunkSizePower, ByteBufferGuard guard) { if (buffers.length == 1) { - return new SingleBufferImpl(resourceDescription, buffers[0], length, chunkSizePower, cleaner, clones); + return new SingleBufferImpl(resourceDescription, buffers[0], length, chunkSizePower, guard); } else { - return new MultiBufferImpl(resourceDescription, buffers, 0, length, chunkSizePower, cleaner, clones); + return new MultiBufferImpl(resourceDescription, buffers, 0, length, chunkSizePower, guard); } } - ByteBufferIndexInput(String resourceDescription, ByteBuffer[] buffers, long length, int chunkSizePower, BufferCleaner cleaner, WeakIdentityMap clones) { + ByteBufferIndexInput(String resourceDescription, ByteBuffer[] buffers, long length, int chunkSizePower, ByteBufferGuard guard) { super(resourceDescription); this.buffers = buffers; this.length = length; this.chunkSizePower = chunkSizePower; this.chunkSizeMask = (1L << chunkSizePower) - 1L; - this.clones = clones; - this.cleaner = cleaner; + this.guard = guard; assert chunkSizePower >= 0 && chunkSizePower <= 30; assert (length >>> chunkSizePower) < Integer.MAX_VALUE; } @@ -73,7 +67,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public final byte readByte() throws IOException { try { - return curBuf.get(); + return guard.getByte(curBuf); } catch (BufferUnderflowException e) { do { curBufIndex++; @@ -83,7 +77,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn curBuf = buffers[curBufIndex]; curBuf.position(0); } while (!curBuf.hasRemaining()); - return curBuf.get(); + return guard.getByte(curBuf); } catch (NullPointerException npe) { throw new AlreadyClosedException("Already closed: " + this); } @@ -92,11 +86,11 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public final void readBytes(byte[] b, int offset, int len) throws IOException { try { - curBuf.get(b, offset, len); + guard.getBytes(curBuf, b, offset, len); } catch (BufferUnderflowException e) { int curAvail = curBuf.remaining(); while (len > curAvail) { - curBuf.get(b, offset, curAvail); + guard.getBytes(curBuf, b, offset, curAvail); len -= curAvail; offset += curAvail; curBufIndex++; @@ -107,7 +101,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn curBuf.position(0); curAvail = curBuf.remaining(); } - curBuf.get(b, offset, len); + guard.getBytes(curBuf, b, offset, len); } catch (NullPointerException npe) { throw new AlreadyClosedException("Already closed: " + this); } @@ -116,7 +110,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public final short readShort() throws IOException { try { - return curBuf.getShort(); + return guard.getShort(curBuf); } catch (BufferUnderflowException e) { return super.readShort(); } catch (NullPointerException npe) { @@ -127,7 +121,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public final int readInt() throws IOException { try { - return curBuf.getInt(); + return guard.getInt(curBuf); } catch (BufferUnderflowException e) { return super.readInt(); } catch (NullPointerException npe) { @@ -138,7 +132,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public final long readLong() throws IOException { try { - return curBuf.getLong(); + return guard.getLong(curBuf); } catch (BufferUnderflowException e) { return super.readLong(); } catch (NullPointerException npe) { @@ -181,7 +175,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn public byte readByte(long pos) throws IOException { try { final int bi = (int) (pos >> chunkSizePower); - return buffers[bi].get((int) (pos & chunkSizeMask)); + return guard.getByte(buffers[bi], (int) (pos & chunkSizeMask)); } catch (IndexOutOfBoundsException ioobe) { throw new EOFException("seek past EOF: " + this); } catch (NullPointerException npe) { @@ -207,7 +201,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn public short readShort(long pos) throws IOException { final int bi = (int) (pos >> chunkSizePower); try { - return buffers[bi].getShort((int) (pos & chunkSizeMask)); + return guard.getShort(buffers[bi], (int) (pos & chunkSizeMask)); } catch (IndexOutOfBoundsException ioobe) { // either it's a boundary, or read past EOF, fall back: setPos(pos, bi); @@ -221,7 +215,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn public int readInt(long pos) throws IOException { final int bi = (int) (pos >> chunkSizePower); try { - return buffers[bi].getInt((int) (pos & chunkSizeMask)); + return guard.getInt(buffers[bi], (int) (pos & chunkSizeMask)); } catch (IndexOutOfBoundsException ioobe) { // either it's a boundary, or read past EOF, fall back: setPos(pos, bi); @@ -235,7 +229,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn public long readLong(long pos) throws IOException { final int bi = (int) (pos >> chunkSizePower); try { - return buffers[bi].getLong((int) (pos & chunkSizeMask)); + return guard.getLong(buffers[bi], (int) (pos & chunkSizeMask)); } catch (IndexOutOfBoundsException ioobe) { // either it's a boundary, or read past EOF, fall back: setPos(pos, bi); @@ -285,11 +279,6 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn final ByteBufferIndexInput clone = newCloneInstance(getFullSliceDescription(sliceDescription), newBuffers, ofs, length); clone.isClone = true; - - // register the new clone in our clone list to clean it up on closing: - if (clones != null) { - this.clones.put(clone, Boolean.TRUE); - } return clone; } @@ -299,9 +288,9 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn protected ByteBufferIndexInput newCloneInstance(String newResourceDescription, ByteBuffer[] newBuffers, int offset, long length) { if (newBuffers.length == 1) { newBuffers[0].position(offset); - return new SingleBufferImpl(newResourceDescription, newBuffers[0].slice(), length, chunkSizePower, this.cleaner, this.clones); + return new SingleBufferImpl(newResourceDescription, newBuffers[0].slice(), length, chunkSizePower, this.guard); } else { - return new MultiBufferImpl(newResourceDescription, newBuffers, offset, length, chunkSizePower, cleaner, clones); + return new MultiBufferImpl(newResourceDescription, newBuffers, offset, length, chunkSizePower, guard); } } @@ -335,25 +324,11 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn // make local copy, then un-set early final ByteBuffer[] bufs = buffers; unsetBuffers(); - if (clones != null) { - clones.remove(this); - } if (isClone) return; - // for extra safety unset also all clones' buffers: - if (clones != null) { - for (Iterator it = this.clones.keyIterator(); it.hasNext();) { - final ByteBufferIndexInput clone = it.next(); - assert clone.isClone; - clone.unsetBuffers(); - } - this.clones.clear(); - } - - for (final ByteBuffer b : bufs) { - freeBuffer(b); - } + // tell the guard to invalidate and later unmap the bytebuffers (if supported): + guard.invalidateAndUnmap(bufs); } finally { unsetBuffers(); } @@ -367,31 +342,12 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn curBuf = null; curBufIndex = 0; } - - /** - * Called when the contents of a buffer will be no longer needed. - */ - private void freeBuffer(ByteBuffer b) throws IOException { - if (cleaner != null) { - cleaner.freeBuffer(this, b); - } - } - - /** - * Pass in an implementation of this interface to cleanup ByteBuffers. - * MMapDirectory implements this to allow unmapping of bytebuffers with private Java APIs. - */ - @FunctionalInterface - static interface BufferCleaner { - void freeBuffer(ByteBufferIndexInput parent, ByteBuffer b) throws IOException; - } /** Optimization of ByteBufferIndexInput for when there is only one buffer */ static final class SingleBufferImpl extends ByteBufferIndexInput { - SingleBufferImpl(String resourceDescription, ByteBuffer buffer, long length, int chunkSizePower, - BufferCleaner cleaner, WeakIdentityMap clones) { - super(resourceDescription, new ByteBuffer[] { buffer }, length, chunkSizePower, cleaner, clones); + SingleBufferImpl(String resourceDescription, ByteBuffer buffer, long length, int chunkSizePower, ByteBufferGuard guard) { + super(resourceDescription, new ByteBuffer[] { buffer }, length, chunkSizePower, guard); this.curBufIndex = 0; this.curBuf = buffer; buffer.position(0); @@ -426,7 +382,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public byte readByte(long pos) throws IOException { try { - return curBuf.get((int) pos); + return guard.getByte(curBuf, (int) pos); } catch (IllegalArgumentException e) { if (pos < 0) { throw new IllegalArgumentException("Seeking to negative position: " + this, e); @@ -441,7 +397,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public short readShort(long pos) throws IOException { try { - return curBuf.getShort((int) pos); + return guard.getShort(curBuf, (int) pos); } catch (IllegalArgumentException e) { if (pos < 0) { throw new IllegalArgumentException("Seeking to negative position: " + this, e); @@ -456,7 +412,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public int readInt(long pos) throws IOException { try { - return curBuf.getInt((int) pos); + return guard.getInt(curBuf, (int) pos); } catch (IllegalArgumentException e) { if (pos < 0) { throw new IllegalArgumentException("Seeking to negative position: " + this, e); @@ -471,7 +427,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public long readLong(long pos) throws IOException { try { - return curBuf.getLong((int) pos); + return guard.getLong(curBuf, (int) pos); } catch (IllegalArgumentException e) { if (pos < 0) { throw new IllegalArgumentException("Seeking to negative position: " + this, e); @@ -489,8 +445,8 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn private final int offset; MultiBufferImpl(String resourceDescription, ByteBuffer[] buffers, int offset, long length, int chunkSizePower, - BufferCleaner cleaner, WeakIdentityMap clones) { - super(resourceDescription, buffers, length, chunkSizePower, cleaner, clones); + ByteBufferGuard guard) { + super(resourceDescription, buffers, length, chunkSizePower, guard); this.offset = offset; try { seek(0L); diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index 60ca103a047..c0e35197f0e 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -36,7 +36,7 @@ import java.util.concurrent.Future; import java.lang.invoke.MethodHandle; import java.lang.reflect.Method; -import org.apache.lucene.store.ByteBufferIndexInput.BufferCleaner; +import org.apache.lucene.store.ByteBufferGuard.BufferCleaner; import org.apache.lucene.util.Constants; import org.apache.lucene.util.SuppressForbidden; @@ -240,7 +240,7 @@ public class MMapDirectory extends FSDirectory { final boolean useUnmap = getUseUnmap(); return ByteBufferIndexInput.newInstance(resourceDescription, map(resourceDescription, c, 0, c.size()), - c.size(), chunkSizePower, useUnmap ? CLEANER : null, useUnmap); + c.size(), chunkSizePower, new ByteBufferGuard(resourceDescription, useUnmap ? CLEANER : null)); } } @@ -370,7 +370,7 @@ public class MMapDirectory extends FSDirectory { final MethodHandle unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)) .asType(methodType(void.class, ByteBuffer.class)); - return (BufferCleaner) (ByteBufferIndexInput parent, ByteBuffer buffer) -> { + return (BufferCleaner) (String resourceDescription, ByteBuffer buffer) -> { if (directBufferClass.isInstance(buffer)) { final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { try { @@ -381,7 +381,7 @@ public class MMapDirectory extends FSDirectory { } }); if (error != null) { - throw new IOException("Unable to unmap the mapped buffer: " + parent.toString(), error); + throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, error); } } }; diff --git a/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestGrowableByteArrayDataOutput.java b/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestGrowableByteArrayDataOutput.java index fb90d925eb9..3820733ecca 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestGrowableByteArrayDataOutput.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestGrowableByteArrayDataOutput.java @@ -58,7 +58,7 @@ public class TestGrowableByteArrayDataOutput extends LuceneTestCase { public void testWriteLargeStrings() throws Exception { int minSizeForDoublePass = GrowableByteArrayDataOutput.MIN_UTF8_SIZE_TO_ENABLE_DOUBLE_PASS_ENCODING; - int num = atLeast(1000); + int num = atLeast(100); for (int i = 0; i < num; i++) { String unicode = TestUtil.randomRealisticUnicodeString(random(), minSizeForDoublePass, 10 * minSizeForDoublePass); byte[] utf8 = new byte[unicode.length() * UnicodeUtil.MAX_UTF8_BYTES_PER_CHAR]; diff --git a/lucene/core/src/test/org/apache/lucene/document/TestField.java b/lucene/core/src/test/org/apache/lucene/document/TestField.java index 92d6a832141..4ef7ffbcdcc 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestField.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestField.java @@ -79,29 +79,7 @@ public class TestField extends LuceneTestCase { assertEquals("DoublePoint ", field.toString()); } - public void testLegacyDoubleField() throws Exception { - Field fields[] = new Field[] { - new LegacyDoubleField("foo", 5d, Field.Store.NO), - new LegacyDoubleField("foo", 5d, Field.Store.YES) - }; - for (Field field : fields) { - trySetBoost(field); - trySetByteValue(field); - trySetBytesValue(field); - trySetBytesRefValue(field); - field.setDoubleValue(6d); // ok - trySetIntValue(field); - trySetFloatValue(field); - trySetLongValue(field); - trySetReaderValue(field); - trySetShortValue(field); - trySetStringValue(field); - trySetTokenStreamValue(field); - - assertEquals(6d, field.numericValue().doubleValue(), 0.0d); - } - } public void testDoubleDocValuesField() throws Exception { DoubleDocValuesField field = new DoubleDocValuesField("foo", 5d); @@ -185,30 +163,6 @@ public class TestField extends LuceneTestCase { assertEquals("FloatPoint ", field.toString()); } - public void testLegacyFloatField() throws Exception { - Field fields[] = new Field[] { - new LegacyFloatField("foo", 5f, Field.Store.NO), - new LegacyFloatField("foo", 5f, Field.Store.YES) - }; - - for (Field field : fields) { - trySetBoost(field); - trySetByteValue(field); - trySetBytesValue(field); - trySetBytesRefValue(field); - trySetDoubleValue(field); - trySetIntValue(field); - field.setFloatValue(6f); // ok - trySetLongValue(field); - trySetReaderValue(field); - trySetShortValue(field); - trySetStringValue(field); - trySetTokenStreamValue(field); - - assertEquals(6f, field.numericValue().floatValue(), 0.0f); - } - } - public void testIntPoint() throws Exception { Field field = new IntPoint("foo", 5); @@ -253,30 +207,6 @@ public class TestField extends LuceneTestCase { assertEquals("IntPoint ", field.toString()); } - public void testLegacyIntField() throws Exception { - Field fields[] = new Field[] { - new LegacyIntField("foo", 5, Field.Store.NO), - new LegacyIntField("foo", 5, Field.Store.YES) - }; - - for (Field field : fields) { - trySetBoost(field); - trySetByteValue(field); - trySetBytesValue(field); - trySetBytesRefValue(field); - trySetDoubleValue(field); - field.setIntValue(6); // ok - trySetFloatValue(field); - trySetLongValue(field); - trySetReaderValue(field); - trySetShortValue(field); - trySetStringValue(field); - trySetTokenStreamValue(field); - - assertEquals(6, field.numericValue().intValue()); - } - } - public void testNumericDocValuesField() throws Exception { NumericDocValuesField field = new NumericDocValuesField("foo", 5L); @@ -340,30 +270,6 @@ public class TestField extends LuceneTestCase { assertEquals("LongPoint ", field.toString()); } - public void testLegacyLongField() throws Exception { - Field fields[] = new Field[] { - new LegacyLongField("foo", 5L, Field.Store.NO), - new LegacyLongField("foo", 5L, Field.Store.YES) - }; - - for (Field field : fields) { - trySetBoost(field); - trySetByteValue(field); - trySetBytesValue(field); - trySetBytesRefValue(field); - trySetDoubleValue(field); - trySetIntValue(field); - trySetFloatValue(field); - field.setLongValue(6); // ok - trySetReaderValue(field); - trySetShortValue(field); - trySetStringValue(field); - trySetTokenStreamValue(field); - - assertEquals(6L, field.numericValue().longValue()); - } - } - public void testSortedBytesDocValuesField() throws Exception { SortedDocValuesField field = new SortedDocValuesField("foo", new BytesRef("bar")); diff --git a/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java b/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java index da76f40fe71..9214cb9f844 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java @@ -18,7 +18,6 @@ package org.apache.lucene.document; import java.lang.reflect.Method; -import org.apache.lucene.document.FieldType.LegacyNumericType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.PointValues; @@ -58,14 +57,6 @@ public class TestFieldType extends LuceneTestCase { ft7.setOmitNorms(true); assertFalse(ft7.equals(ft)); - FieldType ft8 = new FieldType(); - ft8.setNumericType(LegacyNumericType.DOUBLE); - assertFalse(ft8.equals(ft)); - - FieldType ft9 = new FieldType(); - ft9.setNumericPrecisionStep(3); - assertFalse(ft9.equals(ft)); - FieldType ft10 = new FieldType(); ft10.setStoreTermVectors(true); assertFalse(ft10.equals(ft)); diff --git a/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java b/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java index e75ae85b8a6..2cfb2f84002 100644 --- a/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java +++ b/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java @@ -30,7 +30,7 @@ public class TestGeoUtils extends LuceneTestCase { // We rely heavily on GeoUtils.circleToBBox so we test it here: public void testRandomCircleToBBox() throws Exception { - int iters = atLeast(1000); + int iters = atLeast(100); for(int iter=0;iter 0) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java index b36cfefa9ff..977df3da618 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java @@ -24,16 +24,12 @@ import java.util.Collections; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.CannedTokenStream; -import org.apache.lucene.analysis.LegacyNumericTokenStream.LegacyNumericTermAttribute; -import org.apache.lucene.analysis.LegacyNumericTokenStream; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.StringField; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; /** test tokenstream reuse by DefaultIndexingChain */ public class TestFieldReuse extends BaseTokenStreamTestCase { @@ -61,7 +57,7 @@ public class TestFieldReuse extends BaseTokenStreamTestCase { // pass a bogus stream and ensure it's still ok stringField = new StringField("foo", "beer", Field.Store.NO); - TokenStream bogus = new LegacyNumericTokenStream(); + TokenStream bogus = new CannedTokenStream(); ts = stringField.tokenStream(null, bogus); assertNotSame(ts, bogus); assertTokenStreamContents(ts, @@ -71,37 +67,6 @@ public class TestFieldReuse extends BaseTokenStreamTestCase { ); } - public void testNumericReuse() throws IOException { - LegacyIntField legacyIntField = new LegacyIntField("foo", 5, Field.Store.NO); - - // passing null - TokenStream ts = legacyIntField.tokenStream(null, null); - assertTrue(ts instanceof LegacyNumericTokenStream); - assertEquals(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, ((LegacyNumericTokenStream)ts).getPrecisionStep()); - assertNumericContents(5, ts); - - // now reuse previous stream - legacyIntField = new LegacyIntField("foo", 20, Field.Store.NO); - TokenStream ts2 = legacyIntField.tokenStream(null, ts); - assertSame(ts, ts2); - assertNumericContents(20, ts); - - // pass a bogus stream and ensure it's still ok - legacyIntField = new LegacyIntField("foo", 2343, Field.Store.NO); - TokenStream bogus = new CannedTokenStream(new Token("bogus", 0, 5)); - ts = legacyIntField.tokenStream(null, bogus); - assertNotSame(bogus, ts); - assertNumericContents(2343, ts); - - // pass another bogus stream (numeric, but different precision step!) - legacyIntField = new LegacyIntField("foo", 42, Field.Store.NO); - assert 3 != LegacyNumericUtils.PRECISION_STEP_DEFAULT; - bogus = new LegacyNumericTokenStream(3); - ts = legacyIntField.tokenStream(null, bogus); - assertNotSame(bogus, ts); - assertNumericContents(42, ts); - } - static class MyField implements IndexableField { TokenStream lastSeen; TokenStream lastReturned; @@ -163,20 +128,4 @@ public class TestFieldReuse extends BaseTokenStreamTestCase { iw.close(); dir.close(); } - - private void assertNumericContents(int value, TokenStream ts) throws IOException { - assertTrue(ts instanceof LegacyNumericTokenStream); - LegacyNumericTermAttribute numericAtt = ts.getAttribute(LegacyNumericTermAttribute.class); - ts.reset(); - boolean seen = false; - while (ts.incrementToken()) { - if (numericAtt.getShift() == 0) { - assertEquals(value, numericAtt.getRawValue()); - seen = true; - } - } - ts.end(); - ts.close(); - assertTrue(seen); - } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java index 363ccb2ce59..5045396e43e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java @@ -1217,7 +1217,7 @@ public class TestIndexSorting extends LuceneTestCase { if (TEST_NIGHTLY) { numDocs = atLeast(100000); } else { - numDocs = atLeast(10000); + numDocs = atLeast(1000); } List docs = new ArrayList<>(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java index 8d4c5c7903f..55aa6e036d8 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java @@ -72,6 +72,7 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase { dir.close(); } + @Slow public void testStressUpdateSameID() throws Exception { int iters = atLeast(100); for(int iter=0;iter { + try { + shotgun.await(); + for (int i = 0; i < 10; i++) { + clone.seek(0); + clone.readBytes(accum, 0, accum.length); + } + } catch (IOException | AlreadyClosedException ok) { + // OK + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + t1.start(); + shotgun.countDown(); + in.close(); + t1.join(); + dir.close(); + } + } } diff --git a/lucene/join/build.xml b/lucene/join/build.xml index b5360c4c7b9..b6878b809c6 100644 --- a/lucene/join/build.xml +++ b/lucene/join/build.xml @@ -26,6 +26,7 @@ + @@ -34,13 +35,14 @@ - + - + diff --git a/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java index a9b11ed7008..4bb692a50da 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java @@ -19,8 +19,6 @@ package org.apache.lucene.search.join; import java.io.IOException; import java.util.function.LongConsumer; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FieldType.LegacyNumericType; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; @@ -28,10 +26,11 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.legacy.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; abstract class DocValuesTermsCollector extends SimpleCollector { @@ -85,13 +84,13 @@ abstract class DocValuesTermsCollector extends SimpleCollector { return (l) -> LegacyNumericUtils.longToPrefixCoded(l, 0, bytes); default: throw new IllegalArgumentException("Unsupported "+type+ - ". Only "+ LegacyNumericType.INT+" and "+ FieldType.LegacyNumericType.LONG+" are supported." + ". Only "+ LegacyNumericType.INT+" and "+ LegacyNumericType.LONG+" are supported." + "Field "+fieldName ); } } /** this adapter is quite weird. ords are per doc index, don't use ords across different docs*/ - static Function sortedNumericAsSortedSetDocValues(String field, FieldType.LegacyNumericType numTyp) { + static Function sortedNumericAsSortedSetDocValues(String field, LegacyNumericType numTyp) { return (ctx) -> { final SortedNumericDocValues numerics = DocValues.getSortedNumeric(ctx, field); final BytesRefBuilder bytes = new BytesRefBuilder(); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java index b0133e570da..49423947383 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java @@ -26,7 +26,7 @@ import java.util.function.BiConsumer; import java.util.function.LongFunction; import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; @@ -123,8 +123,8 @@ public final class JoinUtil { * @param multipleValuesPerDocument Whether the from field has multiple terms per document * when true fromField might be {@link DocValuesType#SORTED_NUMERIC}, * otherwise fromField should be {@link DocValuesType#NUMERIC} - * @param toField The to field to join to, should be {@link org.apache.lucene.document.LegacyIntField} or {@link org.apache.lucene.document.LegacyLongField} - * @param numericType either {@link org.apache.lucene.document.FieldType.LegacyNumericType#INT} or {@link org.apache.lucene.document.FieldType.LegacyNumericType#LONG}, it should correspond to fromField and toField types + * @param toField The to field to join to, should be {@link org.apache.lucene.legacy.LegacyIntField} or {@link org.apache.lucene.legacy.LegacyLongField} + * @param numericType either {@link LegacyNumericType#INT} or {@link LegacyNumericType#LONG}, it should correspond to fromField and toField types * @param fromQuery The query to match documents on the from side * @param fromSearcher The searcher that executed the specified fromQuery * @param scoreMode Instructs how scores from the fromQuery are mapped to the returned query diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java index 3b03bd39c38..a39c25fcc93 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; @@ -37,7 +38,6 @@ import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash; import org.apache.lucene.util.FixedBitSet; -import org.apache.lucene.util.LegacyNumericUtils; class TermsIncludingScoreQuery extends Query { diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index b29e9ff0e9e..6d9eb2ab339 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -37,12 +37,9 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleDocValuesField; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType.LegacyNumericType; import org.apache.lucene.document.FloatDocValuesField; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; @@ -59,6 +56,9 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.MultiDocValues.OrdinalMap; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.NumericDocValues; diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java index 45e95510dc5..a7857207a39 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java @@ -45,7 +45,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; @@ -457,9 +456,6 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase { Document doc = new Document(); long randomLong = random().nextLong(); doc.add(new NumericDocValuesField("numeric", randomLong)); - if (random().nextBoolean()) { - doc.add(new LegacyLongField("numeric", randomLong, Field.Store.NO)); - } int numValues = atLeast(5); for (int i = 0; i < numValues; i++) { randomLong = random().nextLong(); @@ -468,9 +464,6 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase { // randomly duplicate field/value doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong)); } - if (random().nextBoolean()) { - doc.add(new LegacyLongField("numeric", randomLong, Field.Store.NO)); - } } BytesRef randomTerm = new BytesRef(randomTerm()); doc.add(new BinaryDocValuesField("binary", randomTerm)); diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java index da9fdc5dc5d..05a3b239959 100644 --- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java +++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java @@ -21,11 +21,10 @@ import java.util.HashMap; import java.util.Map; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatDocValuesField; -import org.apache.lucene.document.LegacyFloatField; import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInvertState; @@ -331,7 +330,7 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase { new BytesRef("")); Field weeksAtNumberOneField = new FloatDocValuesField("weeksAtNumberOne", 0.0F); - Field weeksStoredField = new LegacyFloatField("weeks", 0.0F, Store.YES); + Field weeksStoredField = new StoredField("weeks", 0.0F); Field idField = newStringField("id", "", Field.Store.YES); Field songField = newTextField("song", "", Field.Store.NO); Field storedArtistField = newTextField("artistName", "", Field.Store.NO); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java index 89722db075f..32a610bf8a9 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java @@ -265,6 +265,7 @@ public class TestMoreLikeThis extends LuceneTestCase { return writer.numDocs() - 1; } + @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-7161") public void testMultiFieldShouldReturnPerFieldBooleanQuery() throws Exception { IndexReader reader = null; Directory dir = newDirectory(); diff --git a/lucene/queryparser/build.xml b/lucene/queryparser/build.xml index b6e43c2ce26..f1d59a34a4c 100644 --- a/lucene/queryparser/build.xml +++ b/lucene/queryparser/build.xml @@ -25,15 +25,17 @@ + - + - + diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java index 8ae7d5e2e43..0781afb533c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java @@ -16,7 +16,8 @@ */ package org.apache.lucene.queryparser.flexible.standard.builders; -import org.apache.lucene.document.FieldType; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; @@ -25,12 +26,11 @@ import org.apache.lucene.queryparser.flexible.messages.MessageImpl; import org.apache.lucene.queryparser.flexible.standard.config.LegacyNumericConfig; import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericRangeQueryNode; -import org.apache.lucene.search.LegacyNumericRangeQuery; /** - * Builds {@link org.apache.lucene.search.LegacyNumericRangeQuery}s out of {@link LegacyNumericRangeQueryNode}s. + * Builds {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}s out of {@link LegacyNumericRangeQueryNode}s. * - * @see org.apache.lucene.search.LegacyNumericRangeQuery + * @see org.apache.lucene.legacy.LegacyNumericRangeQuery * @see LegacyNumericRangeQueryNode * @deprecated Index with points and use {@link PointRangeQueryNodeBuilder} instead. */ @@ -56,7 +56,7 @@ public class LegacyNumericRangeQueryNodeBuilder implements StandardQueryBuilder Number upperNumber = upperNumericNode.getValue(); LegacyNumericConfig numericConfig = numericRangeNode.getNumericConfig(); - FieldType.LegacyNumericType numberType = numericConfig.getType(); + LegacyNumericType numberType = numericConfig.getType(); String field = StringUtils.toString(numericRangeNode.getField()); boolean minInclusive = numericRangeNode.isLowerInclusive(); boolean maxInclusive = numericRangeNode.isUpperInclusive(); diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java index 6cd3c490e57..038023e65ae 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java @@ -19,14 +19,13 @@ package org.apache.lucene.queryparser.flexible.standard.config; import java.text.NumberFormat; import java.util.Objects; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericType; /** * This class holds the configuration used to parse numeric queries and create - * {@link org.apache.lucene.search.LegacyNumericRangeQuery}s. + * {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}s. * - * @see org.apache.lucene.search.LegacyNumericRangeQuery + * @see org.apache.lucene.legacy.LegacyNumericRangeQuery * @see NumberFormat * @deprecated Index with Points instead and use {@link PointsConfig} */ @@ -37,7 +36,7 @@ public class LegacyNumericConfig { private NumberFormat format; - private FieldType.LegacyNumericType type; + private LegacyNumericType type; /** * Constructs a {@link LegacyNumericConfig} object. @@ -52,7 +51,7 @@ public class LegacyNumericConfig { * * @see LegacyNumericConfig#setPrecisionStep(int) * @see LegacyNumericConfig#setNumberFormat(NumberFormat) - * @see #setType(org.apache.lucene.document.FieldType.LegacyNumericType) + * @see #setType(LegacyNumericType) */ public LegacyNumericConfig(int precisionStep, NumberFormat format, LegacyNumericType type) { @@ -67,7 +66,7 @@ public class LegacyNumericConfig { * * @return the precision used to index the numeric values * - * @see org.apache.lucene.search.LegacyNumericRangeQuery#getPrecisionStep() + * @see org.apache.lucene.legacy.LegacyNumericRangeQuery#getPrecisionStep() */ public int getPrecisionStep() { return precisionStep; @@ -79,7 +78,7 @@ public class LegacyNumericConfig { * @param precisionStep * the precision used to index the numeric values * - * @see org.apache.lucene.search.LegacyNumericRangeQuery#getPrecisionStep() + * @see org.apache.lucene.legacy.LegacyNumericRangeQuery#getPrecisionStep() */ public void setPrecisionStep(int precisionStep) { this.precisionStep = precisionStep; diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java index 27c285eb34c..20cde351ce7 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java @@ -16,8 +16,7 @@ */ package org.apache.lucene.queryparser.flexible.standard.nodes; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; import org.apache.lucene.queryparser.flexible.messages.MessageImpl; @@ -57,13 +56,13 @@ public class LegacyNumericRangeQueryNode extends private static LegacyNumericType getNumericDataType(Number number) throws QueryNodeException { if (number instanceof Long) { - return FieldType.LegacyNumericType.LONG; + return LegacyNumericType.LONG; } else if (number instanceof Integer) { - return FieldType.LegacyNumericType.INT; + return LegacyNumericType.INT; } else if (number instanceof Double) { return LegacyNumericType.DOUBLE; } else if (number instanceof Float) { - return FieldType.LegacyNumericType.FLOAT; + return LegacyNumericType.FLOAT; } else { throw new QueryNodeException( new MessageImpl( diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java index f7aef3f477b..9f4505f056c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java @@ -16,19 +16,19 @@ */ package org.apache.lucene.queryparser.xml.builders; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.util.LegacyNumericUtils; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queryparser.xml.DOMUtils; import org.apache.lucene.queryparser.xml.ParserException; import org.apache.lucene.queryparser.xml.QueryBuilder; import org.w3c.dom.Element; /** - * Creates a {@link org.apache.lucene.search.LegacyNumericRangeQuery}. The table below specifies the required + * Creates a {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. The table below specifies the required * attributes and the defaults if optional attributes are omitted. For more * detail on what each of the attributes actually do, consult the documentation - * for {@link org.apache.lucene.search.LegacyNumericRangeQuery}: + * for {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}: * * * diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java index c6ab7f5ffff..398923e299f 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java @@ -32,15 +32,15 @@ import java.util.TimeZone; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType.LegacyNumericType; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax; import org.apache.lucene.queryparser.flexible.standard.config.NumberDateFormat; @@ -179,7 +179,7 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { ; randomNumberMap.put(LegacyNumericType.LONG.name(), randomLong); - randomNumberMap.put(FieldType.LegacyNumericType.INT.name(), randomInt); + randomNumberMap.put(LegacyNumericType.INT.name(), randomInt); randomNumberMap.put(LegacyNumericType.FLOAT.name(), randomFloat); randomNumberMap.put(LegacyNumericType.DOUBLE.name(), randomDouble); randomNumberMap.put(DATE_FIELD_NAME, randomDate); @@ -201,7 +201,7 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { numericConfigMap.put(type.name(), new LegacyNumericConfig(PRECISION_STEP, NUMBER_FORMAT, type)); - FieldType ft = new FieldType(LegacyIntField.TYPE_NOT_STORED); + LegacyFieldType ft = new LegacyFieldType(LegacyIntField.TYPE_NOT_STORED); ft.setNumericType(type); ft.setStored(true); ft.setNumericPrecisionStep(PRECISION_STEP); @@ -231,7 +231,7 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { numericConfigMap.put(DATE_FIELD_NAME, new LegacyNumericConfig(PRECISION_STEP, DATE_FORMAT, LegacyNumericType.LONG)); - FieldType ft = new FieldType(LegacyLongField.TYPE_NOT_STORED); + LegacyFieldType ft = new LegacyFieldType(LegacyLongField.TYPE_NOT_STORED); ft.setStored(true); ft.setNumericPrecisionStep(PRECISION_STEP); LegacyLongField dateField = new LegacyLongField(DATE_FIELD_NAME, 0l, ft); @@ -268,10 +268,10 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { || DATE_FIELD_NAME.equals(fieldName)) { number = -number.longValue(); - } else if (FieldType.LegacyNumericType.DOUBLE.name().equals(fieldName)) { + } else if (LegacyNumericType.DOUBLE.name().equals(fieldName)) { number = -number.doubleValue(); - } else if (FieldType.LegacyNumericType.FLOAT.name().equals(fieldName)) { + } else if (LegacyNumericType.FLOAT.name().equals(fieldName)) { number = -number.floatValue(); } else if (LegacyNumericType.INT.name().equals(fieldName)) { @@ -299,16 +299,16 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { numericFieldMap.get(LegacyNumericType.DOUBLE.name()).setDoubleValue( number.doubleValue()); - number = getNumberType(numberType, FieldType.LegacyNumericType.INT.name()); - numericFieldMap.get(FieldType.LegacyNumericType.INT.name()).setIntValue( + number = getNumberType(numberType, LegacyNumericType.INT.name()); + numericFieldMap.get(LegacyNumericType.INT.name()).setIntValue( number.intValue()); number = getNumberType(numberType, LegacyNumericType.LONG.name()); - numericFieldMap.get(FieldType.LegacyNumericType.LONG.name()).setLongValue( + numericFieldMap.get(LegacyNumericType.LONG.name()).setLongValue( number.longValue()); - number = getNumberType(numberType, FieldType.LegacyNumericType.FLOAT.name()); - numericFieldMap.get(FieldType.LegacyNumericType.FLOAT.name()).setFloatValue( + number = getNumberType(numberType, LegacyNumericType.FLOAT.name()); + numericFieldMap.get(LegacyNumericType.FLOAT.name()).setFloatValue( number.floatValue()); number = getNumberType(numberType, DATE_FIELD_NAME); @@ -456,7 +456,7 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { StringBuilder sb = new StringBuilder(); - for (LegacyNumericType type : FieldType.LegacyNumericType.values()) { + for (LegacyNumericType type : LegacyNumericType.values()) { String boundStr = numberToString(getNumberType(boundType, type.name())); sb.append("+").append(type.name()).append(operator).append('"').append(boundStr).append('"').append(' '); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java index 71b627e74cd..4763005d985 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java @@ -20,10 +20,10 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.legacy.LegacyIntField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java index 8fc0641e4e7..0bc019595be 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java @@ -16,9 +16,9 @@ */ package org.apache.lucene.queryparser.xml.builders; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; import org.apache.lucene.queryparser.xml.ParserException; import org.w3c.dom.Document; import org.xml.sax.SAXException; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/FloatRangeField.java b/lucene/sandbox/src/java/org/apache/lucene/document/FloatRangeField.java new file mode 100644 index 00000000000..e138ae2057d --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/document/FloatRangeField.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.document.RangeFieldQuery.QueryType; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; + +/** + * An indexed Float Range field. + *

    + * This field indexes dimensional ranges defined as min/max pairs. It supports + * up to a maximum of 4 dimensions (indexed as 8 numeric values). With 1 dimension representing a single float range, + * 2 dimensions representing a bounding box, 3 dimensions a bounding cube, and 4 dimensions a tesseract. + *

    + * Multiple values for the same field in one document is supported, and open ended ranges can be defined using + * {@code Float.NEGATIVE_INFINITY} and {@code Float.POSITIVE_INFINITY}. + * + *

    + * This field defines the following static factory methods for common search operations over float ranges: + *

      + *
    • {@link #newIntersectsQuery newIntersectsQuery()} matches ranges that intersect the defined search range. + *
    • {@link #newWithinQuery newWithinQuery()} matches ranges that are within the defined search range. + *
    • {@link #newContainsQuery newContainsQuery()} matches ranges that contain the defined search range. + *
    + */ +public class FloatRangeField extends Field { + /** stores float values so number of bytes is 4 */ + public static final int BYTES = Float.BYTES; + + /** + * Create a new FloatRangeField type, from min/max parallel arrays + * + * @param name field name. must not be null. + * @param min range min values; each entry is the min value for the dimension + * @param max range max values; each entry is the max value for the dimension + */ + public FloatRangeField(String name, final float[] min, final float[] max) { + super(name, getType(min.length)); + setRangeValues(min, max); + } + + /** set the field type */ + private static FieldType getType(int dimensions) { + if (dimensions > 4) { + throw new IllegalArgumentException("FloatRangeField does not support greater than 4 dimensions"); + } + + FieldType ft = new FieldType(); + // dimensions is set as 2*dimension size (min/max per dimension) + ft.setDimensions(dimensions*2, BYTES); + ft.freeze(); + return ft; + } + + /** + * Changes the values of the field. + * @param min array of min values. (accepts {@code Float.NEGATIVE_INFINITY}) + * @param max array of max values. (accepts {@code Float.POSITIVE_INFINITY}) + * @throws IllegalArgumentException if {@code min} or {@code max} is invalid + */ + public void setRangeValues(float[] min, float[] max) { + checkArgs(min, max); + if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) { + throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2 + + " dimensions; cannot change to (incoming) " + min.length + " dimensions"); + } + + final byte[] bytes; + if (fieldsData == null) { + bytes = new byte[BYTES*2*min.length]; + fieldsData = new BytesRef(bytes); + } else { + bytes = ((BytesRef)fieldsData).bytes; + } + verifyAndEncode(min, max, bytes); + } + + /** validate the arguments */ + private static void checkArgs(final float[] min, final float[] max) { + if (min == null || max == null || min.length == 0 || max.length == 0) { + throw new IllegalArgumentException("min/max range values cannot be null or empty"); + } + if (min.length != max.length) { + throw new IllegalArgumentException("min/max ranges must agree"); + } + if (min.length > 4) { + throw new IllegalArgumentException("FloatRangeField does not support greater than 4 dimensions"); + } + } + + /** + * Encodes the min, max ranges into a byte array + */ + private static byte[] encode(float[] min, float[] max) { + checkArgs(min, max); + byte[] b = new byte[BYTES*2*min.length]; + verifyAndEncode(min, max, b); + return b; + } + + /** + * encode the ranges into a sortable byte array ({@code Float.NaN} not allowed) + *

    + * example for 4 dimensions (8 bytes per dimension value): + * minD1 ... minD4 | maxD1 ... maxD4 + */ + static void verifyAndEncode(float[] min, float[] max, byte[] bytes) { + for (int d=0,i=0,j=min.length*BYTES; d max[d]) { + throw new IllegalArgumentException("min value (" + min[d] + ") is greater than max value (" + max[d] + ")"); + } + encode(min[d], bytes, i); + encode(max[d], bytes, j); + } + } + + /** encode the given value into the byte array at the defined offset */ + private static void encode(float val, byte[] bytes, int offset) { + NumericUtils.intToSortableBytes(NumericUtils.floatToSortableInt(val), bytes, offset); + } + + /** + * Get the min value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded min value + */ + public float getMin(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMin(((BytesRef)fieldsData).bytes, dimension); + } + + /** + * Get the max value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded max value + */ + public float getMax(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMax(((BytesRef)fieldsData).bytes, dimension); + } + + /** decodes the min value (for the defined dimension) from the encoded input byte array */ + static float decodeMin(byte[] b, int dimension) { + int offset = dimension*BYTES; + return NumericUtils.sortableIntToFloat(NumericUtils.sortableBytesToInt(b, offset)); + } + + /** decodes the max value (for the defined dimension) from the encoded input byte array */ + static float decodeMax(byte[] b, int dimension) { + int offset = b.length/2 + dimension*BYTES; + return NumericUtils.sortableIntToFloat(NumericUtils.sortableBytesToInt(b, offset)); + } + + /** + * Create a query for matching indexed ranges that intersect the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Float.NEGATIVE_INFINITY}) + * @param max array of max values. (accepts {@code Float.MAX_VALUE}) + * @return query for matching intersecting ranges (overlap, within, or contains) + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newIntersectsQuery(String field, final float[] min, final float[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.INTERSECTS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return FloatRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed float ranges that contain the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Float.NEGATIVE_INFINITY}) + * @param max array of max values. (accepts {@code Float.POSITIVE_INFINITY}) + * @return query for matching ranges that contain the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newContainsQuery(String field, final float[] min, final float[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.CONTAINS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return FloatRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed ranges that are within the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Float.NEGATIVE_INFINITY}) + * @param max array of max values. (accepts {@code Float.POSITIVE_INFINITY}) + * @return query for matching ranges within the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newWithinQuery(String field, final float[] min, final float[] max) { + checkArgs(min, max); + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.WITHIN) { + @Override + protected String toString(byte[] ranges, int dimension) { + return FloatRangeField.toString(ranges, dimension); + } + }; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append(" <"); + sb.append(name); + sb.append(':'); + byte[] b = ((BytesRef)fieldsData).bytes; + toString(b, 0); + for (int d=1; d'); + + return sb.toString(); + } + + /** + * Returns the String representation for the range at the given dimension + * @param ranges the encoded ranges, never null + * @param dimension the dimension of interest + * @return The string representation for the range at the provided dimension + */ + private static String toString(byte[] ranges, int dimension) { + return "[" + Float.toString(decodeMin(ranges, dimension)) + " : " + + Float.toString(decodeMax(ranges, dimension)) + "]"; + } +} diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/IntRangeField.java b/lucene/sandbox/src/java/org/apache/lucene/document/IntRangeField.java new file mode 100644 index 00000000000..c0ce61d85e3 --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/document/IntRangeField.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.document.RangeFieldQuery.QueryType; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; + +/** + * An indexed Integer Range field. + *

    + * This field indexes dimensional ranges defined as min/max pairs. It supports + * up to a maximum of 4 dimensions (indexed as 8 numeric values). With 1 dimension representing a single integer range, + * 2 dimensions representing a bounding box, 3 dimensions a bounding cube, and 4 dimensions a tesseract. + *

    + * Multiple values for the same field in one document is supported, and open ended ranges can be defined using + * {@code Integer.MIN_VALUE} and {@code Integer.MAX_VALUE}. + * + *

    + * This field defines the following static factory methods for common search operations over integer ranges: + *

      + *
    • {@link #newIntersectsQuery newIntersectsQuery()} matches ranges that intersect the defined search range. + *
    • {@link #newWithinQuery newWithinQuery()} matches ranges that are within the defined search range. + *
    • {@link #newContainsQuery newContainsQuery()} matches ranges that contain the defined search range. + *
    + */ +public class IntRangeField extends Field { + /** stores integer values so number of bytes is 4 */ + public static final int BYTES = Integer.BYTES; + + /** + * Create a new IntRangeField type, from min/max parallel arrays + * + * @param name field name. must not be null. + * @param min range min values; each entry is the min value for the dimension + * @param max range max values; each entry is the max value for the dimension + */ + public IntRangeField(String name, final int[] min, final int[] max) { + super(name, getType(min.length)); + setRangeValues(min, max); + } + + /** set the field type */ + private static FieldType getType(int dimensions) { + if (dimensions > 4) { + throw new IllegalArgumentException("IntRangeField does not support greater than 4 dimensions"); + } + + FieldType ft = new FieldType(); + // dimensions is set as 2*dimension size (min/max per dimension) + ft.setDimensions(dimensions*2, BYTES); + ft.freeze(); + return ft; + } + + /** + * Changes the values of the field. + * @param min array of min values. (accepts {@code Integer.NEGATIVE_INFINITY}) + * @param max array of max values. (accepts {@code Integer.POSITIVE_INFINITY}) + * @throws IllegalArgumentException if {@code min} or {@code max} is invalid + */ + public void setRangeValues(int[] min, int[] max) { + checkArgs(min, max); + if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) { + throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2 + + " dimensions; cannot change to (incoming) " + min.length + " dimensions"); + } + + final byte[] bytes; + if (fieldsData == null) { + bytes = new byte[BYTES*2*min.length]; + fieldsData = new BytesRef(bytes); + } else { + bytes = ((BytesRef)fieldsData).bytes; + } + verifyAndEncode(min, max, bytes); + } + + /** validate the arguments */ + private static void checkArgs(final int[] min, final int[] max) { + if (min == null || max == null || min.length == 0 || max.length == 0) { + throw new IllegalArgumentException("min/max range values cannot be null or empty"); + } + if (min.length != max.length) { + throw new IllegalArgumentException("min/max ranges must agree"); + } + if (min.length > 4) { + throw new IllegalArgumentException("IntRangeField does not support greater than 4 dimensions"); + } + } + + /** + * Encodes the min, max ranges into a byte array + */ + private static byte[] encode(int[] min, int[] max) { + checkArgs(min, max); + byte[] b = new byte[BYTES*2*min.length]; + verifyAndEncode(min, max, b); + return b; + } + + /** + * encode the ranges into a sortable byte array ({@code Double.NaN} not allowed) + *

    + * example for 4 dimensions (8 bytes per dimension value): + * minD1 ... minD4 | maxD1 ... maxD4 + */ + static void verifyAndEncode(int[] min, int[] max, byte[] bytes) { + for (int d=0,i=0,j=min.length*BYTES; d max[d]) { + throw new IllegalArgumentException("min value (" + min[d] + ") is greater than max value (" + max[d] + ")"); + } + encode(min[d], bytes, i); + encode(max[d], bytes, j); + } + } + + /** encode the given value into the byte array at the defined offset */ + private static void encode(int val, byte[] bytes, int offset) { + NumericUtils.intToSortableBytes(val, bytes, offset); + } + + /** + * Get the min value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded min value + */ + public int getMin(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMin(((BytesRef)fieldsData).bytes, dimension); + } + + /** + * Get the max value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded max value + */ + public int getMax(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMax(((BytesRef)fieldsData).bytes, dimension); + } + + /** decodes the min value (for the defined dimension) from the encoded input byte array */ + static int decodeMin(byte[] b, int dimension) { + int offset = dimension*BYTES; + return NumericUtils.sortableBytesToInt(b, offset); + } + + /** decodes the max value (for the defined dimension) from the encoded input byte array */ + static int decodeMax(byte[] b, int dimension) { + int offset = b.length/2 + dimension*BYTES; + return NumericUtils.sortableBytesToInt(b, offset); + } + + /** + * Create a query for matching indexed ranges that intersect the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Integer.MIN_VALUE}) + * @param max array of max values. (accepts {@code Integer.MAX_VALUE}) + * @return query for matching intersecting ranges (overlap, within, or contains) + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newIntersectsQuery(String field, final int[] min, final int[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.INTERSECTS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return IntRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed ranges that contain the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Integer.MIN_VALUE}) + * @param max array of max values. (accepts {@code Integer.MAX_VALUE}) + * @return query for matching ranges that contain the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newContainsQuery(String field, final int[] min, final int[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.CONTAINS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return IntRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed ranges that are within the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Integer.MIN_VALUE}) + * @param max array of max values. (accepts {@code Integer.MAX_VALUE}) + * @return query for matching ranges within the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newWithinQuery(String field, final int[] min, final int[] max) { + checkArgs(min, max); + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.WITHIN) { + @Override + protected String toString(byte[] ranges, int dimension) { + return IntRangeField.toString(ranges, dimension); + } + }; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append(" <"); + sb.append(name); + sb.append(':'); + byte[] b = ((BytesRef)fieldsData).bytes; + toString(b, 0); + for (int d=1; d'); + + return sb.toString(); + } + + /** + * Returns the String representation for the range at the given dimension + * @param ranges the encoded ranges, never null + * @param dimension the dimension of interest + * @return The string representation for the range at the provided dimension + */ + private static String toString(byte[] ranges, int dimension) { + return "[" + Integer.toString(decodeMin(ranges, dimension)) + " : " + + Integer.toString(decodeMax(ranges, dimension)) + "]"; + } +} diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LongRangeField.java b/lucene/sandbox/src/java/org/apache/lucene/document/LongRangeField.java new file mode 100644 index 00000000000..b9298b9d8d3 --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LongRangeField.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.document.RangeFieldQuery.QueryType; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; + +/** + * An indexed Long Range field. + *

    + * This field indexes dimensional ranges defined as min/max pairs. It supports + * up to a maximum of 4 dimensions (indexed as 8 numeric values). With 1 dimension representing a single long range, + * 2 dimensions representing a bounding box, 3 dimensions a bounding cube, and 4 dimensions a tesseract. + *

    + * Multiple values for the same field in one document is supported, and open ended ranges can be defined using + * {@code Long.MIN_VALUE} and {@code Long.MAX_VALUE}. + * + *

    + * This field defines the following static factory methods for common search operations over long ranges: + *

      + *
    • {@link #newIntersectsQuery newIntersectsQuery()} matches ranges that intersect the defined search range. + *
    • {@link #newWithinQuery newWithinQuery()} matches ranges that are within the defined search range. + *
    • {@link #newContainsQuery newContainsQuery()} matches ranges that contain the defined search range. + *
    + */ +public class LongRangeField extends Field { + /** stores long values so number of bytes is 8 */ + public static final int BYTES = Long.BYTES; + + /** + * Create a new LongRangeField type, from min/max parallel arrays + * + * @param name field name. must not be null. + * @param min range min values; each entry is the min value for the dimension + * @param max range max values; each entry is the max value for the dimension + */ + public LongRangeField(String name, final long[] min, final long[] max) { + super(name, getType(min.length)); + setRangeValues(min, max); + } + + /** set the field type */ + private static FieldType getType(int dimensions) { + if (dimensions > 4) { + throw new IllegalArgumentException("LongRangeField does not support greater than 4 dimensions"); + } + + FieldType ft = new FieldType(); + // dimensions is set as 2*dimension size (min/max per dimension) + ft.setDimensions(dimensions*2, BYTES); + ft.freeze(); + return ft; + } + + /** + * Changes the values of the field. + * @param min array of min values. (accepts {@code Long.MIN_VALUE}) + * @param max array of max values. (accepts {@code Long.MAX_VALUE}) + * @throws IllegalArgumentException if {@code min} or {@code max} is invalid + */ + public void setRangeValues(long[] min, long[] max) { + checkArgs(min, max); + if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) { + throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2 + + " dimensions; cannot change to (incoming) " + min.length + " dimensions"); + } + + final byte[] bytes; + if (fieldsData == null) { + bytes = new byte[BYTES*2*min.length]; + fieldsData = new BytesRef(bytes); + } else { + bytes = ((BytesRef)fieldsData).bytes; + } + verifyAndEncode(min, max, bytes); + } + + /** validate the arguments */ + private static void checkArgs(final long[] min, final long[] max) { + if (min == null || max == null || min.length == 0 || max.length == 0) { + throw new IllegalArgumentException("min/max range values cannot be null or empty"); + } + if (min.length != max.length) { + throw new IllegalArgumentException("min/max ranges must agree"); + } + if (min.length > 4) { + throw new IllegalArgumentException("LongRangeField does not support greater than 4 dimensions"); + } + } + + /** Encodes the min, max ranges into a byte array */ + private static byte[] encode(long[] min, long[] max) { + checkArgs(min, max); + byte[] b = new byte[BYTES*2*min.length]; + verifyAndEncode(min, max, b); + return b; + } + + /** + * encode the ranges into a sortable byte array ({@code Double.NaN} not allowed) + *

    + * example for 4 dimensions (8 bytes per dimension value): + * minD1 ... minD4 | maxD1 ... maxD4 + */ + static void verifyAndEncode(long[] min, long[] max, byte[] bytes) { + for (int d=0,i=0,j=min.length*BYTES; d max[d]) { + throw new IllegalArgumentException("min value (" + min[d] + ") is greater than max value (" + max[d] + ")"); + } + encode(min[d], bytes, i); + encode(max[d], bytes, j); + } + } + + /** encode the given value into the byte array at the defined offset */ + private static void encode(long val, byte[] bytes, int offset) { + NumericUtils.longToSortableBytes(val, bytes, offset); + } + + /** + * Get the min value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded min value + */ + public long getMin(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMin(((BytesRef)fieldsData).bytes, dimension); + } + + /** + * Get the max value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded max value + */ + public long getMax(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMax(((BytesRef)fieldsData).bytes, dimension); + } + + /** decodes the min value (for the defined dimension) from the encoded input byte array */ + static long decodeMin(byte[] b, int dimension) { + int offset = dimension*BYTES; + return NumericUtils.sortableBytesToLong(b, offset); + } + + /** decodes the max value (for the defined dimension) from the encoded input byte array */ + static long decodeMax(byte[] b, int dimension) { + int offset = b.length/2 + dimension*BYTES; + return NumericUtils.sortableBytesToLong(b, offset); + } + + /** + * Create a query for matching indexed ranges that intersect the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Long.MIN_VALUE}) + * @param max array of max values. (accepts {@code Long.MAX_VALUE}) + * @return query for matching intersecting ranges (overlap, within, or contains) + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newIntersectsQuery(String field, final long[] min, final long[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.INTERSECTS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return LongRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed ranges that contain the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Long.MIN_VALUE}) + * @param max array of max values. (accepts {@code Long.MAX_VALUE}) + * @return query for matching ranges that contain the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newContainsQuery(String field, final long[] min, final long[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.CONTAINS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return LongRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed ranges that are within the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Long.MIN_VALUE}) + * @param max array of max values. (accepts {@code Long.MAX_VALUE}) + * @return query for matching ranges within the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newWithinQuery(String field, final long[] min, final long[] max) { + checkArgs(min, max); + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.WITHIN) { + @Override + protected String toString(byte[] ranges, int dimension) { + return LongRangeField.toString(ranges, dimension); + } + }; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append(" <"); + sb.append(name); + sb.append(':'); + byte[] b = ((BytesRef)fieldsData).bytes; + toString(b, 0); + for (int d=1; d'); + + return sb.toString(); + } + + /** + * Returns the String representation for the range at the given dimension + * @param ranges the encoded ranges, never null + * @param dimension the dimension of interest + * @return The string representation for the range at the provided dimension + */ + private static String toString(byte[] ranges, int dimension) { + return "[" + Long.toString(decodeMin(ranges, dimension)) + " : " + + Long.toString(decodeMax(ranges, dimension)) + "]"; + } +} diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java b/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java index d9cb830c120..9d293305c70 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java @@ -17,7 +17,6 @@ package org.apache.lucene.search; import java.io.IOException; -import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -41,16 +40,18 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; /** - * Abstract class to do basic tests for a RangeField query. + * Abstract class to do basic tests for a RangeField query. Testing rigor inspired by {@code BaseGeoPointTestCase} */ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase { - protected abstract Field newRangeField(double[] min, double[] max); + protected abstract Field newRangeField(Range box); - protected abstract Query newIntersectsQuery(double[] min, double[] max); + protected abstract Query newIntersectsQuery(Range box); - protected abstract Query newContainsQuery(double[] min, double[] max); + protected abstract Query newContainsQuery(Range box); - protected abstract Query newWithinQuery(double[] min, double[] max); + protected abstract Query newWithinQuery(Range box); + + protected abstract Range nextRange(int dimensions); protected int dimension() { return random().nextInt(4) + 1; @@ -82,18 +83,18 @@ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase { System.out.println("TEST: numDocs=" + numDocs); } - Box[][] boxes = new Box[numDocs][]; + Range[][] ranges = new Range[numDocs][]; boolean haveRealDoc = true; nextdoc: for (int id=0; id 0 && x < 9 && haveRealDoc) { int oldID; int i=0; - // don't step on missing boxes: + // don't step on missing ranges: while (true) { oldID = random().nextInt(id); - if (Double.isNaN(boxes[oldID][0].min[0]) == false) { + if (ranges[oldID][0].isMissing == false) { break; } else if (++i > id) { continue nextdoc; @@ -125,11 +126,11 @@ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase { if (x == dimensions*2) { // Fully identical box (use first box in case current is multivalued but old is not) for (int d=0; d 50000) { + if (ranges.length > 50000) { dir = newFSDirectory(createTempDir(getClass().getSimpleName())); } else { dir = newDirectory(); @@ -173,13 +174,13 @@ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase { Set deleted = new HashSet<>(); IndexWriter w = new IndexWriter(dir, iwc); - for (int id=0; id < boxes.length; ++id) { + for (int id=0; id < ranges.length; ++id) { Document doc = new Document(); doc.add(newStringField("id", ""+id, Field.Store.NO)); doc.add(new NumericDocValuesField("id", id)); - if (Double.isNaN(boxes[id][0].min[0]) == false) { - for (int n=0; n 1) ? "es=" : "=" ) + boxes[id][0]); - for (int n=1; n 1) ? "es=" : "=" ) + ranges[id][0]); + for (int n=1; n other.max[d] || this.max[d] < other.min[d]) { - // disjoint: - return null; - } - } - - // check within - boolean within = true; - for (int d=0; d= other.min[d] && this.max[d] <= other.max[d]) == false) { - // not within: - within = false; - break; - } - } - if (within == true) { + protected QueryType relate(Range other) { + if (isDisjoint(other)) { + // if disjoint; return null: + return null; + } else if (isWithin(other)) { return QueryType.WITHIN; - } - - // check contains - boolean contains = true; - for (int d=0; d= other.max[d]) == false) { - // not contains: - contains = false; - break; - } - } - if (contains == true) { + } else if (contains(other)) { return QueryType.CONTAINS; } return QueryType.INTERSECTS; } - - @Override - public String toString() { - StringBuilder b = new StringBuilder(); - b.append("Box("); - b.append(min[0]); - b.append(" TO "); - b.append(max[0]); - for (int d=1; d 0 && max.length > 0 + : "test box: min/max cannot be null or empty"; + assert min.length == max.length : "test box: min/max length do not agree"; + this.min = new double[min.length]; + this.max = new double[max.length]; + for (int d=0; d max[d]) { + // swap if max < min: + double temp = min[d]; + min[d] = max[d]; + max[d] = temp; + } + } + } + + @Override + protected int numDimensions() { + return min.length; + } + + @Override + protected Double getMin(int dim) { + return min[dim]; + } + + @Override + protected void setMin(int dim, Object val) { + min[dim] = (Double)val; + } + + @Override + protected Double getMax(int dim) { + return max[dim]; + } + + @Override + protected void setMax(int dim, Object val) { + max[dim] = (Double)val; + } + + @Override + protected boolean isEqual(Range other) { + DoubleRange o = (DoubleRange)other; + return Arrays.equals(min, o.min) && Arrays.equals(max, o.max); + } + + @Override + protected boolean isDisjoint(Range o) { + DoubleRange other = (DoubleRange)o; + for (int d=0; d other.max[d] || this.max[d] < other.min[d]) { + // disjoint: + return true; + } + } + return false; + } + + @Override + protected boolean isWithin(Range o) { + DoubleRange other = (DoubleRange)o; + for (int d=0; d= other.min[d] && this.max[d] <= other.max[d]) == false) { + // not within: + return false; + } + } + return true; + } + + @Override + protected boolean contains(Range o) { + DoubleRange other = (DoubleRange) o; + for (int d=0; d= other.max[d]) == false) { + // not contains: + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("Box("); + b.append(min[0]); + b.append(" TO "); + b.append(max[0]); + for (int d=1; d 0 && max.length > 0 + : "test box: min/max cannot be null or empty"; + assert min.length == max.length : "test box: min/max length do not agree"; + this.min = new float[min.length]; + this.max = new float[max.length]; + for (int d=0; d max[d]) { + // swap if max < min: + float temp = min[d]; + min[d] = max[d]; + max[d] = temp; + } + } + } + + @Override + protected int numDimensions() { + return min.length; + } + + @Override + protected Float getMin(int dim) { + return min[dim]; + } + + @Override + protected void setMin(int dim, Object val) { + min[dim] = (Float)val; + } + + @Override + protected Float getMax(int dim) { + return max[dim]; + } + + @Override + protected void setMax(int dim, Object val) { + max[dim] = (Float)val; + } + + @Override + protected boolean isEqual(Range other) { + FloatRange o = (FloatRange)other; + return Arrays.equals(min, o.min) && Arrays.equals(max, o.max); + } + + @Override + protected boolean isDisjoint(Range o) { + FloatRange other = (FloatRange)o; + for (int d=0; d other.max[d] || this.max[d] < other.min[d]) { + // disjoint: + return true; + } + } + return false; + } + + @Override + protected boolean isWithin(Range o) { + FloatRange other = (FloatRange)o; + for (int d=0; d= other.min[d] && this.max[d] <= other.max[d]) == false) { + // not within: + return false; + } + } + return true; + } + + @Override + protected boolean contains(Range o) { + FloatRange other = (FloatRange) o; + for (int d=0; d= other.max[d]) == false) { + // not contains: + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("Box("); + b.append(min[0]); + b.append(" TO "); + b.append(max[0]); + for (int d=1; d 0 && max.length > 0 + : "test box: min/max cannot be null or empty"; + assert min.length == max.length : "test box: min/max length do not agree"; + this.min = new int[min.length]; + this.max = new int[max.length]; + for (int d=0; d max[d]) { + // swap if max < min: + int temp = min[d]; + min[d] = max[d]; + max[d] = temp; + } + } + } + + @Override + protected int numDimensions() { + return min.length; + } + + @Override + protected Integer getMin(int dim) { + return min[dim]; + } + + @Override + protected void setMin(int dim, Object val) { + min[dim] = (Integer)val; + } + + @Override + protected Integer getMax(int dim) { + return max[dim]; + } + + @Override + protected void setMax(int dim, Object val) { + max[dim] = (Integer)val; + } + + @Override + protected boolean isEqual(Range other) { + IntRange o = (IntRange)other; + return Arrays.equals(min, o.min) && Arrays.equals(max, o.max); + } + + @Override + protected boolean isDisjoint(Range o) { + IntRange other = (IntRange)o; + for (int d=0; d other.max[d] || this.max[d] < other.min[d]) { + // disjoint: + return true; + } + } + return false; + } + + @Override + protected boolean isWithin(Range o) { + IntRange other = (IntRange)o; + for (int d=0; d= other.min[d] && this.max[d] <= other.max[d]) == false) { + // not within: + return false; + } + } + return true; + } + + @Override + protected boolean contains(Range o) { + IntRange other = (IntRange) o; + for (int d=0; d= other.max[d]) == false) { + // not contains: + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("Box("); + b.append(min[0]); + b.append(" TO "); + b.append(max[0]); + for (int d=1; d 0 && max.length > 0 + : "test box: min/max cannot be null or empty"; + assert min.length == max.length : "test box: min/max length do not agree"; + this.min = new long[min.length]; + this.max = new long[max.length]; + for (int d=0; d max[d]) { + // swap if max < min: + long temp = min[d]; + min[d] = max[d]; + max[d] = temp; + } + } + } + + @Override + protected int numDimensions() { + return min.length; + } + + @Override + protected Long getMin(int dim) { + return min[dim]; + } + + @Override + protected void setMin(int dim, Object val) { + min[dim] = (Long)val; + } + + @Override + protected Long getMax(int dim) { + return max[dim]; + } + + @Override + protected void setMax(int dim, Object val) { + max[dim] = (Long)val; + } + + @Override + protected boolean isEqual(Range other) { + LongRange o = (LongRange)other; + return Arrays.equals(min, o.min) && Arrays.equals(max, o.max); + } + + @Override + protected boolean isDisjoint(Range o) { + LongRange other = (LongRange)o; + for (int d=0; d other.max[d] || this.max[d] < other.min[d]) { + // disjoint: + return true; + } + } + return false; + } + + @Override + protected boolean isWithin(Range o) { + LongRange other = (LongRange)o; + for (int d=0; d= other.min[d] && this.max[d] <= other.max[d]) == false) { + // not within: + return false; + } + } + return true; + } + + @Override + protected boolean contains(Range o) { + LongRange other = (LongRange) o; + for (int d=0; d= other.max[d]) == false) { + // not contains: + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("Box("); + b.append(min[0]); + b.append(" TO "); + b.append(max[0]); + for (int d=1; d + @@ -42,16 +43,17 @@ - + - + - \ No newline at end of file + diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java index 63a113839b5..90e36d835db 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java @@ -20,17 +20,20 @@ import org.apache.lucene.document.DoubleDocValuesField; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.spatial.SpatialStrategy; @@ -39,7 +42,6 @@ import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.query.UnsupportedSpatialOperation; import org.apache.lucene.spatial.util.DistanceToShapeValueSource; import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; import org.locationtech.spatial4j.context.SpatialContext; import org.locationtech.spatial4j.shape.Point; @@ -87,7 +89,7 @@ public class BBoxStrategy extends SpatialStrategy { public static FieldType DEFAULT_FIELDTYPE; @Deprecated - public static FieldType LEGACY_FIELDTYPE; + public static LegacyFieldType LEGACY_FIELDTYPE; static { // Default: pointValues + docValues FieldType type = new FieldType(); @@ -97,14 +99,14 @@ public class BBoxStrategy extends SpatialStrategy { type.freeze(); DEFAULT_FIELDTYPE = type; // Legacy default: legacyNumerics + docValues - type = new FieldType(); - type.setIndexOptions(IndexOptions.DOCS); - type.setNumericType(FieldType.LegacyNumericType.DOUBLE); - type.setNumericPrecisionStep(8);// same as solr default - type.setDocValuesType(DocValuesType.NUMERIC);//docValues - type.setStored(false); - type.freeze(); - LEGACY_FIELDTYPE = type; + LegacyFieldType legacyType = new LegacyFieldType(); + legacyType.setIndexOptions(IndexOptions.DOCS); + legacyType.setNumericType(LegacyNumericType.DOUBLE); + legacyType.setNumericPrecisionStep(8);// same as solr default + legacyType.setDocValuesType(DocValuesType.NUMERIC);//docValues + legacyType.setStored(false); + legacyType.freeze(); + LEGACY_FIELDTYPE = legacyType; } public static final String SUFFIX_MINX = "__minX"; @@ -130,7 +132,7 @@ public class BBoxStrategy extends SpatialStrategy { private final boolean hasDocVals; private final boolean hasPointVals; // equiv to "hasLegacyNumerics": - private final FieldType legacyNumericFieldType; // not stored; holds precision step. + private final LegacyFieldType legacyNumericFieldType; // not stored; holds precision step. private final FieldType xdlFieldType; /** @@ -177,16 +179,17 @@ public class BBoxStrategy extends SpatialStrategy { if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) { numQuads++; } - if (fieldType.indexOptions() != IndexOptions.NONE && fieldType.numericType() != null) { + if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) { if (hasPointVals) { throw new IllegalArgumentException("pointValues and LegacyNumericType are mutually exclusive"); } - if (fieldType.numericType() != FieldType.LegacyNumericType.DOUBLE) { - throw new IllegalArgumentException(getClass() + " does not support " + fieldType.numericType()); + final LegacyFieldType legacyType = (LegacyFieldType) fieldType; + if (legacyType.numericType() != LegacyNumericType.DOUBLE) { + throw new IllegalArgumentException(getClass() + " does not support " + legacyType.numericType()); } numQuads++; - legacyNumericFieldType = new FieldType(LegacyDoubleField.TYPE_NOT_STORED); - legacyNumericFieldType.setNumericPrecisionStep(fieldType.numericPrecisionStep()); + legacyNumericFieldType = new LegacyFieldType(LegacyDoubleField.TYPE_NOT_STORED); + legacyNumericFieldType.setNumericPrecisionStep(legacyType.numericPrecisionStep()); legacyNumericFieldType.freeze(); } else { legacyNumericFieldType = null; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java index e724ab05fe6..757e2bd38f7 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java @@ -26,7 +26,7 @@ import org.apache.lucene.util.BytesRefIterator; /** * A TokenStream used internally by {@link org.apache.lucene.spatial.prefix.PrefixTreeStrategy}. * - * This is modelled after {@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * This is modelled after {@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * * @lucene.internal */ diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java index 197547c1d56..59aff490916 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java @@ -20,16 +20,18 @@ import org.apache.lucene.document.DoubleDocValuesField; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queries.function.FunctionRangeQuery; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.SpatialStrategy; import org.apache.lucene.spatial.query.SpatialArgs; @@ -85,7 +87,7 @@ public class PointVectorStrategy extends SpatialStrategy { public static FieldType DEFAULT_FIELDTYPE; @Deprecated - public static FieldType LEGACY_FIELDTYPE; + public static LegacyFieldType LEGACY_FIELDTYPE; static { // Default: pointValues + docValues FieldType type = new FieldType(); @@ -95,14 +97,14 @@ public class PointVectorStrategy extends SpatialStrategy { type.freeze(); DEFAULT_FIELDTYPE = type; // Legacy default: legacyNumerics - type = new FieldType(); - type.setIndexOptions(IndexOptions.DOCS); - type.setNumericType(FieldType.LegacyNumericType.DOUBLE); - type.setNumericPrecisionStep(8);// same as solr default - type.setDocValuesType(DocValuesType.NONE);//no docValues! - type.setStored(false); - type.freeze(); - LEGACY_FIELDTYPE = type; + LegacyFieldType legacyType = new LegacyFieldType(); + legacyType.setIndexOptions(IndexOptions.DOCS); + legacyType.setNumericType(LegacyNumericType.DOUBLE); + legacyType.setNumericPrecisionStep(8);// same as solr default + legacyType.setDocValuesType(DocValuesType.NONE);//no docValues! + legacyType.setStored(false); + legacyType.freeze(); + LEGACY_FIELDTYPE = legacyType; } public static final String SUFFIX_X = "__x"; @@ -116,7 +118,7 @@ public class PointVectorStrategy extends SpatialStrategy { private final boolean hasDocVals; private final boolean hasPointVals; // equiv to "hasLegacyNumerics": - private final FieldType legacyNumericFieldType; // not stored; holds precision step. + private final LegacyFieldType legacyNumericFieldType; // not stored; holds precision step. /** * Create a new {@link PointVectorStrategy} instance that uses {@link DoublePoint} and {@link DoublePoint#newRangeQuery} @@ -157,16 +159,17 @@ public class PointVectorStrategy extends SpatialStrategy { if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) { numPairs++; } - if (fieldType.indexOptions() != IndexOptions.NONE && fieldType.numericType() != null) { + if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) { if (hasPointVals) { throw new IllegalArgumentException("pointValues and LegacyNumericType are mutually exclusive"); } - if (fieldType.numericType() != FieldType.LegacyNumericType.DOUBLE) { - throw new IllegalArgumentException(getClass() + " does not support " + fieldType.numericType()); + final LegacyFieldType legacyType = (LegacyFieldType) fieldType; + if (legacyType.numericType() != LegacyNumericType.DOUBLE) { + throw new IllegalArgumentException(getClass() + " does not support " + legacyType.numericType()); } numPairs++; - legacyNumericFieldType = new FieldType(LegacyDoubleField.TYPE_NOT_STORED); - legacyNumericFieldType.setNumericPrecisionStep(fieldType.numericPrecisionStep()); + legacyNumericFieldType = new LegacyFieldType(LegacyDoubleField.TYPE_NOT_STORED); + legacyNumericFieldType.setNumericPrecisionStep(legacyType.numericPrecisionStep()); legacyNumericFieldType.freeze(); } else { legacyNumericFieldType = null; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java index 01e925926d2..20df7305cbe 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java @@ -22,6 +22,7 @@ import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.legacy.LegacyFieldType; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.SpatialMatchConcern; import org.apache.lucene.spatial.prefix.RandomSpatialOpStrategyTestCase; @@ -100,7 +101,12 @@ public class TestBBoxStrategy extends RandomSpatialOpStrategyTestCase { } //test we can disable docValues for predicate tests if (random().nextBoolean()) { - FieldType fieldType = new FieldType(((BBoxStrategy)strategy).getFieldType()); + FieldType fieldType = ((BBoxStrategy)strategy).getFieldType(); + if (fieldType instanceof LegacyFieldType) { + fieldType = new LegacyFieldType((LegacyFieldType)fieldType); + } else { + fieldType = new FieldType(fieldType); + } fieldType.setDocValuesType(DocValuesType.NONE); strategy = new BBoxStrategy(ctx, strategy.getFieldName(), fieldType); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java index b63216085b3..19fcb3bfffb 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java @@ -177,11 +177,14 @@ public final class TestUtil { assert hasNext; T v = iterator.next(); assert allowNull || v != null; - try { - iterator.remove(); - throw new AssertionError("broken iterator (supports remove): " + iterator); - } catch (UnsupportedOperationException expected) { - // ok + // for the first element, check that remove is not supported + if (i == 0) { + try { + iterator.remove(); + throw new AssertionError("broken iterator (supports remove): " + iterator); + } catch (UnsupportedOperationException expected) { + // ok + } } } assert !iterator.hasNext(); diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy index f1d8f106dc2..2a623b70cea 100644 --- a/lucene/tools/junit4/tests.policy +++ b/lucene/tools/junit4/tests.policy @@ -28,10 +28,6 @@ grant { // should be enclosed within common.dir, but just in case: permission java.io.FilePermission "${junit4.childvm.cwd}", "read"; - // jenkins wants to read outside its sandbox, to use a special linedocs file. - // this is best effort and not really supported. - permission java.io.FilePermission "/home/jenkins/lucene-data/enwiki.random.lines.txt", "read"; - // write only to sandbox permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp", "read,write,delete"; permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp${/}-", "read,write,delete"; diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 518f63a10d9..7458f4681d7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -203,6 +203,21 @@ Bug Fixes * SOLR-9397: Config API does not support adding caches (noble) +* SOLR-9405: ConcurrentModificationException in ZkStateReader.getStateWatchers. + (Alan Woodward, Edward Ribeiro, shalin) + +* SOLR-9232: Admin UI now fully implements Swap Cores interface (Alexandre Rafalovitch) + +* SOLR-8715: Admin UI's Schema screen now works for fields with stored=false and some content indexed (Alexandre Rafalovitch) + +* SOLR-8911: In Admin UI, enable scrolling for overflowing Versions and JVM property values (Alexandre Rafalovitch) + +* SOLR-9002: Admin UI now correctly displays json and text files in the collection/Files screen (Upayavira, Alexandre Rafalovitch) + +* SOLR-8993: Admin UI now correctly supports multiple DIH handler end-points (Upayavira, Alexandre Rafalovitch) + +* SOLR-9032: Admin UI now correctly implements Create Alias command (Upayavira, Alexandre Rafalovitch) + Optimizations ---------------------- @@ -260,6 +275,13 @@ Other Changes * SOLR-9331: Remove ReRankQuery's length constructor argument and member. (Christine Poerschke) +* SOLR-9092: For the delete replica command we attempt to send the core admin delete request only + if that node is actually up. (Jessica Cheng Mallet, Varun Thacker) + +* SOLR-9410: Make ReRankQParserPlugin's private ReRankWeight a public class of its own. (Christine Poerschke) + +* SOLR-9404: Refactor move/renames in JSON FacetProcessor and FacetFieldProcessor. (David Smiley) + ================== 6.1.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java index 7a7e697d059..aadb9e2d4ce 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java @@ -20,8 +20,8 @@ import java.io.IOException; import java.time.Instant; import java.util.Arrays; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.TrieDateField; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java index 4d66e0025bc..22dde4c00ab 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java @@ -24,12 +24,12 @@ import java.util.Map; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.LongDocValues; import org.apache.lucene.queries.function.valuesource.LongFieldSource; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueDate; diff --git a/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java index cbcfa8847df..3e600908cd6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java @@ -20,6 +20,7 @@ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; import java.util.List; +import java.util.Locale; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -27,11 +28,13 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Utils; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; + public class DeleteNodeCmd implements OverseerCollectionMessageHandler.Cmd { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -42,43 +45,44 @@ public class DeleteNodeCmd implements OverseerCollectionMessageHandler.Cmd { } @Override - public Object call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { ocmh.checkRequired(message, "node"); String node = message.getStr("node"); if (!state.liveNodesContain(node)) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source Node: " + node + " is not live"); } List sourceReplicas = ReplaceNodeCmd.getReplicasOfNode(node, state); - cleanupReplicas(results, state, sourceReplicas, ocmh); - return null; + cleanupReplicas(results, state, sourceReplicas, ocmh, node); } static void cleanupReplicas(NamedList results, ClusterState clusterState, List sourceReplicas, - OverseerCollectionMessageHandler ocmh) throws InterruptedException { + OverseerCollectionMessageHandler ocmh, String node) throws InterruptedException { CountDownLatch cleanupLatch = new CountDownLatch(sourceReplicas.size()); for (ZkNodeProps sourceReplica : sourceReplicas) { - log.info("deleting replica from from node {} ", Utils.toJSONString(sourceReplica)); + log.info("Deleting replica for collection={} shard={} on node={}", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), node); NamedList deleteResult = new NamedList(); try { ocmh.deleteReplica(clusterState, sourceReplica.plus("parallel", "true"), deleteResult, () -> { cleanupLatch.countDown(); if (deleteResult.get("failure") != null) { synchronized (results) { - results.add("failure", "could not delete because " + deleteResult.get("failure") + " " + Utils.toJSONString(sourceReplica)); + results.add("failure", String.format(Locale.ROOT, "Failed to delete replica for collection=%s shard=%s" + + " on node=%s", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), node)); } } }); } catch (KeeperException e) { - log.info("Error deleting ", e); + log.warn("Error deleting ", e); cleanupLatch.countDown(); } catch (Exception e) { + log.warn("Error deleting ", e); cleanupLatch.countDown(); throw e; } } - log.info("Waiting for deletes to complete"); + log.debug("Waiting for delete node action to complete"); cleanupLatch.await(5, TimeUnit.MINUTES); } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java index 908d35c3e6b..49e094284b3 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java @@ -316,7 +316,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler default: { Cmd command = commandMap.get(action); if (command != null) { - command.call(zkStateReader.getClusterState(),message, results); + command.call(zkStateReader.getClusterState(), message, results); } else { throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:" + operation); @@ -617,7 +617,6 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler @SuppressWarnings("unchecked") void deleteReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete) throws KeeperException, InterruptedException { - log.info("deleteReplica() : {}", Utils.toJSONString(message)); checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP); String collectionName = message.getStr(COLLECTION_PROP); String shard = message.getStr(SHARD_ID_PROP); @@ -663,15 +662,19 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler params.set(CoreAdminParams.DELETE_INSTANCE_DIR, message.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, true)); params.set(CoreAdminParams.DELETE_DATA_DIR, message.getBool(CoreAdminParams.DELETE_DATA_DIR, true)); - sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap.get()); - AtomicReference exp = new AtomicReference<>(); + boolean isLive = zkStateReader.getClusterState().getLiveNodes().contains(replica.getNodeName()); + if (isLive) { + sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap.get()); + } Callable callable = () -> { try { - processResponses(results, shardHandler, false, null, asyncId, requestMap.get()); + if (isLive) { + processResponses(results, shardHandler, false, null, asyncId, requestMap.get()); - //check if the core unload removed the corenode zk entry - if (waitForCoreNodeGone(collectionName, shard, replicaName, 5000)) return Boolean.TRUE; + //check if the core unload removed the corenode zk entry + if (waitForCoreNodeGone(collectionName, shard, replicaName, 5000)) return Boolean.TRUE; + } // try and ensure core info is removed from cluster state deleteCoreNode(collectionName, replicaName, replica, core); @@ -2809,7 +2812,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler interface Cmd { - Object call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception; + void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception; } diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java index 0cfd0894ffe..aad9cc721fb 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java @@ -21,6 +21,7 @@ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -35,7 +36,6 @@ import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Utils; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +53,7 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { } @Override - public Object call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { ZkStateReader zkStateReader = ocmh.zkStateReader; ocmh.checkRequired(message, "source", "target"); String source = message.getStr("source"); @@ -76,23 +76,25 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { for (ZkNodeProps sourceReplica : sourceReplicas) { NamedList nl = new NamedList(); - log.info("going to create replica {}", Utils.toJSONString(sourceReplica)); + log.info("Going to create replica for collection={} shard={} on node={}", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target); ZkNodeProps msg = sourceReplica.plus("parallel", String.valueOf(parallel)).plus(CoreAdminParams.NODE, target); final ZkNodeProps addedReplica = ocmh.addReplica(clusterState, msg, nl, () -> { countDownLatch.countDown(); if (nl.get("failure") != null) { - log.warn("failed to create : " + Utils.toJSONString(msg)); + String errorString = String.format(Locale.ROOT, "Failed to create replica for collection=%s shard=%s" + + " on node=%s", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target); + log.warn(errorString); // one replica creation failed. Make the best attempt to // delete all the replicas created so far in the target // and exit synchronized (results) { - results.add("failure", "Could not create copy of replica " + Utils.toJSONString(sourceReplica)); + results.add("failure", errorString); anyOneFailed.set(true); } } else { - log.info("successfully created : " + Utils.toJSONString(msg)); - + log.debug("Successfully created replica for collection={} shard={} on node={}", + sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target); } }); @@ -101,12 +103,12 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { } } - log.info("Waiting for creates to complete "); + log.debug("Waiting for replace node action to complete"); countDownLatch.await(5, TimeUnit.MINUTES); - log.info("Waiting over for creates to complete "); + log.debug("Finished waiting for replace node action to complete"); if (anyOneFailed.get()) { - log.info("failed to create some cores delete all " + Utils.toJSONString(createdReplicas)); + log.info("Failed to create some replicas. Cleaning up all replicas on target node"); CountDownLatch cleanupLatch = new CountDownLatch(createdReplicas.size()); for (ZkNodeProps createdReplica : createdReplicas) { NamedList deleteResult = new NamedList(); @@ -115,29 +117,27 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { cleanupLatch.countDown(); if (deleteResult.get("failure") != null) { synchronized (results) { - results.add("failure", "could not cleanup, because : " + deleteResult.get("failure") + " " + Utils.toJSONString(createdReplica)); + results.add("failure", "Could not cleanup, because of : " + deleteResult.get("failure")); } } }); } catch (KeeperException e) { cleanupLatch.countDown(); - log.info("Error deleting ", e); + log.warn("Error deleting replica ", e); } catch (Exception e) { - log.error("Unknown Error deleteing", e); + log.warn("Error deleting replica ", e); cleanupLatch.countDown(); throw e; } } cleanupLatch.await(5, TimeUnit.MINUTES); - return null; } // we have reached this far means all replicas could be recreated //now cleanup the replicas in the source node - DeleteNodeCmd.cleanupReplicas(results, state, sourceReplicas, ocmh); - results.add("success", "REPLACENODE completed successfully from : " + source + " to : " + target); - return null; + DeleteNodeCmd.cleanupReplicas(results, state, sourceReplicas, ocmh, source); + results.add("success", "REPLACENODE action completed successfully from : " + source + " to : " + target); } static List getReplicasOfNode(String source, ClusterState state) { @@ -152,7 +152,6 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { ZkStateReader.CORE_NAME_PROP, replica.getCoreName(), ZkStateReader.REPLICA_PROP, replica.getName(), CoreAdminParams.NODE, source); - log.info("src_core : {}", Utils.toJSONString(props)); sourceReplicas.add(props ); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java index 4c2a2b6a5ab..5df1b45cc93 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java @@ -29,8 +29,7 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang.StringUtils; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; @@ -637,13 +636,13 @@ public class StatsField { return null; } - final FieldType.LegacyNumericType hashableNumType = getHashableNumericType(field); + final LegacyNumericType hashableNumType = getHashableNumericType(field); // some sane defaults int log2m = 13; // roughly equivilent to "cardinality='0.33'" int regwidth = 6; // with decent hash, this is plenty for all valid long hashes - if (LegacyNumericType.FLOAT.equals(hashableNumType) || FieldType.LegacyNumericType.INT.equals(hashableNumType)) { + if (LegacyNumericType.FLOAT.equals(hashableNumType) || LegacyNumericType.INT.equals(hashableNumType)) { // for 32bit values, we can adjust our default regwidth down a bit regwidth--; @@ -707,7 +706,7 @@ public class StatsField { if (null == hasher) { // if this is a function, or a non Long field, pre-hashed is invalid // NOTE: we ignore hashableNumType - it's LONG for non numerics like Strings - if (null == field || !FieldType.LegacyNumericType.LONG.equals(field.getType().getNumericType())) { + if (null == field || !LegacyNumericType.LONG.equals(field.getType().getNumericType())) { throw new SolrException(ErrorCode.BAD_REQUEST, "hllPreHashed is only supported with Long based fields"); } } @@ -740,16 +739,16 @@ public class StatsField { } /** - * Returns the effective {@link org.apache.lucene.document.FieldType.LegacyNumericType} for the field for the purposes of hash values. + * Returns the effective {@link LegacyNumericType} for the field for the purposes of hash values. * ie: If the field has an explict LegacyNumericType that is returned; If the field has no explicit - * LegacyNumericType then {@link org.apache.lucene.document.FieldType.LegacyNumericType#LONG} is returned; If field is null, then - * {@link org.apache.lucene.document.FieldType.LegacyNumericType#FLOAT} is assumed for ValueSource. + * LegacyNumericType then {@link LegacyNumericType#LONG} is returned; If field is null, then + * {@link LegacyNumericType#FLOAT} is assumed for ValueSource. */ private static LegacyNumericType getHashableNumericType(SchemaField field) { if (null == field) { return LegacyNumericType.FLOAT; } final LegacyNumericType result = field.getType().getNumericType(); - return null == result ? FieldType.LegacyNumericType.LONG : result; + return null == result ? LegacyNumericType.LONG : result; } } diff --git a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java index fedc7fe0ca4..db26e12218d 100644 --- a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java +++ b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java @@ -25,7 +25,7 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.NumericDocValues; diff --git a/solr/core/src/java/org/apache/solr/request/NumericFacets.java b/solr/core/src/java/org/apache/solr/request/NumericFacets.java index 1034947ca9f..fd85d3fd5d7 100644 --- a/solr/core/src/java/org/apache/solr/request/NumericFacets.java +++ b/solr/core/src/java/org/apache/solr/request/NumericFacets.java @@ -33,6 +33,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.util.Bits; @@ -132,7 +133,7 @@ final class NumericFacets { mincount = Math.max(mincount, 1); final SchemaField sf = searcher.getSchema().getField(fieldName); final FieldType ft = sf.getType(); - final org.apache.lucene.document.FieldType.LegacyNumericType numericType = ft.getNumericType(); + final LegacyNumericType numericType = ft.getNumericType(); if (numericType == null) { throw new IllegalStateException(); } diff --git a/solr/core/src/java/org/apache/solr/schema/BBoxField.java b/solr/core/src/java/org/apache/solr/schema/BBoxField.java index e41d3c6e4e0..d7fda7ce0da 100644 --- a/solr/core/src/java/org/apache/solr/schema/BBoxField.java +++ b/solr/core/src/java/org/apache/solr/schema/BBoxField.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.legacy.LegacyFieldType; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.spatial.bbox.BBoxOverlapRatioValueSource; import org.apache.lucene.spatial.bbox.BBoxStrategy; @@ -141,7 +142,11 @@ public class BBoxField extends AbstractSpatialFieldType implements //and annoyingly this Field isn't going to have a docValues format because Solr uses a separate Field for that if (solrNumField.hasDocValues()) { - luceneType = new org.apache.lucene.document.FieldType(luceneType); + if (luceneType instanceof LegacyFieldType) { + luceneType = new LegacyFieldType((LegacyFieldType)luceneType); + } else { + luceneType = new org.apache.lucene.document.FieldType(luceneType); + } luceneType.setDocValuesType(DocValuesType.NUMERIC); } return new BBoxStrategy(ctx, fieldName, luceneType); diff --git a/solr/core/src/java/org/apache/solr/schema/EnumField.java b/solr/core/src/java/org/apache/solr/schema/EnumField.java index 27f3a0a1ea8..4820e77bb11 100644 --- a/solr/core/src/java/org/apache/solr/schema/EnumField.java +++ b/solr/core/src/java/org/apache/solr/schema/EnumField.java @@ -32,24 +32,25 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.EnumFieldSource; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DocValuesRangeQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.solr.common.EnumFieldValue; import org.apache.solr.common.SolrException; import org.apache.solr.response.TextResponseWriter; @@ -234,8 +235,8 @@ public class EnumField extends PrimitiveFieldType { * {@inheritDoc} */ @Override - public FieldType.LegacyNumericType getNumericType() { - return FieldType.LegacyNumericType.INT; + public LegacyNumericType getNumericType() { + return LegacyNumericType.INT; } /** @@ -387,7 +388,7 @@ public class EnumField extends PrimitiveFieldType { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown value for enum field: " + value.toString()); String intAsString = intValue.toString(); - final FieldType newType = new FieldType(); + final LegacyFieldType newType = new LegacyFieldType(); newType.setTokenized(field.isTokenized()); newType.setStored(field.stored()); @@ -397,7 +398,7 @@ public class EnumField extends PrimitiveFieldType { newType.setStoreTermVectorOffsets(field.storeTermOffsets()); newType.setStoreTermVectorPositions(field.storeTermPositions()); newType.setStoreTermVectorPayloads(field.storeTermPayloads()); - newType.setNumericType(FieldType.LegacyNumericType.INT); + newType.setNumericType(LegacyNumericType.INT); newType.setNumericPrecisionStep(DEFAULT_PRECISION_STEP); final org.apache.lucene.document.Field f; diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 6556ddb77f6..32a91d98316 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -38,6 +38,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.DocValuesRewriteMethod; @@ -621,7 +622,7 @@ public abstract class FieldType extends FieldProperties { /** Return the numeric type of this field, or null if this field is not a * numeric field. */ - public org.apache.lucene.document.FieldType.LegacyNumericType getNumericType() { + public LegacyNumericType getNumericType() { return null; } diff --git a/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java b/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java index 18d80a35146..f6bb782e515 100644 --- a/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.spatial.vector.PointVectorStrategy; /** @@ -78,8 +80,8 @@ public class SpatialPointVectorFieldType extends AbstractSpatialFieldType * For each number being added to this field, multiple terms are generated as per the algorithm described in the above @@ -82,7 +82,7 @@ import org.slf4j.LoggerFactory; * generated, range search will be no faster than any other number field, but sorting will still be possible. * * - * @see org.apache.lucene.search.LegacyNumericRangeQuery + * @see org.apache.lucene.legacy.LegacyNumericRangeQuery * @since solr 1.4 */ public class TrieField extends PrimitiveFieldType { @@ -349,17 +349,17 @@ public class TrieField extends PrimitiveFieldType { } @Override - public FieldType.LegacyNumericType getNumericType() { + public LegacyNumericType getNumericType() { switch (type) { case INTEGER: - return FieldType.LegacyNumericType.INT; + return LegacyNumericType.INT; case LONG: case DATE: - return FieldType.LegacyNumericType.LONG; + return LegacyNumericType.LONG; case FLOAT: - return FieldType.LegacyNumericType.FLOAT; + return LegacyNumericType.FLOAT; case DOUBLE: - return FieldType.LegacyNumericType.DOUBLE; + return LegacyNumericType.DOUBLE; default: throw new AssertionError(); } @@ -666,7 +666,7 @@ public class TrieField extends PrimitiveFieldType { return null; } - FieldType ft = new FieldType(); + LegacyFieldType ft = new LegacyFieldType(); ft.setStored(stored); ft.setTokenized(true); ft.setOmitNorms(field.omitNorms()); @@ -677,16 +677,16 @@ public class TrieField extends PrimitiveFieldType { ft.setNumericType(LegacyNumericType.INT); break; case FLOAT: - ft.setNumericType(FieldType.LegacyNumericType.FLOAT); + ft.setNumericType(LegacyNumericType.FLOAT); break; case LONG: - ft.setNumericType(FieldType.LegacyNumericType.LONG); + ft.setNumericType(LegacyNumericType.LONG); break; case DOUBLE: - ft.setNumericType(FieldType.LegacyNumericType.DOUBLE); + ft.setNumericType(LegacyNumericType.DOUBLE); break; case DATE: - ft.setNumericType(FieldType.LegacyNumericType.LONG); + ft.setNumericType(LegacyNumericType.LONG); break; default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type); diff --git a/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java b/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java index 8d1739e6782..bcc4d4fe3b1 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java @@ -23,13 +23,13 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.FloatDocValues; import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource; import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueFloat; diff --git a/solr/core/src/java/org/apache/solr/schema/TrieIntField.java b/solr/core/src/java/org/apache/solr/schema/TrieIntField.java index 07d9c74f472..6c8622b39e6 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieIntField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieIntField.java @@ -23,13 +23,13 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.IntDocValues; import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource; import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueInt; diff --git a/solr/core/src/java/org/apache/solr/schema/TrieLongField.java b/solr/core/src/java/org/apache/solr/schema/TrieLongField.java index 28345af9622..371d5bb0a14 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieLongField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieLongField.java @@ -23,13 +23,13 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.LongDocValues; import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource; import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueLong; diff --git a/solr/core/src/java/org/apache/solr/search/QueryParsing.java b/solr/core/src/java/org/apache/solr/search/QueryParsing.java index e7a6c3632fa..fb32c6e934d 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryParsing.java +++ b/solr/core/src/java/org/apache/solr/search/QueryParsing.java @@ -17,12 +17,12 @@ package org.apache.solr.search; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FuzzyQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; diff --git a/solr/core/src/java/org/apache/solr/search/QueryWrapperFilter.java b/solr/core/src/java/org/apache/solr/search/QueryWrapperFilter.java index 6bba1de2c64..d526cf394ab 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryWrapperFilter.java +++ b/solr/core/src/java/org/apache/solr/search/QueryWrapperFilter.java @@ -34,7 +34,7 @@ import org.apache.lucene.util.Bits; * Constrains search results to only match those which also match a provided * query. * - *

    This could be used, for example, with a {@link org.apache.lucene.search.LegacyNumericRangeQuery} on a suitably + *

    This could be used, for example, with a {@link org.apache.lucene.legacy.LegacyNumericRangeQuery} on a suitably * formatted date field to implement date filtering. One could re-use a single * CachingWrapperFilter(QueryWrapperFilter) that matches, e.g., only documents modified * within the last week. This would only need to be reconstructed once per day. diff --git a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java index a903968058a..2c462a0aae5 100644 --- a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java @@ -25,8 +25,6 @@ import com.carrotsearch.hppc.IntIntHashMap; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.FilterWeight; -import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; @@ -188,23 +186,8 @@ public class ReRankQParserPlugin extends QParserPlugin { } public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException{ - return new ReRankWeight(mainQuery, reRankQueryRescorer, searcher, needsScores, boost); - } - } - - private class ReRankWeight extends FilterWeight { - private IndexSearcher searcher; - final private Rescorer reRankQueryRescorer; - - public ReRankWeight(Query mainQuery, Rescorer reRankQueryRescorer, IndexSearcher searcher, boolean needsScores, float boost) throws IOException { - super(mainQuery, mainQuery.createWeight(searcher, needsScores, boost)); - this.searcher = searcher; - this.reRankQueryRescorer = reRankQueryRescorer; - } - - public Explanation explain(LeafReaderContext context, int doc) throws IOException { - Explanation mainExplain = in.explain(context, doc); - return reRankQueryRescorer.explain(searcher, mainExplain, context.docBase+doc); + final Weight mainWeight = mainQuery.createWeight(searcher, needsScores, boost); + return new ReRankWeight(mainQuery, reRankQueryRescorer, searcher, mainWeight); } } diff --git a/solr/core/src/java/org/apache/solr/search/ReRankWeight.java b/solr/core/src/java/org/apache/solr/search/ReRankWeight.java new file mode 100644 index 00000000000..9c11a894200 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/ReRankWeight.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import java.io.IOException; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FilterWeight; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Rescorer; +import org.apache.lucene.search.Weight; + +/** + * A {@code Weight} used by reranking queries. + */ +public class ReRankWeight extends FilterWeight { + + final private IndexSearcher searcher; + final private Rescorer reRankQueryRescorer; + + public ReRankWeight(Query mainQuery, Rescorer reRankQueryRescorer, IndexSearcher searcher, Weight mainWeight) throws IOException { + super(mainQuery, mainWeight); + this.searcher = searcher; + this.reRankQueryRescorer = reRankQueryRescorer; + } + + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + final Explanation mainExplain = in.explain(context, doc); + return reRankQueryRescorer.explain(searcher, mainExplain, context.docBase+doc); + } + +} diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java index a5ec1dbab91..92c64e74b16 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java @@ -16,39 +16,13 @@ */ package org.apache.solr.search.facet; -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiPostingsEnum; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.PriorityQueue; -import org.apache.lucene.util.StringHelper; -import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.SchemaField; -import org.apache.solr.schema.TrieField; -import org.apache.solr.search.DocSet; -import org.apache.solr.search.HashDocSet; -import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.search.SortedIntDocSet; public class FacetField extends FacetRequest { @@ -69,7 +43,7 @@ public class FacetField extends FacetRequest { Boolean perSeg; // TODO: put this somewhere more generic? - public static enum SortDirection { + public enum SortDirection { asc(-1) , desc(1); @@ -84,7 +58,7 @@ public class FacetField extends FacetRequest { } } - public static enum FacetMethod { + public enum FacetMethod { DV, // DocValues UIF, // UnInvertedField ENUM, @@ -109,7 +83,6 @@ public class FacetField extends FacetRequest { } } - @Override public FacetProcessor createFacetProcessor(FacetContext fcontext) { SchemaField sf = fcontext.searcher.getSchema().getField(field); @@ -119,18 +92,18 @@ public class FacetField extends FacetRequest { if (method == FacetMethod.ENUM && sf.indexed()) { throw new UnsupportedOperationException(); } else if (method == FacetMethod.STREAM && sf.indexed()) { - return new FacetFieldProcessorStream(fcontext, this, sf); + return new FacetFieldProcessorByEnumTermsStream(fcontext, this, sf); } - org.apache.lucene.document.FieldType.LegacyNumericType ntype = ft.getNumericType(); + LegacyNumericType ntype = ft.getNumericType(); if (!multiToken) { if (ntype != null) { // single valued numeric (docvalues or fieldcache) - return new FacetFieldProcessorNumeric(fcontext, this, sf); + return new FacetFieldProcessorByHashNumeric(fcontext, this, sf); } else { // single valued string... - return new FacetFieldProcessorDV(fcontext, this, sf); + return new FacetFieldProcessorByArrayDV(fcontext, this, sf); } } @@ -138,11 +111,11 @@ public class FacetField extends FacetRequest { if (sf.hasDocValues() || method == FacetMethod.DV) { // single and multi-valued string docValues - return new FacetFieldProcessorDV(fcontext, this, sf); + return new FacetFieldProcessorByArrayDV(fcontext, this, sf); } // Top-level multi-valued field cache (UIF) - return new FacetFieldProcessorUIF(fcontext, this, sf); + return new FacetFieldProcessorByArrayUIF(fcontext, this, sf); } @Override @@ -152,918 +125,12 @@ public class FacetField extends FacetRequest { @Override public Map getFacetDescription() { - Map descr = new HashMap(); + Map descr = new HashMap<>(); descr.put("field", field); - descr.put("limit", new Long(limit)); + descr.put("limit", limit); return descr; } } -abstract class FacetFieldProcessor extends FacetProcessor { - SchemaField sf; - SlotAcc indexOrderAcc; - int effectiveMincount; - - Map deferredAggs; // null if none - - // TODO: push any of this down to base class? - - // - // For sort="x desc", collectAcc would point to "x", and sortAcc would also point to "x". - // collectAcc would be used to accumulate all buckets, and sortAcc would be used to sort those buckets. - // - SlotAcc collectAcc; // Accumulator to collect across entire domain (in addition to the countAcc). May be null. - SlotAcc sortAcc; // Accumulator to use for sorting *only* (i.e. not used for collection). May be an alias of countAcc, collectAcc, or indexOrderAcc - SlotAcc[] otherAccs; // Accumulators that do not need to be calculated across all buckets. - - SpecialSlotAcc allBucketsAcc; // this can internally refer to otherAccs and/or collectAcc. setNextReader should be called on otherAccs directly if they exist. - - - FacetFieldProcessor(FacetContext fcontext, FacetField freq, SchemaField sf) { - super(fcontext, freq); - this.sf = sf; - this.effectiveMincount = (int)(fcontext.isShard() ? Math.min(1 , freq.mincount) : freq.mincount); - } - - @Override - public Object getResponse() { - return response; - } - - // This is used to create accs for second phase (or to create accs for all aggs) - @Override - protected void createAccs(int docCount, int slotCount) throws IOException { - if (accMap == null) { - accMap = new LinkedHashMap<>(); - } - - // allow a custom count acc to be used - if (countAcc == null) { - countAcc = new CountSlotArrAcc(fcontext, slotCount); - countAcc.key = "count"; - } - - if (accs != null) { - // reuse these accs, but reset them first - for (SlotAcc acc : accs) { - acc.reset(); - } - return; - } else { - accs = new SlotAcc[ freq.getFacetStats().size() ]; - } - - int accIdx = 0; - for (Map.Entry entry : freq.getFacetStats().entrySet()) { - SlotAcc acc = null; - if (slotCount == 1) { - acc = accMap.get(entry.getKey()); - if (acc != null) { - acc.reset(); - } - } - if (acc == null) { - acc = entry.getValue().createSlotAcc(fcontext, docCount, slotCount); - acc.key = entry.getKey(); - accMap.put(acc.key, acc); - } - accs[accIdx++] = acc; - } - } - - void createCollectAcc(int numDocs, int numSlots) throws IOException { - accMap = new LinkedHashMap<>(); - - // we always count... - // allow a subclass to set a custom counter. - if (countAcc == null) { - countAcc = new CountSlotArrAcc(fcontext, numSlots); - } - - if ("count".equals(freq.sortVariable)) { - sortAcc = countAcc; - deferredAggs = freq.getFacetStats(); - } else if ("index".equals(freq.sortVariable)) { - // allow subclass to set indexOrderAcc first - if (indexOrderAcc == null) { - // This sorting accumulator just goes by the slot number, so does not need to be collected - // and hence does not need to find it's way into the accMap or accs array. - indexOrderAcc = new SortSlotAcc(fcontext); - } - sortAcc = indexOrderAcc; - deferredAggs = freq.getFacetStats(); - } else { - AggValueSource sortAgg = freq.getFacetStats().get(freq.sortVariable); - if (sortAgg != null) { - collectAcc = sortAgg.createSlotAcc(fcontext, numDocs, numSlots); - collectAcc.key = freq.sortVariable; // TODO: improve this - } - sortAcc = collectAcc; - deferredAggs = new HashMap<>(freq.getFacetStats()); - deferredAggs.remove(freq.sortVariable); - } - - if (deferredAggs.size() == 0) { - deferredAggs = null; - } - - boolean needOtherAccs = freq.allBuckets; // TODO: use for missing too... - - if (!needOtherAccs) { - // we may need them later, but we don't want to create them now - // otherwise we won't know if we need to call setNextReader on them. - return; - } - - // create the deferred aggs up front for use by allBuckets - createOtherAccs(numDocs, 1); - } - - - void createOtherAccs(int numDocs, int numSlots) throws IOException { - if (otherAccs != null) { - // reuse existing accumulators - for (SlotAcc acc : otherAccs) { - acc.reset(); // todo - make reset take numDocs and numSlots? - } - return; - } - - int numDeferred = deferredAggs == null ? 0 : deferredAggs.size(); - if (numDeferred <= 0) return; - - otherAccs = new SlotAcc[ numDeferred ]; - - int otherAccIdx = 0; - for (Map.Entry entry : deferredAggs.entrySet()) { - AggValueSource agg = entry.getValue(); - SlotAcc acc = agg.createSlotAcc(fcontext, numDocs, numSlots); - acc.key = entry.getKey(); - accMap.put(acc.key, acc); - otherAccs[otherAccIdx++] = acc; - } - - if (numDeferred == freq.getFacetStats().size()) { - // accs and otherAccs are the same... - accs = otherAccs; - } - } - - - int collectFirstPhase(DocSet docs, int slot) throws IOException { - int num = -1; - if (collectAcc != null) { - num = collectAcc.collect(docs, slot); - } - if (allBucketsAcc != null) { - num = allBucketsAcc.collect(docs, slot); - } - return num >= 0 ? num : docs.size(); - } - - void collectFirstPhase(int segDoc, int slot) throws IOException { - if (collectAcc != null) { - collectAcc.collect(segDoc, slot); - } - if (allBucketsAcc != null) { - allBucketsAcc.collect(segDoc, slot); - } - } - - - void fillBucket(SimpleOrderedMap target, int count, int slotNum, DocSet subDomain, Query filter) throws IOException { - target.add("count", count); - if (count <= 0 && !freq.processEmpty) return; - - if (collectAcc != null && slotNum >= 0) { - collectAcc.setValues(target, slotNum); - } - - createOtherAccs(-1, 1); - - if (otherAccs == null && freq.subFacets.isEmpty()) return; - - if (subDomain == null) { - subDomain = fcontext.searcher.getDocSet(filter, fcontext.base); - } - - // if no subFacets, we only need a DocSet - // otherwise we need more? - // TODO: save something generic like "slotNum" in the context and use that to implement things like filter exclusion if necessary? - // Hmmm, but we need to look up some stuff anyway (for the label?) - // have a method like "DocSet applyConstraint(facet context, DocSet parent)" - // that's needed for domain changing things like joins anyway??? - - if (otherAccs != null) { - // do acc at a time (traversing domain each time) or do all accs for each doc? - for (SlotAcc acc : otherAccs) { - acc.reset(); // TODO: only needed if we previously used for allBuckets or missing - acc.collect(subDomain, 0); - acc.setValues(target, 0); - } - } - - processSubs(target, filter, subDomain); - } - - - @Override - protected void processStats(SimpleOrderedMap bucket, DocSet docs, int docCount) throws IOException { - if (docCount == 0 && !freq.processEmpty || freq.getFacetStats().size() == 0) { - bucket.add("count", docCount); - return; - } - createAccs(docCount, 1); - int collected = collect(docs, 0); - - // countAcc.incrementCount(0, collected); // should we set the counton the acc instead of just passing it? - - assert collected == docCount; - addStats(bucket, collected, 0); - } - - // overrides but with different signature! - void addStats(SimpleOrderedMap target, int count, int slotNum) throws IOException { - target.add("count", count); - if (count > 0 || freq.processEmpty) { - for (SlotAcc acc : accs) { - acc.setValues(target, slotNum); - } - } - } - - @Override - void setNextReader(LeafReaderContext ctx) throws IOException { - // base class calls this (for missing bucket...) ... go over accs[] in that case - super.setNextReader(ctx); - } - - void setNextReaderFirstPhase(LeafReaderContext ctx) throws IOException { - if (collectAcc != null) { - collectAcc.setNextReader(ctx); - } - if (otherAccs != null) { - for (SlotAcc acc : otherAccs) { - acc.setNextReader(ctx); - } - } - } - - static class Slot { - int slot; - public int tiebreakCompare(int slotA, int slotB) { - return slotB - slotA; - } - } -} - -class SpecialSlotAcc extends SlotAcc { - SlotAcc collectAcc; - SlotAcc[] otherAccs; - int collectAccSlot; - int otherAccsSlot; - long count; - - public SpecialSlotAcc(FacetContext fcontext, SlotAcc collectAcc, int collectAccSlot, SlotAcc[] otherAccs, int otherAccsSlot) { - super(fcontext); - this.collectAcc = collectAcc; - this.collectAccSlot = collectAccSlot; - this.otherAccs = otherAccs; - this.otherAccsSlot = otherAccsSlot; - } - - public int getCollectAccSlot() { return collectAccSlot; } - public int getOtherAccSlot() { return otherAccsSlot; } - - public long getSpecialCount() { - return count; - } - - @Override - public void collect(int doc, int slot) throws IOException { - assert slot != collectAccSlot || slot < 0; - count++; - if (collectAcc != null) { - collectAcc.collect(doc, collectAccSlot); - } - if (otherAccs != null) { - for (SlotAcc otherAcc : otherAccs) { - otherAcc.collect(doc, otherAccsSlot); - } - } - } - - @Override - public void setNextReader(LeafReaderContext readerContext) throws IOException { - // collectAcc and otherAccs will normally have setNextReader called directly on them. - // This, however, will be used when collect(DocSet,slot) variant is used on this Acc. - if (collectAcc != null) { - collectAcc.setNextReader(readerContext); - } - if (otherAccs != null) { - for (SlotAcc otherAcc : otherAccs) { - otherAcc.setNextReader(readerContext); - } - } - } - - @Override - public int compare(int slotA, int slotB) { - throw new UnsupportedOperationException(); - } - - @Override - public Object getValue(int slotNum) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public void setValues(SimpleOrderedMap bucket, int slotNum) throws IOException { - if (collectAcc != null) { - collectAcc.setValues(bucket, collectAccSlot); - } - if (otherAccs != null) { - for (SlotAcc otherAcc : otherAccs) { - otherAcc.setValues(bucket, otherAccsSlot); - } - } - } - - @Override - public void reset() { - // reset should be called on underlying accs - // TODO: but in case something does need to be done here, should we require this method to be called but do nothing for now? - throw new UnsupportedOperationException(); - } - - @Override - public void resize(Resizer resizer) { - // someone else will call resize on collectAcc directly - if (collectAccSlot >= 0) { - collectAccSlot = resizer.getNewSlot(collectAccSlot); - } - } -} - - - - -// base class for FC style of facet counting (single and multi-valued strings) -abstract class FacetFieldProcessorFCBase extends FacetFieldProcessor { - BytesRefBuilder prefixRef; - int startTermIndex; - int endTermIndex; - int nTerms; - int nDocs; - int maxSlots; - - int allBucketsSlot = -1; // slot for the primary Accs (countAcc, collectAcc) - - public FacetFieldProcessorFCBase(FacetContext fcontext, FacetField freq, SchemaField sf) { - super(fcontext, freq, sf); - } - - @Override - public void process() throws IOException { - super.process(); - sf = fcontext.searcher.getSchema().getField(freq.field); - response = getFieldCacheCounts(); - } - - - /** this BytesRef may be shared across calls and should be deep-cloned if necessary */ - abstract protected BytesRef lookupOrd(int ord) throws IOException; - abstract protected void findStartAndEndOrds() throws IOException; - abstract protected void collectDocs() throws IOException; - - - public SimpleOrderedMap getFieldCacheCounts() throws IOException { - String prefix = freq.prefix; - if (prefix == null || prefix.length() == 0) { - prefixRef = null; - } else { - prefixRef = new BytesRefBuilder(); - prefixRef.copyChars(prefix); - } - - findStartAndEndOrds(); - - maxSlots = nTerms; - - if (freq.allBuckets) { - allBucketsSlot = maxSlots++; - } - - createCollectAcc(nDocs, maxSlots); - - if (freq.allBuckets) { - allBucketsAcc = new SpecialSlotAcc(fcontext, collectAcc, allBucketsSlot, otherAccs, 0); - } - - collectDocs(); - - return findTopSlots(); - } - - - protected SimpleOrderedMap findTopSlots() throws IOException { - SimpleOrderedMap res = new SimpleOrderedMap<>(); - - int numBuckets = 0; - List bucketVals = null; - if (freq.numBuckets && fcontext.isShard()) { - bucketVals = new ArrayList(100); - } - - int off = fcontext.isShard() ? 0 : (int) freq.offset; - // add a modest amount of over-request if this is a shard request - int lim = freq.limit >= 0 ? (fcontext.isShard() ? (int)(freq.limit*1.1+4) : (int)freq.limit) : Integer.MAX_VALUE; - - int maxsize = (int)(freq.limit >= 0 ? freq.offset + lim : Integer.MAX_VALUE - 1); - maxsize = Math.min(maxsize, nTerms); - - final int sortMul = freq.sortDirection.getMultiplier(); - final SlotAcc sortAcc = this.sortAcc; - - PriorityQueue queue = new PriorityQueue(maxsize) { - @Override - protected boolean lessThan(Slot a, Slot b) { - int cmp = sortAcc.compare(a.slot, b.slot) * sortMul; - return cmp == 0 ? b.slot < a.slot : cmp < 0; - } - }; - - Slot bottom = null; - for (int i = 0; i < nTerms; i++) { - // screen out buckets not matching mincount immediately (i.e. don't even increment numBuckets) - if (effectiveMincount > 0 && countAcc.getCount(i) < effectiveMincount) { - continue; - } - - numBuckets++; - if (bucketVals != null && bucketVals.size()<100) { - int ord = startTermIndex + i; - BytesRef br = lookupOrd(ord); - Object val = sf.getType().toObject(sf, br); - bucketVals.add(val); - } - - - if (bottom != null) { - if (sortAcc.compare(bottom.slot, i) * sortMul < 0) { - bottom.slot = i; - bottom = queue.updateTop(); - } - } else if (lim > 0) { - // queue not full - Slot s = new Slot(); - s.slot = i; - queue.add(s); - if (queue.size() >= maxsize) { - bottom = queue.top(); - } - } - } - - if (freq.numBuckets) { - if (!fcontext.isShard()) { - res.add("numBuckets", numBuckets); - } else { - SimpleOrderedMap map = new SimpleOrderedMap(2); - map.add("numBuckets", numBuckets); - map.add("vals", bucketVals); - res.add("numBuckets", map); - } - } - - FacetDebugInfo fdebug = fcontext.getDebugInfo(); - if (fdebug != null) fdebug.putInfoItem("numBuckets", new Long(numBuckets)); - - // if we are deep paging, we don't have to order the highest "offset" counts. - int collectCount = Math.max(0, queue.size() - off); - assert collectCount <= lim; - int[] sortedSlots = new int[collectCount]; - for (int i = collectCount - 1; i >= 0; i--) { - sortedSlots[i] = queue.pop().slot; - } - - if (freq.allBuckets) { - SimpleOrderedMap allBuckets = new SimpleOrderedMap<>(); - allBuckets.add("count", allBucketsAcc.getSpecialCount()); - if (allBucketsAcc != null) { - allBucketsAcc.setValues(allBuckets, allBucketsSlot); - } - res.add("allBuckets", allBuckets); - } - - ArrayList bucketList = new ArrayList(collectCount); - res.add("buckets", bucketList); - - - // TODO: do this with a callback instead? - boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0; - - for (int slotNum : sortedSlots) { - SimpleOrderedMap bucket = new SimpleOrderedMap<>(); - - // get the ord of the slot... - int ord = startTermIndex + slotNum; - - BytesRef br = lookupOrd(ord); - Object val = sf.getType().toObject(sf, br); - - bucket.add("val", val); - - TermQuery filter = needFilter ? new TermQuery(new Term(sf.getName(), br)) : null; - fillBucket(bucket, countAcc.getCount(slotNum), slotNum, null, filter); - - bucketList.add(bucket); - } - - if (freq.missing) { - SimpleOrderedMap missingBucket = new SimpleOrderedMap<>(); - fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null); - res.add("missing", missingBucket); - } - - return res; - } - - -} - - - - - -// UnInvertedField implementation of field faceting -class FacetFieldProcessorUIF extends FacetFieldProcessorFCBase { - UnInvertedField uif; - TermsEnum te; - - FacetFieldProcessorUIF(FacetContext fcontext, FacetField freq, SchemaField sf) { - super(fcontext, freq, sf); - } - - @Override - protected void findStartAndEndOrds() throws IOException { - uif = UnInvertedField.getUnInvertedField(freq.field, fcontext.searcher); - te = uif.getOrdTermsEnum( fcontext.searcher.getLeafReader() ); // "te" can be null - - startTermIndex = 0; - endTermIndex = uif.numTerms(); // one past the end - - if (prefixRef != null && te != null) { - if (te.seekCeil(prefixRef.get()) == TermsEnum.SeekStatus.END) { - startTermIndex = uif.numTerms(); - } else { - startTermIndex = (int) te.ord(); - } - prefixRef.append(UnicodeUtil.BIG_TERM); - if (te.seekCeil(prefixRef.get()) == TermsEnum.SeekStatus.END) { - endTermIndex = uif.numTerms(); - } else { - endTermIndex = (int) te.ord(); - } - } - - nTerms = endTermIndex - startTermIndex; - } - - @Override - protected BytesRef lookupOrd(int ord) throws IOException { - return uif.getTermValue(te, ord); - } - - @Override - protected void collectDocs() throws IOException { - uif.collectDocs(this); - } -} - - - -class FacetFieldProcessorStream extends FacetFieldProcessor implements Closeable { - long bucketsToSkip; - long bucketsReturned; - - boolean closed; - boolean countOnly; - boolean hasSubFacets; // true if there are subfacets - int minDfFilterCache; - DocSet docs; - DocSet fastForRandomSet; - TermsEnum termsEnum = null; - SolrIndexSearcher.DocsEnumState deState = null; - PostingsEnum postingsEnum; - BytesRef startTermBytes; - BytesRef term; - LeafReaderContext[] leaves; - - - - FacetFieldProcessorStream(FacetContext fcontext, FacetField freq, SchemaField sf) { - super(fcontext, freq, sf); - } - - @Override - public void close() throws IOException { - if (!closed) { - closed = true; - // fcontext.base.decref(); // OFF-HEAP - } - } - - - @Override - public void process() throws IOException { - super.process(); - - // We need to keep the fcontext open after processing is done (since we will be streaming in the response writer). - // But if the connection is broken, we want to clean up. - // fcontext.base.incref(); // OFF-HEAP - fcontext.qcontext.addCloseHook(this); - - setup(); - response = new SimpleOrderedMap<>(); - response.add("buckets", new Iterator() { - boolean retrieveNext = true; - Object val; - - @Override - public boolean hasNext() { - if (retrieveNext) { - val = nextBucket(); - } - retrieveNext = false; - return val != null; - } - - @Override - public Object next() { - if (retrieveNext) { - val = nextBucket(); - } - retrieveNext = true; - if (val == null) { - // Last value, so clean up. In the case that we are doing streaming facets within streaming facets, - // the number of close hooks could grow very large, so we want to remove ourselves. - boolean removed = fcontext.qcontext.removeCloseHook(FacetFieldProcessorStream.this); - assert removed; - try { - close(); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error during facet streaming close", e); - } - } - return val; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }); - } - - - - public void setup() throws IOException { - - countOnly = freq.facetStats.size() == 0 || freq.facetStats.values().iterator().next() instanceof CountAgg; - hasSubFacets = freq.subFacets.size() > 0; - bucketsToSkip = freq.offset; - - createAccs(-1, 1); - - // Minimum term docFreq in order to use the filterCache for that term. - if (freq.cacheDf == -1) { // -1 means never cache - minDfFilterCache = Integer.MAX_VALUE; - } else if (freq.cacheDf == 0) { // default; compute as fraction of maxDoc - minDfFilterCache = Math.max(fcontext.searcher.maxDoc() >> 4, 3); // (minimum of 3 is for test coverage purposes) - } else { - minDfFilterCache = freq.cacheDf; - } - - docs = fcontext.base; - fastForRandomSet = null; - - if (freq.prefix != null) { - String indexedPrefix = sf.getType().toInternal(freq.prefix); - startTermBytes = new BytesRef(indexedPrefix); - } else if (sf.getType().getNumericType() != null) { - String triePrefix = TrieField.getMainValuePrefix(sf.getType()); - if (triePrefix != null) { - startTermBytes = new BytesRef(triePrefix); - } - } - - Fields fields = fcontext.searcher.getLeafReader().fields(); - Terms terms = fields == null ? null : fields.terms(sf.getName()); - - - termsEnum = null; - deState = null; - term = null; - - - if (terms != null) { - - termsEnum = terms.iterator(); - - // TODO: OPT: if seek(ord) is supported for this termsEnum, then we could use it for - // facet.offset when sorting by index order. - - if (startTermBytes != null) { - if (termsEnum.seekCeil(startTermBytes) == TermsEnum.SeekStatus.END) { - termsEnum = null; - } else { - term = termsEnum.term(); - } - } else { - // position termsEnum on first term - term = termsEnum.next(); - } - } - - List leafList = fcontext.searcher.getTopReaderContext().leaves(); - leaves = leafList.toArray( new LeafReaderContext[ leafList.size() ]); - } - - - public SimpleOrderedMap nextBucket() { - try { - return _nextBucket(); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error during facet streaming", e); - } - } - - public SimpleOrderedMap _nextBucket() throws IOException { - DocSet termSet = null; - - try { - while (term != null) { - - if (startTermBytes != null && !StringHelper.startsWith(term, startTermBytes)) { - break; - } - - int df = termsEnum.docFreq(); - if (df < effectiveMincount) { - term = termsEnum.next(); - continue; - } - - if (termSet != null) { - // termSet.decref(); // OFF-HEAP - termSet = null; - } - - int c = 0; - - if (hasSubFacets || df >= minDfFilterCache) { - // use the filter cache - - if (deState == null) { - deState = new SolrIndexSearcher.DocsEnumState(); - deState.fieldName = sf.getName(); - deState.liveDocs = fcontext.searcher.getLeafReader().getLiveDocs(); - deState.termsEnum = termsEnum; - deState.postingsEnum = postingsEnum; - deState.minSetSizeCached = minDfFilterCache; - } - - if (hasSubFacets || !countOnly) { - DocSet termsAll = fcontext.searcher.getDocSet(deState); - termSet = docs.intersection(termsAll); - // termsAll.decref(); // OFF-HEAP - c = termSet.size(); - } else { - c = fcontext.searcher.numDocs(docs, deState); - } - postingsEnum = deState.postingsEnum; - - resetStats(); - - if (!countOnly) { - collect(termSet, 0); - } - - } else { - // We don't need the docset here (meaning no sub-facets). - // if countOnly, then we are calculating some other stats... - resetStats(); - - // lazy convert to fastForRandomSet - if (fastForRandomSet == null) { - fastForRandomSet = docs; - if (docs instanceof SortedIntDocSet) { // OFF-HEAP todo: also check for native version - SortedIntDocSet sset = (SortedIntDocSet) docs; - fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size()); - } - } - // iterate over TermDocs to calculate the intersection - postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE); - - if (postingsEnum instanceof MultiPostingsEnum) { - MultiPostingsEnum.EnumWithSlice[] subs = ((MultiPostingsEnum) postingsEnum).getSubs(); - int numSubs = ((MultiPostingsEnum) postingsEnum).getNumSubs(); - for (int subindex = 0; subindex < numSubs; subindex++) { - MultiPostingsEnum.EnumWithSlice sub = subs[subindex]; - if (sub.postingsEnum == null) continue; - int base = sub.slice.start; - int docid; - - if (countOnly) { - while ((docid = sub.postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (fastForRandomSet.exists(docid + base)) c++; - } - } else { - setNextReader(leaves[sub.slice.readerIndex]); - while ((docid = sub.postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (fastForRandomSet.exists(docid + base)) { - c++; - collect(docid, 0); - } - } - } - - } - } else { - int docid; - if (countOnly) { - while ((docid = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (fastForRandomSet.exists(docid)) c++; - } - } else { - setNextReader(leaves[0]); - while ((docid = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (fastForRandomSet.exists(docid)) { - c++; - collect(docid, 0); - } - } - } - } - - } - - - - if (c < effectiveMincount) { - term = termsEnum.next(); - continue; - } - - // handle offset and limit - if (bucketsToSkip > 0) { - bucketsToSkip--; - term = termsEnum.next(); - continue; - } - - if (freq.limit >= 0 && ++bucketsReturned > freq.limit) { - return null; - } - - // set count in case other stats depend on it - countAcc.incrementCount(0, c); - - // OK, we have a good bucket to return... first get bucket value before moving to next term - Object bucketVal = sf.getType().toObject(sf, term); - TermQuery bucketQuery = hasSubFacets ? new TermQuery(new Term(freq.field, term)) : null; - term = termsEnum.next(); - - SimpleOrderedMap bucket = new SimpleOrderedMap<>(); - bucket.add("val", bucketVal); - addStats(bucket, 0); - if (hasSubFacets) { - processSubs(bucket, bucketQuery, termSet); - } - - // TODO... termSet needs to stick around for streaming sub-facets? - - return bucket; - - } - - } finally { - if (termSet != null) { - // termSet.decref(); // OFF-HEAP - termSet = null; - } - } - - - // end of the iteration - return null; - } - - - -} - - diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java new file mode 100644 index 00000000000..a73732135e8 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java @@ -0,0 +1,369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.facet; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Query; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.DocSet; + +/** + * Facet processing based on field values. (not range nor by query) + * @see FacetField + */ +abstract class FacetFieldProcessor extends FacetProcessor { + SchemaField sf; + SlotAcc indexOrderAcc; + int effectiveMincount; + + Map deferredAggs; // null if none + + // TODO: push any of this down to base class? + + // + // For sort="x desc", collectAcc would point to "x", and sortAcc would also point to "x". + // collectAcc would be used to accumulate all buckets, and sortAcc would be used to sort those buckets. + // + SlotAcc collectAcc; // Accumulator to collect across entire domain (in addition to the countAcc). May be null. + SlotAcc sortAcc; // Accumulator to use for sorting *only* (i.e. not used for collection). May be an alias of countAcc, collectAcc, or indexOrderAcc + SlotAcc[] otherAccs; // Accumulators that do not need to be calculated across all buckets. + + SpecialSlotAcc allBucketsAcc; // this can internally refer to otherAccs and/or collectAcc. setNextReader should be called on otherAccs directly if they exist. + + FacetFieldProcessor(FacetContext fcontext, FacetField freq, SchemaField sf) { + super(fcontext, freq); + this.sf = sf; + this.effectiveMincount = (int)(fcontext.isShard() ? Math.min(1 , freq.mincount) : freq.mincount); + } + + // This is used to create accs for second phase (or to create accs for all aggs) + @Override + protected void createAccs(int docCount, int slotCount) throws IOException { + if (accMap == null) { + accMap = new LinkedHashMap<>(); + } + + // allow a custom count acc to be used + if (countAcc == null) { + countAcc = new CountSlotArrAcc(fcontext, slotCount); + countAcc.key = "count"; + } + + if (accs != null) { + // reuse these accs, but reset them first + for (SlotAcc acc : accs) { + acc.reset(); + } + return; + } else { + accs = new SlotAcc[ freq.getFacetStats().size() ]; + } + + int accIdx = 0; + for (Map.Entry entry : freq.getFacetStats().entrySet()) { + SlotAcc acc = null; + if (slotCount == 1) { + acc = accMap.get(entry.getKey()); + if (acc != null) { + acc.reset(); + } + } + if (acc == null) { + acc = entry.getValue().createSlotAcc(fcontext, docCount, slotCount); + acc.key = entry.getKey(); + accMap.put(acc.key, acc); + } + accs[accIdx++] = acc; + } + } + + void createCollectAcc(int numDocs, int numSlots) throws IOException { + accMap = new LinkedHashMap<>(); + + // we always count... + // allow a subclass to set a custom counter. + if (countAcc == null) { + countAcc = new CountSlotArrAcc(fcontext, numSlots); + } + + if ("count".equals(freq.sortVariable)) { + sortAcc = countAcc; + deferredAggs = freq.getFacetStats(); + } else if ("index".equals(freq.sortVariable)) { + // allow subclass to set indexOrderAcc first + if (indexOrderAcc == null) { + // This sorting accumulator just goes by the slot number, so does not need to be collected + // and hence does not need to find it's way into the accMap or accs array. + indexOrderAcc = new SortSlotAcc(fcontext); + } + sortAcc = indexOrderAcc; + deferredAggs = freq.getFacetStats(); + } else { + AggValueSource sortAgg = freq.getFacetStats().get(freq.sortVariable); + if (sortAgg != null) { + collectAcc = sortAgg.createSlotAcc(fcontext, numDocs, numSlots); + collectAcc.key = freq.sortVariable; // TODO: improve this + } + sortAcc = collectAcc; + deferredAggs = new HashMap<>(freq.getFacetStats()); + deferredAggs.remove(freq.sortVariable); + } + + if (deferredAggs.size() == 0) { + deferredAggs = null; + } + + boolean needOtherAccs = freq.allBuckets; // TODO: use for missing too... + + if (!needOtherAccs) { + // we may need them later, but we don't want to create them now + // otherwise we won't know if we need to call setNextReader on them. + return; + } + + // create the deferred aggs up front for use by allBuckets + createOtherAccs(numDocs, 1); + } + + private void createOtherAccs(int numDocs, int numSlots) throws IOException { + if (otherAccs != null) { + // reuse existing accumulators + for (SlotAcc acc : otherAccs) { + acc.reset(); // todo - make reset take numDocs and numSlots? + } + return; + } + + int numDeferred = deferredAggs == null ? 0 : deferredAggs.size(); + if (numDeferred <= 0) return; + + otherAccs = new SlotAcc[ numDeferred ]; + + int otherAccIdx = 0; + for (Map.Entry entry : deferredAggs.entrySet()) { + AggValueSource agg = entry.getValue(); + SlotAcc acc = agg.createSlotAcc(fcontext, numDocs, numSlots); + acc.key = entry.getKey(); + accMap.put(acc.key, acc); + otherAccs[otherAccIdx++] = acc; + } + + if (numDeferred == freq.getFacetStats().size()) { + // accs and otherAccs are the same... + accs = otherAccs; + } + } + + int collectFirstPhase(DocSet docs, int slot) throws IOException { + int num = -1; + if (collectAcc != null) { + num = collectAcc.collect(docs, slot); + } + if (allBucketsAcc != null) { + num = allBucketsAcc.collect(docs, slot); + } + return num >= 0 ? num : docs.size(); + } + + void collectFirstPhase(int segDoc, int slot) throws IOException { + if (collectAcc != null) { + collectAcc.collect(segDoc, slot); + } + if (allBucketsAcc != null) { + allBucketsAcc.collect(segDoc, slot); + } + } + + void fillBucket(SimpleOrderedMap target, int count, int slotNum, DocSet subDomain, Query filter) throws IOException { + target.add("count", count); + if (count <= 0 && !freq.processEmpty) return; + + if (collectAcc != null && slotNum >= 0) { + collectAcc.setValues(target, slotNum); + } + + createOtherAccs(-1, 1); + + if (otherAccs == null && freq.subFacets.isEmpty()) return; + + if (subDomain == null) { + subDomain = fcontext.searcher.getDocSet(filter, fcontext.base); + } + + // if no subFacets, we only need a DocSet + // otherwise we need more? + // TODO: save something generic like "slotNum" in the context and use that to implement things like filter exclusion if necessary? + // Hmmm, but we need to look up some stuff anyway (for the label?) + // have a method like "DocSet applyConstraint(facet context, DocSet parent)" + // that's needed for domain changing things like joins anyway??? + + if (otherAccs != null) { + // do acc at a time (traversing domain each time) or do all accs for each doc? + for (SlotAcc acc : otherAccs) { + acc.reset(); // TODO: only needed if we previously used for allBuckets or missing + acc.collect(subDomain, 0); + acc.setValues(target, 0); + } + } + + processSubs(target, filter, subDomain); + } + + @Override + protected void processStats(SimpleOrderedMap bucket, DocSet docs, int docCount) throws IOException { + if (docCount == 0 && !freq.processEmpty || freq.getFacetStats().size() == 0) { + bucket.add("count", docCount); + return; + } + createAccs(docCount, 1); + int collected = collect(docs, 0); + + // countAcc.incrementCount(0, collected); // should we set the counton the acc instead of just passing it? + + assert collected == docCount; + addStats(bucket, collected, 0); + } + + // overrides but with different signature! + private void addStats(SimpleOrderedMap target, int count, int slotNum) throws IOException { + target.add("count", count); + if (count > 0 || freq.processEmpty) { + for (SlotAcc acc : accs) { + acc.setValues(target, slotNum); + } + } + } + + @Override + void setNextReader(LeafReaderContext ctx) throws IOException { + // base class calls this (for missing bucket...) ... go over accs[] in that case + super.setNextReader(ctx); + } + + void setNextReaderFirstPhase(LeafReaderContext ctx) throws IOException { + if (collectAcc != null) { + collectAcc.setNextReader(ctx); + } + if (otherAccs != null) { + for (SlotAcc acc : otherAccs) { + acc.setNextReader(ctx); + } + } + } + + static class Slot { + int slot; + public int tiebreakCompare(int slotA, int slotB) { + return slotB - slotA; + } + } + + static class SpecialSlotAcc extends SlotAcc { + SlotAcc collectAcc; + SlotAcc[] otherAccs; + int collectAccSlot; + int otherAccsSlot; + long count; + + SpecialSlotAcc(FacetContext fcontext, SlotAcc collectAcc, int collectAccSlot, SlotAcc[] otherAccs, int otherAccsSlot) { + super(fcontext); + this.collectAcc = collectAcc; + this.collectAccSlot = collectAccSlot; + this.otherAccs = otherAccs; + this.otherAccsSlot = otherAccsSlot; + } + + public int getCollectAccSlot() { return collectAccSlot; } + public int getOtherAccSlot() { return otherAccsSlot; } + + long getSpecialCount() { + return count; + } + + @Override + public void collect(int doc, int slot) throws IOException { + assert slot != collectAccSlot || slot < 0; + count++; + if (collectAcc != null) { + collectAcc.collect(doc, collectAccSlot); + } + if (otherAccs != null) { + for (SlotAcc otherAcc : otherAccs) { + otherAcc.collect(doc, otherAccsSlot); + } + } + } + + @Override + public void setNextReader(LeafReaderContext readerContext) throws IOException { + // collectAcc and otherAccs will normally have setNextReader called directly on them. + // This, however, will be used when collect(DocSet,slot) variant is used on this Acc. + if (collectAcc != null) { + collectAcc.setNextReader(readerContext); + } + if (otherAccs != null) { + for (SlotAcc otherAcc : otherAccs) { + otherAcc.setNextReader(readerContext); + } + } + } + + @Override + public int compare(int slotA, int slotB) { + throw new UnsupportedOperationException(); + } + + @Override + public Object getValue(int slotNum) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void setValues(SimpleOrderedMap bucket, int slotNum) throws IOException { + if (collectAcc != null) { + collectAcc.setValues(bucket, collectAccSlot); + } + if (otherAccs != null) { + for (SlotAcc otherAcc : otherAccs) { + otherAcc.setValues(bucket, otherAccsSlot); + } + } + } + + @Override + public void reset() { + // reset should be called on underlying accs + // TODO: but in case something does need to be done here, should we require this method to be called but do nothing for now? + throw new UnsupportedOperationException(); + } + + @Override + public void resize(Resizer resizer) { + // someone else will call resize on collectAcc directly + if (collectAccSlot >= 0) { + collectAccSlot = resizer.getNewSlot(collectAccSlot); + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java new file mode 100644 index 00000000000..10aa4d9b393 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.facet; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.PriorityQueue; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.schema.SchemaField; + +/** + * Base class for DV/UIF accumulating counts into an array by ordinal. + * It can handle terms (strings), not numbers directly but those encoded as terms, and is multi-valued capable. + */ +abstract class FacetFieldProcessorByArray extends FacetFieldProcessor { + BytesRefBuilder prefixRef; + int startTermIndex; + int endTermIndex; + int nTerms; + int nDocs; + int maxSlots; + + int allBucketsSlot = -1; // slot for the primary Accs (countAcc, collectAcc) + + FacetFieldProcessorByArray(FacetContext fcontext, FacetField freq, SchemaField sf) { + super(fcontext, freq, sf); + } + + abstract protected void findStartAndEndOrds() throws IOException; + + abstract protected void collectDocs() throws IOException; + + /** this BytesRef may be shared across calls and should be deep-cloned if necessary */ + abstract protected BytesRef lookupOrd(int ord) throws IOException; + + @Override + public void process() throws IOException { + super.process(); + sf = fcontext.searcher.getSchema().getField(freq.field); + response = getFieldCacheCounts(); + } + + private SimpleOrderedMap getFieldCacheCounts() throws IOException { + String prefix = freq.prefix; + if (prefix == null || prefix.length() == 0) { + prefixRef = null; + } else { + prefixRef = new BytesRefBuilder(); + prefixRef.copyChars(prefix); + } + + findStartAndEndOrds(); + + maxSlots = nTerms; + + if (freq.allBuckets) { + allBucketsSlot = maxSlots++; + } + + createCollectAcc(nDocs, maxSlots); + + if (freq.allBuckets) { + allBucketsAcc = new SpecialSlotAcc(fcontext, collectAcc, allBucketsSlot, otherAccs, 0); + } + + collectDocs(); + + return findTopSlots(); + } + + private SimpleOrderedMap findTopSlots() throws IOException { + SimpleOrderedMap res = new SimpleOrderedMap<>(); + + int numBuckets = 0; + List bucketVals = null; + if (freq.numBuckets && fcontext.isShard()) { + bucketVals = new ArrayList<>(100); + } + + int off = fcontext.isShard() ? 0 : (int) freq.offset; + // add a modest amount of over-request if this is a shard request + int lim = freq.limit >= 0 ? (fcontext.isShard() ? (int)(freq.limit*1.1+4) : (int)freq.limit) : Integer.MAX_VALUE; + + int maxsize = (int)(freq.limit >= 0 ? freq.offset + lim : Integer.MAX_VALUE - 1); + maxsize = Math.min(maxsize, nTerms); + + final int sortMul = freq.sortDirection.getMultiplier(); + final SlotAcc sortAcc = this.sortAcc; + + PriorityQueue queue = new PriorityQueue(maxsize) { + @Override + protected boolean lessThan(Slot a, Slot b) { + int cmp = sortAcc.compare(a.slot, b.slot) * sortMul; + return cmp == 0 ? b.slot < a.slot : cmp < 0; + } + }; + + Slot bottom = null; + for (int i = 0; i < nTerms; i++) { + // screen out buckets not matching mincount immediately (i.e. don't even increment numBuckets) + if (effectiveMincount > 0 && countAcc.getCount(i) < effectiveMincount) { + continue; + } + + numBuckets++; + if (bucketVals != null && bucketVals.size()<100) { + int ord = startTermIndex + i; + BytesRef br = lookupOrd(ord); + Object val = sf.getType().toObject(sf, br); + bucketVals.add(val); + } + + if (bottom != null) { + if (sortAcc.compare(bottom.slot, i) * sortMul < 0) { + bottom.slot = i; + bottom = queue.updateTop(); + } + } else if (lim > 0) { + // queue not full + Slot s = new Slot(); + s.slot = i; + queue.add(s); + if (queue.size() >= maxsize) { + bottom = queue.top(); + } + } + } + + if (freq.numBuckets) { + if (!fcontext.isShard()) { + res.add("numBuckets", numBuckets); + } else { + SimpleOrderedMap map = new SimpleOrderedMap<>(2); + map.add("numBuckets", numBuckets); + map.add("vals", bucketVals); + res.add("numBuckets", map); + } + } + + FacetDebugInfo fdebug = fcontext.getDebugInfo(); + if (fdebug != null) fdebug.putInfoItem("numBuckets", (long) numBuckets); + + // if we are deep paging, we don't have to order the highest "offset" counts. + int collectCount = Math.max(0, queue.size() - off); + assert collectCount <= lim; + int[] sortedSlots = new int[collectCount]; + for (int i = collectCount - 1; i >= 0; i--) { + sortedSlots[i] = queue.pop().slot; + } + + if (freq.allBuckets) { + SimpleOrderedMap allBuckets = new SimpleOrderedMap<>(); + allBuckets.add("count", allBucketsAcc.getSpecialCount()); + if (allBucketsAcc != null) { + allBucketsAcc.setValues(allBuckets, allBucketsSlot); + } + res.add("allBuckets", allBuckets); + } + + ArrayList> bucketList = new ArrayList<>(collectCount); + res.add("buckets", bucketList); + + // TODO: do this with a callback instead? + boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0; + + for (int slotNum : sortedSlots) { + SimpleOrderedMap bucket = new SimpleOrderedMap<>(); + + // get the ord of the slot... + int ord = startTermIndex + slotNum; + + BytesRef br = lookupOrd(ord); + Object val = sf.getType().toObject(sf, br); + + bucket.add("val", val); + + TermQuery filter = needFilter ? new TermQuery(new Term(sf.getName(), br)) : null; + fillBucket(bucket, countAcc.getCount(slotNum), slotNum, null, filter); + + bucketList.add(bucket); + } + + if (freq.missing) { + SimpleOrderedMap missingBucket = new SimpleOrderedMap<>(); + fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null); + res.add("missing", missingBucket); + } + + return res; + } + +} diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorDV.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java similarity index 98% rename from solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorDV.java rename to solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java index 12056aa118a..1ef42848ffb 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorDV.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java @@ -34,23 +34,22 @@ import org.apache.solr.common.SolrException; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.Filter; -class FacetFieldProcessorDV extends FacetFieldProcessorFCBase { +/** + * Grabs values from {@link DocValues}. + */ +class FacetFieldProcessorByArrayDV extends FacetFieldProcessorByArray { static boolean unwrap_singleValued_multiDv = true; // only set to false for test coverage boolean multiValuedField; SortedSetDocValues si; // only used for term lookups (for both single and multi-valued) MultiDocValues.OrdinalMap ordinalMap = null; // maps per-segment ords to global ords - - public FacetFieldProcessorDV(FacetContext fcontext, FacetField freq, SchemaField sf) { + FacetFieldProcessorByArrayDV(FacetContext fcontext, FacetField freq, SchemaField sf) { super(fcontext, freq, sf); multiValuedField = sf.multiValued() || sf.getType().multiValuedFieldCache(); } - protected BytesRef lookupOrd(int ord) throws IOException { - return si.lookupOrd(ord); - } - + @Override protected void findStartAndEndOrds() throws IOException { if (multiValuedField) { si = FieldUtil.getSortedSetDocValues(fcontext.qcontext, sf, null); @@ -175,16 +174,9 @@ class FacetFieldProcessorDV extends FacetFieldProcessorFCBase { reuse = null; // better GC } - private int[] reuse; - private int[] getCountArr(int maxNeeded) { - if (reuse == null) { - // make the count array large enough for any segment - // FUTURE: (optionally) directly use the array of the CountAcc for an optimized index.. - reuse = new int[(int) si.getValueCount() + 1]; - } else { - Arrays.fill(reuse, 0, maxNeeded, 0); - } - return reuse; + @Override + protected BytesRef lookupOrd(int ord) throws IOException { + return si.lookupOrd(ord); } private void collectPerSeg(SortedDocValues singleDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException { @@ -205,7 +197,6 @@ class FacetFieldProcessorDV extends FacetFieldProcessorFCBase { } } - private void collectPerSeg(SortedSetDocValues multiDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException { int segMax = (int)multiDv.getValueCount(); final int[] counts = getCountArr( segMax ); @@ -229,6 +220,18 @@ class FacetFieldProcessorDV extends FacetFieldProcessorFCBase { } } + private int[] reuse; + private int[] getCountArr(int maxNeeded) { + if (reuse == null) { + // make the count array large enough for any segment + // FUTURE: (optionally) directly use the array of the CountAcc for an optimized index.. + reuse = new int[(int) si.getValueCount() + 1]; + } else { + Arrays.fill(reuse, 0, maxNeeded, 0); + } + return reuse; + } + private void collectDocs(SortedDocValues singleDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException { int doc; while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayUIF.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayUIF.java new file mode 100644 index 00000000000..dfee257cb9e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayUIF.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.facet; + +import java.io.IOException; + +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.solr.schema.SchemaField; + +/** {@link UnInvertedField} implementation of field faceting. + * It's a top-level term cache. */ +class FacetFieldProcessorByArrayUIF extends FacetFieldProcessorByArray { + UnInvertedField uif; + TermsEnum te; + + FacetFieldProcessorByArrayUIF(FacetContext fcontext, FacetField freq, SchemaField sf) { + super(fcontext, freq, sf); + } + + @Override + protected void findStartAndEndOrds() throws IOException { + uif = UnInvertedField.getUnInvertedField(freq.field, fcontext.searcher); + te = uif.getOrdTermsEnum( fcontext.searcher.getLeafReader() ); // "te" can be null + + startTermIndex = 0; + endTermIndex = uif.numTerms(); // one past the end + + if (prefixRef != null && te != null) { + if (te.seekCeil(prefixRef.get()) == TermsEnum.SeekStatus.END) { + startTermIndex = uif.numTerms(); + } else { + startTermIndex = (int) te.ord(); + } + prefixRef.append(UnicodeUtil.BIG_TERM); + if (te.seekCeil(prefixRef.get()) == TermsEnum.SeekStatus.END) { + endTermIndex = uif.numTerms(); + } else { + endTermIndex = (int) te.ord(); + } + } + + nTerms = endTermIndex - startTermIndex; + } + + @Override + protected void collectDocs() throws IOException { + uif.collectDocs(this); + } + + @Override + protected BytesRef lookupOrd(int ord) throws IOException { + return uif.getTermValue(te, ord); + } +} diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java new file mode 100644 index 00000000000..005f4d1f174 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.facet; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiPostingsEnum; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.StringHelper; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.TrieField; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.HashDocSet; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SortedIntDocSet; + +/** + * Enumerates indexed terms in order in a streaming fashion. + * It's able to stream since no data needs to be accumulated so long as it's index order. + */ +class FacetFieldProcessorByEnumTermsStream extends FacetFieldProcessor implements Closeable { + long bucketsToSkip; + long bucketsReturned; + + boolean closed; + boolean countOnly; + boolean hasSubFacets; // true if there are subfacets + int minDfFilterCache; + DocSet docs; + DocSet fastForRandomSet; + TermsEnum termsEnum = null; + SolrIndexSearcher.DocsEnumState deState = null; + PostingsEnum postingsEnum; + BytesRef startTermBytes; + BytesRef term; + LeafReaderContext[] leaves; + + FacetFieldProcessorByEnumTermsStream(FacetContext fcontext, FacetField freq, SchemaField sf) { + super(fcontext, freq, sf); + } + + @Override + public void close() throws IOException { + if (!closed) { + closed = true; + // fcontext.base.decref(); // OFF-HEAP + } + } + + @Override + public void process() throws IOException { + super.process(); + + // We need to keep the fcontext open after processing is done (since we will be streaming in the response writer). + // But if the connection is broken, we want to clean up. + // fcontext.base.incref(); // OFF-HEAP + fcontext.qcontext.addCloseHook(this); + + setup(); + response = new SimpleOrderedMap<>(); + response.add("buckets", new Iterator() { + boolean retrieveNext = true; + Object val; + + @Override + public boolean hasNext() { + if (retrieveNext) { + val = nextBucket(); + } + retrieveNext = false; + return val != null; + } + + @Override + public Object next() { + if (retrieveNext) { + val = nextBucket(); + } + retrieveNext = true; + if (val == null) { + // Last value, so clean up. In the case that we are doing streaming facets within streaming facets, + // the number of close hooks could grow very large, so we want to remove ourselves. + boolean removed = fcontext.qcontext.removeCloseHook(FacetFieldProcessorByEnumTermsStream.this); + assert removed; + try { + close(); + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error during facet streaming close", e); + } + } + return val; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }); + } + + private void setup() throws IOException { + + countOnly = freq.facetStats.size() == 0 || freq.facetStats.values().iterator().next() instanceof CountAgg; + hasSubFacets = freq.subFacets.size() > 0; + bucketsToSkip = freq.offset; + + createAccs(-1, 1); + + // Minimum term docFreq in order to use the filterCache for that term. + if (freq.cacheDf == -1) { // -1 means never cache + minDfFilterCache = Integer.MAX_VALUE; + } else if (freq.cacheDf == 0) { // default; compute as fraction of maxDoc + minDfFilterCache = Math.max(fcontext.searcher.maxDoc() >> 4, 3); // (minimum of 3 is for test coverage purposes) + } else { + minDfFilterCache = freq.cacheDf; + } + + docs = fcontext.base; + fastForRandomSet = null; + + if (freq.prefix != null) { + String indexedPrefix = sf.getType().toInternal(freq.prefix); + startTermBytes = new BytesRef(indexedPrefix); + } else if (sf.getType().getNumericType() != null) { + String triePrefix = TrieField.getMainValuePrefix(sf.getType()); + if (triePrefix != null) { + startTermBytes = new BytesRef(triePrefix); + } + } + + Fields fields = fcontext.searcher.getLeafReader().fields(); + Terms terms = fields == null ? null : fields.terms(sf.getName()); + + termsEnum = null; + deState = null; + term = null; + + + if (terms != null) { + + termsEnum = terms.iterator(); + + // TODO: OPT: if seek(ord) is supported for this termsEnum, then we could use it for + // facet.offset when sorting by index order. + + if (startTermBytes != null) { + if (termsEnum.seekCeil(startTermBytes) == TermsEnum.SeekStatus.END) { + termsEnum = null; + } else { + term = termsEnum.term(); + } + } else { + // position termsEnum on first term + term = termsEnum.next(); + } + } + + List leafList = fcontext.searcher.getTopReaderContext().leaves(); + leaves = leafList.toArray( new LeafReaderContext[ leafList.size() ]); + } + + private SimpleOrderedMap nextBucket() { + try { + return _nextBucket(); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error during facet streaming", e); + } + } + + private SimpleOrderedMap _nextBucket() throws IOException { + DocSet termSet = null; + + try { + while (term != null) { + + if (startTermBytes != null && !StringHelper.startsWith(term, startTermBytes)) { + break; + } + + int df = termsEnum.docFreq(); + if (df < effectiveMincount) { + term = termsEnum.next(); + continue; + } + + if (termSet != null) { + // termSet.decref(); // OFF-HEAP + termSet = null; + } + + int c = 0; + + if (hasSubFacets || df >= minDfFilterCache) { + // use the filter cache + + if (deState == null) { + deState = new SolrIndexSearcher.DocsEnumState(); + deState.fieldName = sf.getName(); + deState.liveDocs = fcontext.searcher.getLeafReader().getLiveDocs(); + deState.termsEnum = termsEnum; + deState.postingsEnum = postingsEnum; + deState.minSetSizeCached = minDfFilterCache; + } + + if (hasSubFacets || !countOnly) { + DocSet termsAll = fcontext.searcher.getDocSet(deState); + termSet = docs.intersection(termsAll); + // termsAll.decref(); // OFF-HEAP + c = termSet.size(); + } else { + c = fcontext.searcher.numDocs(docs, deState); + } + postingsEnum = deState.postingsEnum; + + resetStats(); + + if (!countOnly) { + collect(termSet, 0); + } + + } else { + // We don't need the docset here (meaning no sub-facets). + // if countOnly, then we are calculating some other stats... + resetStats(); + + // lazy convert to fastForRandomSet + if (fastForRandomSet == null) { + fastForRandomSet = docs; + if (docs instanceof SortedIntDocSet) { // OFF-HEAP todo: also check for native version + SortedIntDocSet sset = (SortedIntDocSet) docs; + fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size()); + } + } + // iterate over TermDocs to calculate the intersection + postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE); + + if (postingsEnum instanceof MultiPostingsEnum) { + MultiPostingsEnum.EnumWithSlice[] subs = ((MultiPostingsEnum) postingsEnum).getSubs(); + int numSubs = ((MultiPostingsEnum) postingsEnum).getNumSubs(); + for (int subindex = 0; subindex < numSubs; subindex++) { + MultiPostingsEnum.EnumWithSlice sub = subs[subindex]; + if (sub.postingsEnum == null) continue; + int base = sub.slice.start; + int docid; + + if (countOnly) { + while ((docid = sub.postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (fastForRandomSet.exists(docid + base)) c++; + } + } else { + setNextReader(leaves[sub.slice.readerIndex]); + while ((docid = sub.postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (fastForRandomSet.exists(docid + base)) { + c++; + collect(docid, 0); + } + } + } + + } + } else { + int docid; + if (countOnly) { + while ((docid = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (fastForRandomSet.exists(docid)) c++; + } + } else { + setNextReader(leaves[0]); + while ((docid = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (fastForRandomSet.exists(docid)) { + c++; + collect(docid, 0); + } + } + } + } + + } + + if (c < effectiveMincount) { + term = termsEnum.next(); + continue; + } + + // handle offset and limit + if (bucketsToSkip > 0) { + bucketsToSkip--; + term = termsEnum.next(); + continue; + } + + if (freq.limit >= 0 && ++bucketsReturned > freq.limit) { + return null; + } + + // set count in case other stats depend on it + countAcc.incrementCount(0, c); + + // OK, we have a good bucket to return... first get bucket value before moving to next term + Object bucketVal = sf.getType().toObject(sf, term); + TermQuery bucketQuery = hasSubFacets ? new TermQuery(new Term(freq.field, term)) : null; + term = termsEnum.next(); + + SimpleOrderedMap bucket = new SimpleOrderedMap<>(); + bucket.add("val", bucketVal); + addStats(bucket, 0); + if (hasSubFacets) { + processSubs(bucket, bucketQuery, termSet); + } + + // TODO... termSet needs to stick around for streaming sub-facets? + + return bucket; + + } + + } finally { + if (termSet != null) { + // termSet.decref(); // OFF-HEAP + termSet = null; + } + } + + // end of the iteration + return null; + } + +} diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorNumeric.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java similarity index 94% rename from solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorNumeric.java rename to solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java index 6ab4c26cb02..842df202400 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorNumeric.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java @@ -32,10 +32,15 @@ import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocIterator; -class FacetFieldProcessorNumeric extends FacetFieldProcessor { +/** + * Facets numbers into a hash table. + * It currently only works with {@link NumericDocValues} (single-valued). + */ +class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor { static int MAXIMUM_STARTING_TABLE_SIZE=1024; // must be a power of two, non-final to support setting by tests - static class LongCounts { + /** a hash table with long keys (what we're counting) and integer values (counts) */ + private static class LongCounts { static final float LOAD_FACTOR = 0.7f; @@ -55,7 +60,7 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { } /** Current number of slots in the hash table */ - public int numSlots() { + int numSlots() { return vals.length; } @@ -130,69 +135,22 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { } + int allBucketsSlot = -1; - - FacetFieldProcessorNumeric(FacetContext fcontext, FacetField freq, SchemaField sf) { + FacetFieldProcessorByHashNumeric(FacetContext fcontext, FacetField freq, SchemaField sf) { super(fcontext, freq, sf); } - int allBucketsSlot = -1; - @Override public void process() throws IOException { super.process(); response = calcFacets(); } - private void doRehash(LongCounts table) { - if (collectAcc == null && allBucketsAcc == null) return; - - // Our "count" acc is backed by the hash table and will already be rehashed - // otherAccs don't need to be rehashed - - int newTableSize = table.numSlots(); - int numSlots = newTableSize; - final int oldAllBucketsSlot = allBucketsSlot; - if (oldAllBucketsSlot >= 0) { - allBucketsSlot = numSlots++; - } - - final int finalNumSlots = numSlots; - final int[] mapping = table.oldToNewMapping; - - SlotAcc.Resizer resizer = new SlotAcc.Resizer() { - @Override - public int getNewSize() { - return finalNumSlots; - } - - @Override - public int getNewSlot(int oldSlot) { - if (oldSlot < mapping.length) { - return mapping[oldSlot]; - } - if (oldSlot == oldAllBucketsSlot) { - return allBucketsSlot; - } - return -1; - } - }; - - // NOTE: resizing isn't strictly necessary for missing/allBuckets... we could just set the new slot directly - if (collectAcc != null) { - collectAcc.resize(resizer); - } - if (allBucketsAcc != null) { - allBucketsAcc.resize(resizer); - } - } - - public SimpleOrderedMap calcFacets() throws IOException { - + private SimpleOrderedMap calcFacets() throws IOException { final FacetRangeProcessor.Calc calc = FacetRangeProcessor.getNumericCalc(sf); - // TODO: it would be really nice to know the number of unique values!!!! int possibleValues = fcontext.base.size(); @@ -212,7 +170,6 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { int numMissing = 0; - if (freq.allBuckets) { allBucketsSlot = numSlots++; } @@ -325,7 +282,6 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { } } - // // collection done, time to find the top slots // @@ -333,7 +289,7 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { int numBuckets = 0; List bucketVals = null; if (freq.numBuckets && fcontext.isShard()) { - bucketVals = new ArrayList(100); + bucketVals = new ArrayList<>(100); } int off = fcontext.isShard() ? 0 : (int) freq.offset; @@ -378,13 +334,12 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { bottom = queue.insertWithOverflow(bottom); } - - SimpleOrderedMap res = new SimpleOrderedMap(); + SimpleOrderedMap res = new SimpleOrderedMap<>(); if (freq.numBuckets) { if (!fcontext.isShard()) { res.add("numBuckets", numBuckets); } else { - SimpleOrderedMap map = new SimpleOrderedMap(2); + SimpleOrderedMap map = new SimpleOrderedMap<>(2); map.add("numBuckets", numBuckets); map.add("vals", bucketVals); res.add("numBuckets", map); @@ -392,7 +347,7 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { } FacetDebugInfo fdebug = fcontext.getDebugInfo(); - if (fdebug != null) fdebug.putInfoItem("numBuckets", new Long(numBuckets)); + if (fdebug != null) fdebug.putInfoItem("numBuckets", (long) numBuckets); if (freq.allBuckets) { SimpleOrderedMap allBuckets = new SimpleOrderedMap<>(); @@ -419,7 +374,7 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { sortedSlots[i] = queue.pop().slot; } - ArrayList bucketList = new ArrayList(collectCount); + ArrayList bucketList = new ArrayList<>(collectCount); res.add("buckets", bucketList); boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0; @@ -436,8 +391,49 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { bucketList.add(bucket); } - - return res; } + + private void doRehash(LongCounts table) { + if (collectAcc == null && allBucketsAcc == null) return; + + // Our "count" acc is backed by the hash table and will already be rehashed + // otherAccs don't need to be rehashed + + int newTableSize = table.numSlots(); + int numSlots = newTableSize; + final int oldAllBucketsSlot = allBucketsSlot; + if (oldAllBucketsSlot >= 0) { + allBucketsSlot = numSlots++; + } + + final int finalNumSlots = numSlots; + final int[] mapping = table.oldToNewMapping; + + SlotAcc.Resizer resizer = new SlotAcc.Resizer() { + @Override + public int getNewSize() { + return finalNumSlots; + } + + @Override + public int getNewSlot(int oldSlot) { + if (oldSlot < mapping.length) { + return mapping[oldSlot]; + } + if (oldSlot == oldAllBucketsSlot) { + return allBucketsSlot; + } + return -1; + } + }; + + // NOTE: resizing isn't strictly necessary for missing/allBuckets... we could just set the new slot directly + if (collectAcc != null) { + collectAcc.resize(resizer); + } + if (allBucketsAcc != null) { + allBucketsAcc.resize(resizer); + } + } } diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java index b1281f4b23b..fa26319e812 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java @@ -45,27 +45,18 @@ import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SyntaxError; import org.apache.solr.util.RTimer; -public class FacetProcessor { - protected SimpleOrderedMap response; - protected FacetContext fcontext; - protected FacetRequestT freq; +public abstract class FacetProcessor { + SimpleOrderedMap response; + FacetContext fcontext; + FacetRequestT freq; LinkedHashMap accMap; - protected SlotAcc[] accs; - protected CountSlotAcc countAcc; + SlotAcc[] accs; + CountSlotAcc countAcc; - FacetProcessor(FacetContext fcontext, FacetRequestT freq) { - this.fcontext = fcontext; - this.freq = freq; - } - - public void process() throws IOException { - handleDomainChanges(); - } - /** factory method for invoking json facet framework as whole */ - public static FacetProcessor createProcessor(SolrQueryRequest req, - Map params, DocSet docs){ + public static FacetProcessor createProcessor(SolrQueryRequest req, + Map params, DocSet docs){ FacetParser parser = new FacetTopParser(req); FacetRequest facetRequest = null; try { @@ -83,39 +74,25 @@ public class FacetProcessor { return facetRequest.createFacetProcessor(fcontext); } - protected void handleDomainChanges() throws IOException { + FacetProcessor(FacetContext fcontext, FacetRequestT freq) { + this.fcontext = fcontext; + this.freq = freq; + } + + public Object getResponse() { + return response; + } + + public void process() throws IOException { + handleDomainChanges(); + } + + private void handleDomainChanges() throws IOException { if (freq.domain == null) return; handleFilterExclusions(); handleBlockJoin(); } - private void handleBlockJoin() throws IOException { - if (!(freq.domain.toChildren || freq.domain.toParent)) return; - - // TODO: avoid query parsing per-bucket somehow... - String parentStr = freq.domain.parents; - Query parentQuery; - try { - QParser parser = QParser.getParser(parentStr, fcontext.req); - parentQuery = parser.getQuery(); - } catch (SyntaxError err) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing block join parent specification: " + parentStr); - } - - BitDocSet parents = fcontext.searcher.getDocSetBits(parentQuery); - DocSet input = fcontext.base; - DocSet result; - - if (freq.domain.toChildren) { - DocSet filt = fcontext.searcher.getDocSetBits( new MatchAllDocsQuery() ); - result = BlockJoin.toChildren(input, parents, filt, fcontext.qcontext); - } else { - result = BlockJoin.toParents(input, parents, fcontext.qcontext); - } - - fcontext.base = result; - } - private void handleFilterExclusions() throws IOException { List excludeTags = freq.domain.excludeTags; @@ -177,11 +154,44 @@ public class FacetProcessor { fcontext.base = fcontext.searcher.getDocSet(qlist); } + private void handleBlockJoin() throws IOException { + if (!(freq.domain.toChildren || freq.domain.toParent)) return; - public Object getResponse() { - return null; + // TODO: avoid query parsing per-bucket somehow... + String parentStr = freq.domain.parents; + Query parentQuery; + try { + QParser parser = QParser.getParser(parentStr, fcontext.req); + parentQuery = parser.getQuery(); + } catch (SyntaxError err) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing block join parent specification: " + parentStr); + } + + BitDocSet parents = fcontext.searcher.getDocSetBits(parentQuery); + DocSet input = fcontext.base; + DocSet result; + + if (freq.domain.toChildren) { + DocSet filt = fcontext.searcher.getDocSetBits( new MatchAllDocsQuery() ); + result = BlockJoin.toChildren(input, parents, filt, fcontext.qcontext); + } else { + result = BlockJoin.toParents(input, parents, fcontext.qcontext); + } + + fcontext.base = result; } + protected void processStats(SimpleOrderedMap bucket, DocSet docs, int docCount) throws IOException { + if (docCount == 0 && !freq.processEmpty || freq.getFacetStats().size() == 0) { + bucket.add("count", docCount); + return; + } + createAccs(docCount, 1); + int collected = collect(docs, 0); + countAcc.incrementCount(0, collected); + assert collected == docCount; + addStats(bucket, 0); + } protected void createAccs(int docCount, int slotCount) throws IOException { accMap = new LinkedHashMap<>(); @@ -198,7 +208,6 @@ public class FacetProcessor { accMap.put(acc.key, acc); } - accs = new SlotAcc[accMap.size()]; int i=0; for (SlotAcc acc : accMap.values()) { @@ -206,63 +215,14 @@ public class FacetProcessor { } } - - protected void resetStats() { + // note: only called by enum/stream prior to collect + void resetStats() { countAcc.reset(); for (SlotAcc acc : accs) { acc.reset(); } } - protected void processStats(SimpleOrderedMap bucket, DocSet docs, int docCount) throws IOException { - if (docCount == 0 && !freq.processEmpty || freq.getFacetStats().size() == 0) { - bucket.add("count", docCount); - return; - } - createAccs(docCount, 1); - int collected = collect(docs, 0); - countAcc.incrementCount(0, collected); - assert collected == docCount; - addStats(bucket, 0); - } - - - protected void processSubs(SimpleOrderedMap response, Query filter, DocSet domain) throws IOException { - - // TODO: what if a zero bucket has a sub-facet with an exclusion that would yield results? - // should we check for domain-altering exclusions, or even ask the sub-facet for - // it's domain and then only skip it if it's 0? - - if (domain == null || domain.size() == 0 && !freq.processEmpty) { - return; - } - - for (Map.Entry sub : freq.getSubFacets().entrySet()) { - // make a new context for each sub-facet since they can change the domain - FacetContext subContext = fcontext.sub(filter, domain); - FacetProcessor subProcessor = sub.getValue().createFacetProcessor(subContext); - if (fcontext.getDebugInfo() != null) { // if fcontext.debugInfo != null, it means rb.debug() == true - FacetDebugInfo fdebug = new FacetDebugInfo(); - subContext.setDebugInfo(fdebug); - fcontext.getDebugInfo().addChild(fdebug); - - fdebug.setReqDescription(sub.getValue().getFacetDescription()); - fdebug.setProcessor(subProcessor.getClass().getSimpleName()); - if (subContext.filter != null) fdebug.setFilter(subContext.filter.toString()); - - final RTimer timer = new RTimer(); - subProcessor.process(); - long timeElapsed = (long) timer.getTime(); - fdebug.setElapse(timeElapsed); - fdebug.putInfoItem("domainSize", (long)subContext.base.size()); - } else { - subProcessor.process(); - } - - response.add( sub.getKey(), subProcessor.getResponse() ); - } - } - int collect(DocSet docs, int slot) throws IOException { int count = 0; SolrIndexSearcher searcher = fcontext.searcher; @@ -310,7 +270,6 @@ public class FacetProcessor { } } - void addStats(SimpleOrderedMap target, int slotNum) throws IOException { int count = countAcc.getCount(slotNum); target.add("count", count); @@ -321,8 +280,7 @@ public class FacetProcessor { } } - - public void fillBucket(SimpleOrderedMap bucket, Query q, DocSet result) throws IOException { + void fillBucket(SimpleOrderedMap bucket, Query q, DocSet result) throws IOException { boolean needDocSet = freq.getFacetStats().size() > 0 || freq.getSubFacets().size() > 0; // TODO: always collect counts or not??? @@ -348,7 +306,7 @@ public class FacetProcessor { } try { - processStats(bucket, result, (int) count); + processStats(bucket, result, count); processSubs(bucket, q, result); } finally { if (result != null) { @@ -358,7 +316,44 @@ public class FacetProcessor { } } - public static DocSet getFieldMissing(SolrIndexSearcher searcher, DocSet docs, String fieldName) throws IOException { + void processSubs(SimpleOrderedMap response, Query filter, DocSet domain) throws IOException { + + // TODO: what if a zero bucket has a sub-facet with an exclusion that would yield results? + // should we check for domain-altering exclusions, or even ask the sub-facet for + // it's domain and then only skip it if it's 0? + + if (domain == null || domain.size() == 0 && !freq.processEmpty) { + return; + } + + for (Map.Entry sub : freq.getSubFacets().entrySet()) { + // make a new context for each sub-facet since they can change the domain + FacetContext subContext = fcontext.sub(filter, domain); + FacetProcessor subProcessor = sub.getValue().createFacetProcessor(subContext); + if (fcontext.getDebugInfo() != null) { // if fcontext.debugInfo != null, it means rb.debug() == true + FacetDebugInfo fdebug = new FacetDebugInfo(); + subContext.setDebugInfo(fdebug); + fcontext.getDebugInfo().addChild(fdebug); + + fdebug.setReqDescription(sub.getValue().getFacetDescription()); + fdebug.setProcessor(subProcessor.getClass().getSimpleName()); + if (subContext.filter != null) fdebug.setFilter(subContext.filter.toString()); + + final RTimer timer = new RTimer(); + subProcessor.process(); + long timeElapsed = (long) timer.getTime(); + fdebug.setElapse(timeElapsed); + fdebug.putInfoItem("domainSize", (long)subContext.base.size()); + } else { + subProcessor.process(); + } + + response.add( sub.getKey(), subProcessor.getResponse() ); + } + } + + @SuppressWarnings("unused") + static DocSet getFieldMissing(SolrIndexSearcher searcher, DocSet docs, String fieldName) throws IOException { SchemaField sf = searcher.getSchema().getField(fieldName); DocSet hasVal = searcher.getDocSet(sf.getType().getRangeQuery(null, sf, null, null, false, false)); DocSet answer = docs.andNot(hasVal); @@ -366,7 +361,7 @@ public class FacetProcessor { return answer; } - public static Query getFieldMissingQuery(SolrIndexSearcher searcher, String fieldName) throws IOException { + static Query getFieldMissingQuery(SolrIndexSearcher searcher, String fieldName) throws IOException { SchemaField sf = searcher.getSchema().getField(fieldName); Query hasVal = sf.getType().getRangeQuery(null, sf, null, null, false, false); BooleanQuery.Builder noVal = new BooleanQuery.Builder(); diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java b/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java index ac6d7f1c680..174b832f006 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java @@ -53,11 +53,6 @@ class FacetQueryProcessor extends FacetProcessor { super(fcontext, freq); } - @Override - public Object getResponse() { - return response; - } - @Override public void process() throws IOException { super.process(); diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java index 8d3d0f5b9a8..1b98de0ed4b 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java @@ -93,11 +93,6 @@ class FacetRangeProcessor extends FacetProcessor { response = getRangeCounts(); } - @Override - public Object getResponse() { - return response; - } - private static class Range { Object label; Comparable low; diff --git a/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java b/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java index aa8f395915e..ad3baf01bc9 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java +++ b/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java @@ -305,7 +305,7 @@ public class UnInvertedField extends DocTermOrds { - private void getCounts(FacetFieldProcessorUIF processor, CountSlotAcc counts) throws IOException { + private void getCounts(FacetFieldProcessorByArrayUIF processor, CountSlotAcc counts) throws IOException { DocSet docs = processor.fcontext.base; int baseSize = docs.size(); int maxDoc = searcher.maxDoc(); @@ -397,7 +397,7 @@ public class UnInvertedField extends DocTermOrds { - public void collectDocs(FacetFieldProcessorUIF processor) throws IOException { + public void collectDocs(FacetFieldProcessorByArrayUIF processor) throws IOException { if (processor.collectAcc==null && processor.allBucketsAcc == null && processor.startTermIndex == 0 && processor.endTermIndex >= numTermsInField) { getCounts(processor, processor.countAcc); return; @@ -408,7 +408,7 @@ public class UnInvertedField extends DocTermOrds { // called from FieldFacetProcessor // TODO: do a callback version that can be specialized! - public void collectDocsGeneric(FacetFieldProcessorUIF processor) throws IOException { + public void collectDocsGeneric(FacetFieldProcessorByArrayUIF processor) throws IOException { use.incrementAndGet(); int startTermIndex = processor.startTermIndex; diff --git a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java index 46b927efd97..0f85feb13c9 100644 --- a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java +++ b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.regex.Pattern; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.mlt.MoreLikeThis; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -30,7 +31,6 @@ import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrException; import org.apache.solr.common.StringUtils; diff --git a/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java index 962f452e918..da3a4874681 100644 --- a/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java +++ b/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java @@ -16,6 +16,7 @@ */ package org.apache.solr.search.mlt; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.mlt.MoreLikeThis; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -25,7 +26,6 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.solr.common.SolrException; import org.apache.solr.common.StringUtils; import org.apache.solr.common.params.SolrParams; diff --git a/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java b/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java index 7ef495618e2..be08a603b40 100644 --- a/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java +++ b/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java @@ -28,10 +28,10 @@ import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.RamUsageEstimator; @@ -161,8 +161,8 @@ interface FieldCache { }; /** - * A parser instance for int values encoded by {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed - * via {@link org.apache.lucene.document.LegacyIntField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * A parser instance for int values encoded by {@link org.apache.lucene.legacy.LegacyNumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.legacy.LegacyIntField}/{@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * @deprecated Index with points and use {@link #INT_POINT_PARSER} instead. */ @Deprecated @@ -184,8 +184,8 @@ interface FieldCache { }; /** - * A parser instance for float values encoded with {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed - * via {@link org.apache.lucene.document.LegacyFloatField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * A parser instance for float values encoded with {@link org.apache.lucene.legacy.LegacyNumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.legacy.LegacyFloatField}/{@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * @deprecated Index with points and use {@link #FLOAT_POINT_PARSER} instead. */ @Deprecated @@ -209,8 +209,8 @@ interface FieldCache { }; /** - * A parser instance for long values encoded by {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed - * via {@link org.apache.lucene.document.LegacyLongField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * A parser instance for long values encoded by {@link org.apache.lucene.legacy.LegacyNumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.legacy.LegacyLongField}/{@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * @deprecated Index with points and use {@link #LONG_POINT_PARSER} instead. */ @Deprecated @@ -231,8 +231,8 @@ interface FieldCache { }; /** - * A parser instance for double values encoded with {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed - * via {@link org.apache.lucene.document.LegacyDoubleField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * A parser instance for double values encoded with {@link org.apache.lucene.legacy.LegacyNumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.legacy.LegacyDoubleField}/{@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * @deprecated Index with points and use {@link #DOUBLE_POINT_PARSER} instead. */ @Deprecated @@ -279,7 +279,7 @@ interface FieldCache { * @param parser * Computes long for string values. May be {@code null} if the * requested field was indexed as {@link NumericDocValuesField} or - * {@link org.apache.lucene.document.LegacyLongField}. + * {@link org.apache.lucene.legacy.LegacyLongField}. * @param setDocsWithField * If true then {@link #getDocsWithField} will also be computed and * stored in the FieldCache. diff --git a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java index 4450cbb7d86..42b2f7628ba 100644 --- a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java +++ b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java @@ -86,7 +86,7 @@ public class UninvertingReader extends FilterLeafReader { */ DOUBLE_POINT, /** - * Single-valued Integer, (e.g. indexed with {@link org.apache.lucene.document.LegacyIntField}) + * Single-valued Integer, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyIntField}) *

    * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. @@ -95,7 +95,7 @@ public class UninvertingReader extends FilterLeafReader { @Deprecated LEGACY_INTEGER, /** - * Single-valued Long, (e.g. indexed with {@link org.apache.lucene.document.LegacyLongField}) + * Single-valued Long, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyLongField}) *

    * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. @@ -104,7 +104,7 @@ public class UninvertingReader extends FilterLeafReader { @Deprecated LEGACY_LONG, /** - * Single-valued Float, (e.g. indexed with {@link org.apache.lucene.document.LegacyFloatField}) + * Single-valued Float, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyFloatField}) *

    * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. @@ -113,7 +113,7 @@ public class UninvertingReader extends FilterLeafReader { @Deprecated LEGACY_FLOAT, /** - * Single-valued Double, (e.g. indexed with {@link org.apache.lucene.document.LegacyDoubleField}) + * Single-valued Double, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyDoubleField}) *

    * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. @@ -143,28 +143,28 @@ public class UninvertingReader extends FilterLeafReader { */ SORTED_SET_BINARY, /** - * Multi-valued Integer, (e.g. indexed with {@link org.apache.lucene.document.LegacyIntField}) + * Multi-valued Integer, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyIntField}) *

    * Fields with this type act as if they were indexed with * {@link SortedSetDocValuesField}. */ SORTED_SET_INTEGER, /** - * Multi-valued Float, (e.g. indexed with {@link org.apache.lucene.document.LegacyFloatField}) + * Multi-valued Float, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyFloatField}) *

    * Fields with this type act as if they were indexed with * {@link SortedSetDocValuesField}. */ SORTED_SET_FLOAT, /** - * Multi-valued Long, (e.g. indexed with {@link org.apache.lucene.document.LegacyLongField}) + * Multi-valued Long, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyLongField}) *

    * Fields with this type act as if they were indexed with * {@link SortedSetDocValuesField}. */ SORTED_SET_LONG, /** - * Multi-valued Double, (e.g. indexed with {@link org.apache.lucene.document.LegacyDoubleField}) + * Multi-valued Double, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyDoubleField}) *

    * Fields with this type act as if they were indexed with * {@link SortedSetDocValuesField}. diff --git a/solr/core/src/java/org/apache/solr/update/VersionInfo.java b/solr/core/src/java/org/apache/solr/update/VersionInfo.java index bee30f500d8..0003c24358b 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionInfo.java +++ b/solr/core/src/java/org/apache/solr/update/VersionInfo.java @@ -25,12 +25,12 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Terms; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.BitUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.index.SlowCompositeReaderWrapper; diff --git a/solr/core/src/test/org/apache/solr/search/TestMaxScoreQueryParser.java b/solr/core/src/test/org/apache/solr/search/TestMaxScoreQueryParser.java index 32e3eb207c7..6059528d21f 100644 --- a/solr/core/src/test/org/apache/solr/search/TestMaxScoreQueryParser.java +++ b/solr/core/src/test/org/apache/solr/search/TestMaxScoreQueryParser.java @@ -17,6 +17,7 @@ package org.apache.solr.search; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; import org.apache.lucene.search.*; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.util.AbstractSolrTestCase; diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java index 93369bec056..7b5a561168b 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java @@ -47,8 +47,8 @@ public class TestJsonFacets extends SolrTestCaseHS { @BeforeClass public static void beforeTests() throws Exception { JSONTestUtil.failRepeatedKeys = true; - origTableSize = FacetFieldProcessorNumeric.MAXIMUM_STARTING_TABLE_SIZE; - FacetFieldProcessorNumeric.MAXIMUM_STARTING_TABLE_SIZE=2; // stress test resizing + origTableSize = FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE; + FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE=2; // stress test resizing initCore("solrconfig-tlog.xml","schema_latest.xml"); } @@ -61,7 +61,7 @@ public class TestJsonFacets extends SolrTestCaseHS { @AfterClass public static void afterTests() throws Exception { JSONTestUtil.failRepeatedKeys = false; - FacetFieldProcessorNumeric.MAXIMUM_STARTING_TABLE_SIZE=origTableSize; + FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE=origTableSize; if (servers != null) { servers.stop(); servers = null; @@ -349,11 +349,11 @@ public class TestJsonFacets extends SolrTestCaseHS { doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sd", "cat_s","cat_sd", "where_s","where_sd", "num_d","num_dd", "num_i","num_id", "num_is","num_lds", "num_fs","num_dds", "super_s","super_sd", "val_b","val_b", "date","date_dtd", "sparse_s","sparse_sd" ,"multi_ss","multi_sds") ); // multi-valued docvalues - FacetFieldProcessorDV.unwrap_singleValued_multiDv = false; // better multi-valued coverage + FacetFieldProcessorByArrayDV.unwrap_singleValued_multiDv = false; // better multi-valued coverage doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "num_is","num_ids", "num_fs","num_fds", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") ); // multi-valued docvalues - FacetFieldProcessorDV.unwrap_singleValued_multiDv = true; + FacetFieldProcessorByArrayDV.unwrap_singleValued_multiDv = true; doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "num_is","num_ids", "num_fs","num_fds", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") ); } diff --git a/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java b/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java index 785ee55883a..b3a70ae6d39 100644 --- a/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java +++ b/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java @@ -21,8 +21,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.TextField; @@ -31,6 +29,8 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.FloatFieldSource; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java index f1627a6ee35..46339a75ba0 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java @@ -28,8 +28,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValues; @@ -45,10 +43,12 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.TestUtil; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java index d54d5792447..a5958f9af60 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java @@ -21,14 +21,14 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyDoubleField; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.MultiReader; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.index.SlowCompositeReaderWrapper; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSort.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSort.java index 34d92379b39..d53f610f653 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSort.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSort.java @@ -24,13 +24,9 @@ import java.util.Map; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; @@ -41,6 +37,10 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java b/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java index 1192f4b77c8..5220460d811 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java @@ -28,10 +28,6 @@ import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyDoubleField; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; @@ -45,11 +41,15 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.apache.solr.index.SlowCompositeReaderWrapper; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms32.java b/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms32.java index 2b861adb43d..6fed73b829f 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms32.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms32.java @@ -21,12 +21,12 @@ import java.util.Map; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; @@ -62,17 +62,17 @@ public class TestNumericTerms32 extends LuceneTestCase { .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) .setMergePolicy(newLogMergePolicy())); - final FieldType storedInt = new FieldType(LegacyIntField.TYPE_NOT_STORED); + final LegacyFieldType storedInt = new LegacyFieldType(LegacyIntField.TYPE_NOT_STORED); storedInt.setStored(true); storedInt.freeze(); - final FieldType storedInt8 = new FieldType(storedInt); + final LegacyFieldType storedInt8 = new LegacyFieldType(storedInt); storedInt8.setNumericPrecisionStep(8); - final FieldType storedInt4 = new FieldType(storedInt); + final LegacyFieldType storedInt4 = new LegacyFieldType(storedInt); storedInt4.setNumericPrecisionStep(4); - final FieldType storedInt2 = new FieldType(storedInt); + final LegacyFieldType storedInt2 = new LegacyFieldType(storedInt); storedInt2.setNumericPrecisionStep(2); LegacyIntField diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms64.java b/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms64.java index 4da8be98c0f..2f341b708c2 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms64.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms64.java @@ -21,12 +21,12 @@ import java.util.Map; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; @@ -62,20 +62,20 @@ public class TestNumericTerms64 extends LuceneTestCase { .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) .setMergePolicy(newLogMergePolicy())); - final FieldType storedLong = new FieldType(LegacyLongField.TYPE_NOT_STORED); + final LegacyFieldType storedLong = new LegacyFieldType(LegacyLongField.TYPE_NOT_STORED); storedLong.setStored(true); storedLong.freeze(); - final FieldType storedLong8 = new FieldType(storedLong); + final LegacyFieldType storedLong8 = new LegacyFieldType(storedLong); storedLong8.setNumericPrecisionStep(8); - final FieldType storedLong4 = new FieldType(storedLong); + final LegacyFieldType storedLong4 = new LegacyFieldType(storedLong); storedLong4.setNumericPrecisionStep(4); - final FieldType storedLong6 = new FieldType(storedLong); + final LegacyFieldType storedLong6 = new LegacyFieldType(storedLong); storedLong6.setNumericPrecisionStep(6); - final FieldType storedLong2 = new FieldType(storedLong); + final LegacyFieldType storedLong2 = new LegacyFieldType(storedLong); storedLong2.setNumericPrecisionStep(2); LegacyLongField diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java index 2ecc63e6d11..c0188e4358c 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java @@ -28,10 +28,7 @@ import java.util.Set; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; @@ -43,9 +40,12 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.apache.solr.index.SlowCompositeReaderWrapper; @@ -224,7 +224,7 @@ public class TestUninvertingReader extends LuceneTestCase { final Directory dir = newDirectory(); final IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null)); - final FieldType NO_TRIE_TYPE = new FieldType(LegacyIntField.TYPE_NOT_STORED); + final LegacyFieldType NO_TRIE_TYPE = new LegacyFieldType(LegacyIntField.TYPE_NOT_STORED); NO_TRIE_TYPE.setNumericPrecisionStep(Integer.MAX_VALUE); final Map UNINVERT_MAP = new LinkedHashMap(); diff --git a/solr/site/SYSTEM_REQUIREMENTS.mdtext b/solr/site/SYSTEM_REQUIREMENTS.mdtext index 2c0fa87981e..bd41bb217b0 100644 --- a/solr/site/SYSTEM_REQUIREMENTS.mdtext +++ b/solr/site/SYSTEM_REQUIREMENTS.mdtext @@ -1,6 +1,6 @@ # System Requirements -Apache Solr runs of Java 8 or greater. +Apache Solr runs on Java 8 or greater. It is also recommended to always use the latest update version of your Java VM, because bugs may affect Solr. An overview of known JVM bugs diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index 9df4a76aa8e..a3de324e0a7 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -147,7 +147,7 @@ public class ZkStateReader implements Closeable { private class CollectionWatch { int coreRefCount = 0; - Set stateWatchers = new HashSet<>(); + Set stateWatchers = ConcurrentHashMap.newKeySet(); public boolean canBeRemoved() { return coreRefCount + stateWatchers.size() == 0; @@ -1273,10 +1273,14 @@ public class ZkStateReader implements Closeable { /* package-private for testing */ Set getStateWatchers(String collection) { - CollectionWatch watch = collectionWatches.get(collection); - if (watch == null) - return null; - return new HashSet<>(watch.stateWatchers); + final Set watchers = new HashSet<>(); + collectionWatches.compute(collection, (k, v) -> { + if (v != null) { + watchers.addAll(v.stateWatchers); + } + return v; + }); + return watchers; } // returns true if the state has changed diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java b/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java index d959aa83c9b..fca0e35728c 100644 --- a/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java +++ b/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java @@ -19,8 +19,9 @@ package org.apache.solr.common.cloud; import java.lang.invoke.MethodHandles; import java.util.HashMap; -import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -81,6 +82,31 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase { }); } + private static void waitFor(String message, long timeout, TimeUnit unit, Callable predicate) + throws InterruptedException, ExecutionException { + Future future = executor.submit(() -> { + try { + while (true) { + if (predicate.call()) + return true; + TimeUnit.MILLISECONDS.sleep(10); + } + } + catch (InterruptedException e) { + return false; + } + }); + try { + if (future.get(timeout, unit) == true) { + return; + } + } + catch (TimeoutException e) { + // pass failure message on + } + future.cancel(true); + fail(message); + } @Test public void testSimpleCollectionWatch() throws Exception { @@ -113,9 +139,8 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase { cluster.stopJettySolrRunner(random().nextInt(cluster.getJettySolrRunners().size())); assertTrue("CollectionStateWatcher was never notified of cluster change", latch.await(MAX_WAIT_TIMEOUT, TimeUnit.SECONDS)); - Set watchers = client.getZkStateReader().getStateWatchers("testcollection"); - assertTrue("CollectionStateWatcher wasn't cleared after completion", - watchers == null || watchers.size() == 0); + waitFor("CollectionStateWatcher wasn't cleared after completion", 1, TimeUnit.SECONDS, + () -> client.getZkStateReader().getStateWatchers("testcollection").isEmpty()); } @@ -144,8 +169,8 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase { assertTrue("CollectionStateWatcher isn't called when registering for already-watched collection", latch.await(MAX_WAIT_TIMEOUT, TimeUnit.SECONDS)); - assertEquals("CollectionStateWatcher should be removed", - 1, client.getZkStateReader().getStateWatchers("currentstate").size()); + waitFor("CollectionStateWatcher should be removed", 1, TimeUnit.SECONDS, + () -> client.getZkStateReader().getStateWatchers("currentstate").size() == 1); } @Test @@ -189,9 +214,8 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase { expectThrows(TimeoutException.class, () -> { client.waitForState("nosuchcollection", 1, TimeUnit.SECONDS, ((liveNodes, collectionState) -> false)); }); - Set watchers = client.getZkStateReader().getStateWatchers("nosuchcollection"); - assertTrue("Watchers for collection should be removed after timeout", - watchers == null || watchers.size() == 0); + waitFor("Watchers for collection should be removed after timeout", 1, TimeUnit.SECONDS, + () -> client.getZkStateReader().getStateWatchers("nosuchcollection").isEmpty()); } @@ -229,18 +253,17 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase { } @Test - public void testWatcherIsRemovedAfterTimeout() { + public void testWatcherIsRemovedAfterTimeout() throws Exception { CloudSolrClient client = cluster.getSolrClient(); assertTrue("There should be no watchers for a non-existent collection!", - client.getZkStateReader().getStateWatchers("no-such-collection") == null); + client.getZkStateReader().getStateWatchers("no-such-collection").isEmpty()); expectThrows(TimeoutException.class, () -> { client.waitForState("no-such-collection", 10, TimeUnit.MILLISECONDS, (n, c) -> DocCollection.isFullyActive(n, c, 1, 1)); }); - Set watchers = client.getZkStateReader().getStateWatchers("no-such-collection"); - assertTrue("Watchers for collection should be removed after timeout", - watchers == null || watchers.size() == 0); + waitFor("Watchers for collection should be removed after timeout", 1, TimeUnit.SECONDS, + () -> client.getZkStateReader().getStateWatchers("no-such-collection").isEmpty()); } diff --git a/solr/webapp/web/css/angular/index.css b/solr/webapp/web/css/angular/index.css index 5b77a15a866..e07b8d62686 100644 --- a/solr/webapp/web/css/angular/index.css +++ b/solr/webapp/web/css/angular/index.css @@ -110,6 +110,17 @@ limitations under the License. width: 40%; } +#content #index .data +{ + padding-bottom: 12px; + overflow: hidden; +} + +#content #index .data:hover +{ + overflow-x: auto; +} + #content #index .data li { padding-top: 3px; @@ -127,7 +138,6 @@ limitations under the License. { float: right; text-overflow: ellipsis; - overflow: hidden; white-space: nowrap; width: 80% } diff --git a/solr/webapp/web/js/angular/controllers/collections.js b/solr/webapp/web/js/angular/controllers/collections.js index e6229057a1f..2bd6ab601ba 100644 --- a/solr/webapp/web/js/angular/controllers/collections.js +++ b/solr/webapp/web/js/angular/controllers/collections.js @@ -111,8 +111,11 @@ solrAdminApp.controller('CollectionsController', } $scope.createAlias = function() { - var collections = $scope.aliasCollections.join(","); - Collections.createAlias({name: $scope.aliasToCreate, collections: collections}, function(data) { + var collections = []; + for (var i in $scope.aliasCollections) { + collections.push($scope.aliasCollections[i].name); + } + Collections.createAlias({name: $scope.aliasToCreate, collections: collections.join(",")}, function(data) { $scope.hideAll(); }); } diff --git a/solr/webapp/web/js/angular/controllers/cores.js b/solr/webapp/web/js/angular/controllers/cores.js index 347dbf4c1f8..d135395d8f2 100644 --- a/solr/webapp/web/js/angular/controllers/cores.js +++ b/solr/webapp/web/js/angular/controllers/cores.js @@ -17,7 +17,7 @@ // @todo test optimize (delete stuff, watch button appear, test button/form) solrAdminApp.controller('CoreAdminController', - function($scope, $routeParams, $location, $timeout, Cores, Update, Constants){ + function($scope, $routeParams, $location, $timeout, $route, Cores, Update, Constants){ $scope.resetMenu("cores", Constants.IS_ROOT_PAGE); $scope.selectedCore = $routeParams.corename; // use 'corename' not 'core' to distinguish from /solr/:core/ $scope.refresh = function() { @@ -129,15 +129,15 @@ solrAdminApp.controller('CoreAdminController', }; $scope.swapCores = function() { - if ($scope.swapOther) { - $swapMessage = "Please select a core to swap with"; + if (!$scope.swapOther) { + $scope.swapMessage = "Please select a core to swap with"; } else if ($scope.swapOther == $scope.selectedCore) { - $swapMessage = "Cannot swap with the same core"; + $scope.swapMessage = "Cannot swap with the same core"; } else { Cores.swap({core: $scope.selectedCore, other: $scope.swapOther}, function(data) { $location.path("/~cores/" + $scope.swapOther); delete $scope.swapOther; - $scope.cancelSwap(); + $scope.cancelSwapCores(); }); } }; diff --git a/solr/webapp/web/js/angular/controllers/dataimport.js b/solr/webapp/web/js/angular/controllers/dataimport.js index a051ad2178b..d8fbc4fea1c 100644 --- a/solr/webapp/web/js/angular/controllers/dataimport.js +++ b/solr/webapp/web/js/angular/controllers/dataimport.js @@ -39,24 +39,27 @@ solrAdminApp.controller('DataImportController', } }); - DataImport.config({core: $routeParams.core}, function (data) { - try { - var xml = $.parseXML(data.config); - } catch (err) { - $scope.hasHandlers = false; - return; - } - $scope.config = data.config; - $scope.entities = []; - $('document > entity', xml).each(function (i, element) { - $scope.entities.push($(element).attr('name')); + $scope.handler = $routeParams.handler; + if ($scope.handler && $scope.handler[0]=="/") { + $scope.handler = $scope.handler.substr(1); + } + if ($scope.handler) { + DataImport.config({core: $routeParams.core, name: $scope.handler}, function (data) { + try { + $scope.config = data.config; + var xml = $.parseXML(data.config); + $scope.entities = []; + $('document > entity', xml).each(function (i, element) { + $scope.entities.push($(element).attr('name')); + }); + $scope.refreshStatus(); + } catch (err) { + console.log(err); + } }); - }); - + } $scope.lastUpdate = "unknown"; $scope.lastUpdateUTC = ""; - - $scope.refreshStatus(); }; $scope.toggleDebug = function () { @@ -81,7 +84,7 @@ solrAdminApp.controller('DataImportController', } $scope.reload = function () { - DataImport.reload({core: $routeParams.core}, function () { + DataImport.reload({core: $routeParams.core, name: $scope.handler}, function () { $scope.reloaded = true; $timeout(function () { $scope.reloaded = false; @@ -126,6 +129,7 @@ solrAdminApp.controller('DataImportController', } params.core = $routeParams.core; + params.name = $scope.handler; DataImport.post(params, function (data) { $scope.rawResponse = JSON.stringify(data, null, 2); @@ -135,7 +139,7 @@ solrAdminApp.controller('DataImportController', $scope.abort = function () { $scope.isAborting = true; - DataImport.abort({core: $routeParams.core}, function () { + DataImport.abort({core: $routeParams.core, name: $scope.handler}, function () { $timeout(function () { $scope.isAborting = false; $scope.refreshStatus(); @@ -148,7 +152,7 @@ solrAdminApp.controller('DataImportController', console.log("Refresh Status"); $scope.isStatusLoading = true; - DataImport.status({core: $routeParams.core}, function (data) { + DataImport.status({core: $routeParams.core, name: $scope.handler}, function (data) { if (data[0] == "<") { $scope.hasHandlers = false; return; diff --git a/solr/webapp/web/js/angular/controllers/files.js b/solr/webapp/web/js/angular/controllers/files.js index 1cb9e5cd55e..00ea4b19ff7 100644 --- a/solr/webapp/web/js/angular/controllers/files.js +++ b/solr/webapp/web/js/angular/controllers/files.js @@ -16,7 +16,7 @@ */ var contentTypeMap = { xml : 'text/xml', html : 'text/html', js : 'text/javascript', json : 'application/json', 'css' : 'text/css' }; -var languages = {js: "javascript", xml:"xml", xsl:"xml", vm: "xml", html: "xml", json: "text", css: "css"}; +var languages = {js: "javascript", xml:"xml", xsl:"xml", vm: "xml", html: "xml", json: "json", css: "css"}; solrAdminApp.controller('FilesController', function($scope, $rootScope, $routeParams, $location, Files, Constants) { @@ -82,10 +82,10 @@ solrAdminApp.controller('FilesController', Files.get({core: $routeParams.core, file: $scope.file, contentType: contentType}, function(data) { $scope.content = data.data; $scope.url = $scope.baseurl + data.config.url + "?" + $.param(data.config.params); - if (contentType.indexOf("text/plain") && data.data.indexOf("=0) || data.data.indexOf("

    Attribute name