diff --git a/dev-tools/idea/lucene/join/join.iml b/dev-tools/idea/lucene/join/join.iml index 1f9e80b8adb..6de5e90a06e 100644 --- a/dev-tools/idea/lucene/join/join.iml +++ b/dev-tools/idea/lucene/join/join.iml @@ -14,6 +14,7 @@ + diff --git a/dev-tools/idea/lucene/queryparser/queryparser.iml b/dev-tools/idea/lucene/queryparser/queryparser.iml index cd2915fe3a9..86a50a58aab 100644 --- a/dev-tools/idea/lucene/queryparser/queryparser.iml +++ b/dev-tools/idea/lucene/queryparser/queryparser.iml @@ -17,5 +17,6 @@ + diff --git a/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml b/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml index 5694371fd6f..6285d261ee4 100644 --- a/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml +++ b/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml @@ -27,6 +27,7 @@ + \ No newline at end of file diff --git a/dev-tools/idea/solr/contrib/analytics/analytics.iml b/dev-tools/idea/solr/contrib/analytics/analytics.iml index 2ff93365d37..10f51a7ef25 100644 --- a/dev-tools/idea/solr/contrib/analytics/analytics.iml +++ b/dev-tools/idea/solr/contrib/analytics/analytics.iml @@ -20,6 +20,7 @@ + diff --git a/dev-tools/idea/solr/core/src/java/solr-core.iml b/dev-tools/idea/solr/core/src/java/solr-core.iml index 822b24f6cab..6cf1ab175f4 100644 --- a/dev-tools/idea/solr/core/src/java/solr-core.iml +++ b/dev-tools/idea/solr/core/src/java/solr-core.iml @@ -31,5 +31,6 @@ + diff --git a/dev-tools/idea/solr/core/src/solr-core-tests.iml b/dev-tools/idea/solr/core/src/solr-core-tests.iml index 56f768b49f3..99297d0a70d 100644 --- a/dev-tools/idea/solr/core/src/solr-core-tests.iml +++ b/dev-tools/idea/solr/core/src/solr-core-tests.iml @@ -32,5 +32,6 @@ + diff --git a/dev-tools/scripts/addVersion.py b/dev-tools/scripts/addVersion.py index 262e099a166..e95a51f4181 100644 --- a/dev-tools/scripts/addVersion.py +++ b/dev-tools/scripts/addVersion.py @@ -217,7 +217,9 @@ def main(): update_changes('lucene/CHANGES.txt', c.version) update_changes('solr/CHANGES.txt', c.version, get_solr_init_changes()) - if current_version.is_back_compat_with(c.version): + is_back_compat = current_version.major == c.version.major or current_version.is_back_compat_with(c.version) + + if is_back_compat: add_constant(c.version, not c.is_latest_version) else: print('\nNot adding constant for version %s because it is no longer supported' % c.version) @@ -232,7 +234,7 @@ def main(): print('\nTODO: ') print(' - Move backcompat oldIndexes to unsupportedIndexes in TestBackwardsCompatibility') print(' - Update IndexFormatTooOldException throw cases') - elif current_version.is_back_compat_with(c.version): + elif is_back_compat: print('\nTesting changes') check_lucene_version_tests() check_solr_version_tests() diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py index 1deb7985de5..e34c94316d9 100644 --- a/dev-tools/scripts/buildAndPushRelease.py +++ b/dev-tools/scripts/buildAndPushRelease.py @@ -218,7 +218,7 @@ def check_cmdline_tools(): # Fail fast if there are cmdline tool problems if os.system('git --version >/dev/null 2>/dev/null'): raise RuntimeError('"git --version" returned a non-zero exit code.') antVersion = os.popen('ant -version').read().strip() - if not antVersion.startswith('Apache Ant(TM) version 1.8'): + if not antVersion.startswith('Apache Ant(TM) version 1.8') and not antVersion.startswith('Apache Ant(TM) version 1.9'): raise RuntimeError('ant version is not 1.8.X: "%s"' % antVersion) def main(): diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index c5db1438d8b..fbe016bedec 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -30,6 +30,9 @@ Other * LUCENE-7360: Remove Explanation.toHtml() (Alan Woodward) +======================= Lucene 6.3.0 ======================= +(No Changes) + ======================= Lucene 6.2.0 ======================= API Changes @@ -38,6 +41,9 @@ API Changes New Features +* LUCENE-7388: Add point based IntRangeField, FloatRangeField, LongRangeField along with + supporting queries and tests (Nick Knize) + * LUCENE-7381: Add point based DoubleRangeField and RangeFieldQuery for indexing and querying on Ranges up to 4 dimensions (Nick Knize) @@ -85,6 +91,12 @@ Bug Fixes * LUCENE-7391: Fix performance regression in MemoryIndex's fields() introduced in Lucene 6. (Steve Mason via David Smiley) +* SOLR-9413: Fix analysis/kuromoji's CSVUtil.quoteEscape logic, add TestCSVUtil test. + (AppChecker, Christine Poerschke) + +* LUCENE-7419: Fix performance bug with TokenStream.end(), where it would lookup + PositionIncrementAttribute every time. (Mike McCandless, Robert Muir) + Improvements * LUCENE-7323: Compound file writing now verifies the incoming @@ -142,6 +154,13 @@ Improvements because the ICU word-breaking algorithm has some issues. This allows for the previous tokenization used before Lucene 5. (AM, Robert Muir) +* LUCENE-7409: Changed MMapDirectory's unmapping to work safer, but still with + no guarantees. This uses a store-store barrier and yields the current thread + before unmapping to allow in-flight requests to finish. The new code no longer + uses WeakIdentityMap as it delegates all ByteBuffer reads throgh a new + ByteBufferGuard wrapper that is shared between all ByteBufferIndexInput clones. + (Robert Muir, Uwe Schindler) + Optimizations * LUCENE-7330, LUCENE-7339: Speed up conjunction queries. (Adrien Grand) diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java index 6301d2c05ca..04f86038d4a 100644 --- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java +++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java @@ -101,7 +101,7 @@ public final class CSVUtil { String result = original; if (result.indexOf('\"') >= 0) { - result.replace("\"", ESCAPED_QUOTE); + result = result.replace("\"", ESCAPED_QUOTE); } if(result.indexOf(COMMA) >= 0) { result = "\"" + result + "\""; diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestCSVUtil.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestCSVUtil.java new file mode 100644 index 00000000000..01545dbfe36 --- /dev/null +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestCSVUtil.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.analysis.ja; + +import java.io.IOException; + +import org.apache.lucene.analysis.ja.util.CSVUtil; +import org.apache.lucene.util.LuceneTestCase; + +/* + * Tests for the CSVUtil class. + */ +public class TestCSVUtil extends LuceneTestCase { + + public void testQuoteEscapeQuotes() throws IOException { + final String input = "\"Let It Be\" is a song and album by the The Beatles."; + final String expectedOutput = input.replace("\"", "\"\""); + implTestQuoteEscape(input, expectedOutput); + } + + public void testQuoteEscapeComma() throws IOException { + final String input = "To be, or not to be ..."; + final String expectedOutput = '"'+input+'"'; + implTestQuoteEscape(input, expectedOutput); + } + + public void testQuoteEscapeQuotesAndComma() throws IOException { + final String input = "\"To be, or not to be ...\" is a well-known phrase from Shakespeare's Hamlet."; + final String expectedOutput = '"'+input.replace("\"", "\"\"")+'"'; + implTestQuoteEscape(input, expectedOutput); + } + + private void implTestQuoteEscape(String input, String expectedOutput) throws IOException { + final String actualOutput = CSVUtil.quoteEscape(input); + assertEquals(expectedOutput, actualOutput); + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/document/LegacyDoubleField.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyDoubleField.java similarity index 81% rename from lucene/core/src/java/org/apache/lucene/document/LegacyDoubleField.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyDoubleField.java index 55ba81cb120..e98a4f0f567 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LegacyDoubleField.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyDoubleField.java @@ -14,9 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.document; +package org.apache.lucene.legacy; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.index.IndexOptions; @@ -49,7 +51,7 @@ import org.apache.lucene.index.IndexOptions; * LegacyFloatField}. * *

To perform range querying or filtering against a - * LegacyDoubleField, use {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + * LegacyDoubleField, use {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. * To sort according to a * LegacyDoubleField, use the normal numeric sort types, eg * {@link org.apache.lucene.search.SortField.Type#DOUBLE}. LegacyDoubleField @@ -79,11 +81,11 @@ import org.apache.lucene.index.IndexOptions; * but may result in faster range search performance. The * default value, 16, was selected for a reasonable tradeoff * of disk space consumption versus performance. You can - * create a custom {@link FieldType} and invoke the {@link - * FieldType#setNumericPrecisionStep} method if you'd + * create a custom {@link LegacyFieldType} and invoke the {@link + * LegacyFieldType#setNumericPrecisionStep} method if you'd * like to change the value. Note that you must also * specify a congruent value when creating {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. * For low cardinality fields larger precision steps are good. * If the cardinality is < 100, it is fair * to use {@link Integer#MAX_VALUE}, which produces one @@ -91,9 +93,9 @@ import org.apache.lucene.index.IndexOptions; * *

For more information on the internals of numeric trie * indexing, including the precisionStep - * configuration, see {@link org.apache.lucene.search.LegacyNumericRangeQuery}. The format of - * indexed values is described in {@link org.apache.lucene.util.LegacyNumericUtils}. + * href="LegacyNumericRangeQuery.html#precisionStepDesc">precisionStep + * configuration, see {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. The format of + * indexed values is described in {@link org.apache.lucene.legacy.LegacyNumericUtils}. * *

If you only need to sort by numeric value, and never * run range querying/filtering, you can index using a @@ -101,7 +103,7 @@ import org.apache.lucene.index.IndexOptions; * This will minimize disk space consumed.

* *

More advanced users can instead use {@link - * org.apache.lucene.analysis.LegacyNumericTokenStream} directly, when indexing numbers. This + * org.apache.lucene.legacy.LegacyNumericTokenStream} directly, when indexing numbers. This * class is a wrapper around this token stream type for * easier, more intuitive usage.

* @@ -111,18 +113,18 @@ import org.apache.lucene.index.IndexOptions; */ @Deprecated -public final class LegacyDoubleField extends Field { +public final class LegacyDoubleField extends LegacyField { /** * Type for a LegacyDoubleField that is not stored: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_NOT_STORED = new FieldType(); + public static final LegacyFieldType TYPE_NOT_STORED = new LegacyFieldType(); static { TYPE_NOT_STORED.setTokenized(true); TYPE_NOT_STORED.setOmitNorms(true); TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_NOT_STORED.setNumericType(FieldType.LegacyNumericType.DOUBLE); + TYPE_NOT_STORED.setNumericType(LegacyNumericType.DOUBLE); TYPE_NOT_STORED.freeze(); } @@ -130,19 +132,19 @@ public final class LegacyDoubleField extends Field { * Type for a stored LegacyDoubleField: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_STORED = new FieldType(); + public static final LegacyFieldType TYPE_STORED = new LegacyFieldType(); static { TYPE_STORED.setTokenized(true); TYPE_STORED.setOmitNorms(true); TYPE_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_STORED.setNumericType(FieldType.LegacyNumericType.DOUBLE); + TYPE_STORED.setNumericType(LegacyNumericType.DOUBLE); TYPE_STORED.setStored(true); TYPE_STORED.freeze(); } /** Creates a stored or un-stored LegacyDoubleField with the provided value * and default precisionStep {@link - * org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). + * org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). * @param name field name * @param value 64-bit double value * @param stored Store.YES if the content should also be stored @@ -154,17 +156,17 @@ public final class LegacyDoubleField extends Field { } /** Expert: allows you to customize the {@link - * FieldType}. + * LegacyFieldType}. * @param name field name * @param value 64-bit double value - * @param type customized field type: must have {@link FieldType#numericType()} - * of {@link org.apache.lucene.document.FieldType.LegacyNumericType#DOUBLE}. + * @param type customized field type: must have {@link LegacyFieldType#numericType()} + * of {@link LegacyNumericType#DOUBLE}. * @throws IllegalArgumentException if the field name or type is null, or * if the field type does not have a DOUBLE numericType() */ - public LegacyDoubleField(String name, double value, FieldType type) { + public LegacyDoubleField(String name, double value, LegacyFieldType type) { super(name, type); - if (type.numericType() != FieldType.LegacyNumericType.DOUBLE) { + if (type.numericType() != LegacyNumericType.DOUBLE) { throw new IllegalArgumentException("type.numericType() must be DOUBLE but got " + type.numericType()); } fieldsData = Double.valueOf(value); diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyField.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyField.java new file mode 100644 index 00000000000..87ac0e566cf --- /dev/null +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyField.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexOptions; + +/** + * Field extension with support for legacy numerics + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ +@Deprecated +public class LegacyField extends Field { + + /** + * Expert: creates a field with no initial value. + * Intended only for custom LegacyField subclasses. + * @param name field name + * @param type field type + * @throws IllegalArgumentException if either the name or type + * is null. + */ + public LegacyField(String name, LegacyFieldType type) { + super(name, type); + } + + @Override + public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) { + if (fieldType().indexOptions() == IndexOptions.NONE) { + // Not indexed + return null; + } + final LegacyFieldType fieldType = (LegacyFieldType) fieldType(); + final LegacyNumericType numericType = fieldType.numericType(); + if (numericType != null) { + if (!(reuse instanceof LegacyNumericTokenStream && ((LegacyNumericTokenStream)reuse).getPrecisionStep() == fieldType.numericPrecisionStep())) { + // lazy init the TokenStream as it is heavy to instantiate + // (attributes,...) if not needed (stored field loading) + reuse = new LegacyNumericTokenStream(fieldType.numericPrecisionStep()); + } + final LegacyNumericTokenStream nts = (LegacyNumericTokenStream) reuse; + // initialize value in TokenStream + final Number val = (Number) fieldsData; + switch (numericType) { + case INT: + nts.setIntValue(val.intValue()); + break; + case LONG: + nts.setLongValue(val.longValue()); + break; + case FLOAT: + nts.setFloatValue(val.floatValue()); + break; + case DOUBLE: + nts.setDoubleValue(val.doubleValue()); + break; + default: + throw new AssertionError("Should never get here"); + } + return reuse; + } + return super.tokenStream(analyzer, reuse); + } + + @Override + public void setTokenStream(TokenStream tokenStream) { + final LegacyFieldType fieldType = (LegacyFieldType) fieldType(); + if (fieldType.numericType() != null) { + throw new IllegalArgumentException("cannot set private TokenStream on numeric fields"); + } + super.setTokenStream(tokenStream); + } + +} diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFieldType.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFieldType.java new file mode 100644 index 00000000000..1f4b0af4768 --- /dev/null +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFieldType.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; + +/** + * FieldType extension with support for legacy numerics + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ +@Deprecated +public final class LegacyFieldType extends FieldType { + private LegacyNumericType numericType; + private int numericPrecisionStep = LegacyNumericUtils.PRECISION_STEP_DEFAULT; + + /** + * Create a new mutable LegacyFieldType with all of the properties from ref + */ + public LegacyFieldType(LegacyFieldType ref) { + super(ref); + this.numericType = ref.numericType; + this.numericPrecisionStep = ref.numericPrecisionStep; + } + + /** + * Create a new FieldType with default properties. + */ + public LegacyFieldType() { + } + + /** + * Specifies the field's numeric type. + * @param type numeric type, or null if the field has no numeric type. + * @throws IllegalStateException if this FieldType is frozen against + * future modifications. + * @see #numericType() + * + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ + @Deprecated + public void setNumericType(LegacyNumericType type) { + checkIfFrozen(); + numericType = type; + } + + /** + * LegacyNumericType: if non-null then the field's value will be indexed + * numerically so that {@link org.apache.lucene.legacy.LegacyNumericRangeQuery} can be used at + * search time. + *

+ * The default is null (no numeric type) + * @see #setNumericType(LegacyNumericType) + * + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ + @Deprecated + public LegacyNumericType numericType() { + return numericType; + } + + /** + * Sets the numeric precision step for the field. + * @param precisionStep numeric precision step for the field + * @throws IllegalArgumentException if precisionStep is less than 1. + * @throws IllegalStateException if this FieldType is frozen against + * future modifications. + * @see #numericPrecisionStep() + * + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ + @Deprecated + public void setNumericPrecisionStep(int precisionStep) { + checkIfFrozen(); + if (precisionStep < 1) { + throw new IllegalArgumentException("precisionStep must be >= 1 (got " + precisionStep + ")"); + } + this.numericPrecisionStep = precisionStep; + } + + /** + * Precision step for numeric field. + *

+ * This has no effect if {@link #numericType()} returns null. + *

+ * The default is {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} + * @see #setNumericPrecisionStep(int) + * + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ + @Deprecated + public int numericPrecisionStep() { + return numericPrecisionStep; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + numericPrecisionStep; + result = prime * result + ((numericType == null) ? 0 : numericType.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (!super.equals(obj)) { + return false; + } + if (getClass() != obj.getClass()) return false; + LegacyFieldType other = (LegacyFieldType) obj; + if (numericPrecisionStep != other.numericPrecisionStep) return false; + if (numericType != other.numericType) return false; + return true; + } + + /** Prints a Field for human consumption. */ + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append(super.toString()); + if (indexOptions() != IndexOptions.NONE) { + if (result.length() > 0) { + result.append(","); + } + if (numericType != null) { + result.append(",numericType="); + result.append(numericType); + result.append(",numericPrecisionStep="); + result.append(numericPrecisionStep); + } + } + return result.toString(); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/document/LegacyFloatField.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFloatField.java similarity index 80% rename from lucene/core/src/java/org/apache/lucene/document/LegacyFloatField.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFloatField.java index e24bf30fa54..ea3b84ab65f 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LegacyFloatField.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyFloatField.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.document; - +package org.apache.lucene.legacy; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FloatPoint; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.util.LegacyNumericUtils; /** *

@@ -49,7 +49,7 @@ import org.apache.lucene.util.LegacyNumericUtils; * LegacyDoubleField}. * *

To perform range querying or filtering against a - * LegacyFloatField, use {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + * LegacyFloatField, use {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. * To sort according to a * LegacyFloatField, use the normal numeric sort types, eg * {@link org.apache.lucene.search.SortField.Type#FLOAT}. LegacyFloatField @@ -79,11 +79,11 @@ import org.apache.lucene.util.LegacyNumericUtils; * but may result in faster range search performance. The * default value, 8, was selected for a reasonable tradeoff * of disk space consumption versus performance. You can - * create a custom {@link FieldType} and invoke the {@link - * FieldType#setNumericPrecisionStep} method if you'd + * create a custom {@link LegacyFieldType} and invoke the {@link + * LegacyFieldType#setNumericPrecisionStep} method if you'd * like to change the value. Note that you must also * specify a congruent value when creating {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. * For low cardinality fields larger precision steps are good. * If the cardinality is < 100, it is fair * to use {@link Integer#MAX_VALUE}, which produces one @@ -91,9 +91,9 @@ import org.apache.lucene.util.LegacyNumericUtils; * *

For more information on the internals of numeric trie * indexing, including the precisionStep - * configuration, see {@link org.apache.lucene.search.LegacyNumericRangeQuery}. The format of - * indexed values is described in {@link org.apache.lucene.util.LegacyNumericUtils}. + * href="LegacyNumericRangeQuery.html#precisionStepDesc">precisionStep + * configuration, see {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. The format of + * indexed values is described in {@link org.apache.lucene.legacy.LegacyNumericUtils}. * *

If you only need to sort by numeric value, and never * run range querying/filtering, you can index using a @@ -101,7 +101,7 @@ import org.apache.lucene.util.LegacyNumericUtils; * This will minimize disk space consumed.

* *

More advanced users can instead use {@link - * org.apache.lucene.analysis.LegacyNumericTokenStream} directly, when indexing numbers. This + * org.apache.lucene.legacy.LegacyNumericTokenStream} directly, when indexing numbers. This * class is a wrapper around this token stream type for * easier, more intuitive usage.

* @@ -111,18 +111,18 @@ import org.apache.lucene.util.LegacyNumericUtils; */ @Deprecated -public final class LegacyFloatField extends Field { +public final class LegacyFloatField extends LegacyField { /** * Type for a LegacyFloatField that is not stored: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_NOT_STORED = new FieldType(); + public static final LegacyFieldType TYPE_NOT_STORED = new LegacyFieldType(); static { TYPE_NOT_STORED.setTokenized(true); TYPE_NOT_STORED.setOmitNorms(true); TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_NOT_STORED.setNumericType(FieldType.LegacyNumericType.FLOAT); + TYPE_NOT_STORED.setNumericType(LegacyNumericType.FLOAT); TYPE_NOT_STORED.setNumericPrecisionStep(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32); TYPE_NOT_STORED.freeze(); } @@ -131,12 +131,12 @@ public final class LegacyFloatField extends Field { * Type for a stored LegacyFloatField: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_STORED = new FieldType(); + public static final LegacyFieldType TYPE_STORED = new LegacyFieldType(); static { TYPE_STORED.setTokenized(true); TYPE_STORED.setOmitNorms(true); TYPE_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_STORED.setNumericType(FieldType.LegacyNumericType.FLOAT); + TYPE_STORED.setNumericType(LegacyNumericType.FLOAT); TYPE_STORED.setNumericPrecisionStep(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32); TYPE_STORED.setStored(true); TYPE_STORED.freeze(); @@ -144,7 +144,7 @@ public final class LegacyFloatField extends Field { /** Creates a stored or un-stored LegacyFloatField with the provided value * and default precisionStep {@link - * org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). + * org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). * @param name field name * @param value 32-bit double value * @param stored Store.YES if the content should also be stored @@ -156,17 +156,17 @@ public final class LegacyFloatField extends Field { } /** Expert: allows you to customize the {@link - * FieldType}. + * LegacyFieldType}. * @param name field name * @param value 32-bit float value - * @param type customized field type: must have {@link FieldType#numericType()} - * of {@link org.apache.lucene.document.FieldType.LegacyNumericType#FLOAT}. + * @param type customized field type: must have {@link LegacyFieldType#numericType()} + * of {@link LegacyNumericType#FLOAT}. * @throws IllegalArgumentException if the field name or type is null, or * if the field type does not have a FLOAT numericType() */ - public LegacyFloatField(String name, float value, FieldType type) { + public LegacyFloatField(String name, float value, LegacyFieldType type) { super(name, type); - if (type.numericType() != FieldType.LegacyNumericType.FLOAT) { + if (type.numericType() != LegacyNumericType.FLOAT) { throw new IllegalArgumentException("type.numericType() must be FLOAT but got " + type.numericType()); } fieldsData = Float.valueOf(value); diff --git a/lucene/core/src/java/org/apache/lucene/document/LegacyIntField.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyIntField.java similarity index 80% rename from lucene/core/src/java/org/apache/lucene/document/LegacyIntField.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyIntField.java index 6eb0376ee64..e3ae9658b1d 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LegacyIntField.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyIntField.java @@ -14,11 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.document; +package org.apache.lucene.legacy; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.util.LegacyNumericUtils; /** *

@@ -49,7 +50,7 @@ import org.apache.lucene.util.LegacyNumericUtils; * LegacyDoubleField}. * *

To perform range querying or filtering against a - * LegacyIntField, use {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + * LegacyIntField, use {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. * To sort according to a * LegacyIntField, use the normal numeric sort types, eg * {@link org.apache.lucene.search.SortField.Type#INT}. LegacyIntField @@ -79,11 +80,11 @@ import org.apache.lucene.util.LegacyNumericUtils; * but may result in faster range search performance. The * default value, 8, was selected for a reasonable tradeoff * of disk space consumption versus performance. You can - * create a custom {@link FieldType} and invoke the {@link - * FieldType#setNumericPrecisionStep} method if you'd + * create a custom {@link LegacyFieldType} and invoke the {@link + * LegacyFieldType#setNumericPrecisionStep} method if you'd * like to change the value. Note that you must also * specify a congruent value when creating {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. * For low cardinality fields larger precision steps are good. * If the cardinality is < 100, it is fair * to use {@link Integer#MAX_VALUE}, which produces one @@ -91,9 +92,9 @@ import org.apache.lucene.util.LegacyNumericUtils; * *

For more information on the internals of numeric trie * indexing, including the precisionStep - * configuration, see {@link org.apache.lucene.search.LegacyNumericRangeQuery}. The format of - * indexed values is described in {@link org.apache.lucene.util.LegacyNumericUtils}. + * href="LegacyNumericRangeQuery.html#precisionStepDesc">precisionStep + * configuration, see {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. The format of + * indexed values is described in {@link org.apache.lucene.legacy.LegacyNumericUtils}. * *

If you only need to sort by numeric value, and never * run range querying/filtering, you can index using a @@ -101,7 +102,7 @@ import org.apache.lucene.util.LegacyNumericUtils; * This will minimize disk space consumed.

* *

More advanced users can instead use {@link - * org.apache.lucene.analysis.LegacyNumericTokenStream} directly, when indexing numbers. This + * org.apache.lucene.legacy.LegacyNumericTokenStream} directly, when indexing numbers. This * class is a wrapper around this token stream type for * easier, more intuitive usage.

* @@ -111,18 +112,18 @@ import org.apache.lucene.util.LegacyNumericUtils; */ @Deprecated -public final class LegacyIntField extends Field { +public final class LegacyIntField extends LegacyField { /** * Type for an LegacyIntField that is not stored: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_NOT_STORED = new FieldType(); + public static final LegacyFieldType TYPE_NOT_STORED = new LegacyFieldType(); static { TYPE_NOT_STORED.setTokenized(true); TYPE_NOT_STORED.setOmitNorms(true); TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_NOT_STORED.setNumericType(FieldType.LegacyNumericType.INT); + TYPE_NOT_STORED.setNumericType(LegacyNumericType.INT); TYPE_NOT_STORED.setNumericPrecisionStep(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32); TYPE_NOT_STORED.freeze(); } @@ -131,12 +132,12 @@ public final class LegacyIntField extends Field { * Type for a stored LegacyIntField: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_STORED = new FieldType(); + public static final LegacyFieldType TYPE_STORED = new LegacyFieldType(); static { TYPE_STORED.setTokenized(true); TYPE_STORED.setOmitNorms(true); TYPE_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_STORED.setNumericType(FieldType.LegacyNumericType.INT); + TYPE_STORED.setNumericType(LegacyNumericType.INT); TYPE_STORED.setNumericPrecisionStep(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32); TYPE_STORED.setStored(true); TYPE_STORED.freeze(); @@ -144,7 +145,7 @@ public final class LegacyIntField extends Field { /** Creates a stored or un-stored LegacyIntField with the provided value * and default precisionStep {@link - * org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). + * org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). * @param name field name * @param value 32-bit integer value * @param stored Store.YES if the content should also be stored @@ -156,17 +157,17 @@ public final class LegacyIntField extends Field { } /** Expert: allows you to customize the {@link - * FieldType}. + * LegacyFieldType}. * @param name field name * @param value 32-bit integer value - * @param type customized field type: must have {@link FieldType#numericType()} - * of {@link org.apache.lucene.document.FieldType.LegacyNumericType#INT}. + * @param type customized field type: must have {@link LegacyFieldType#numericType()} + * of {@link LegacyNumericType#INT}. * @throws IllegalArgumentException if the field name or type is null, or * if the field type does not have a INT numericType() */ - public LegacyIntField(String name, int value, FieldType type) { + public LegacyIntField(String name, int value, LegacyFieldType type) { super(name, type); - if (type.numericType() != FieldType.LegacyNumericType.INT) { + if (type.numericType() != LegacyNumericType.INT) { throw new IllegalArgumentException("type.numericType() must be INT but got " + type.numericType()); } fieldsData = Integer.valueOf(value); diff --git a/lucene/core/src/java/org/apache/lucene/document/LegacyLongField.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyLongField.java similarity index 81% rename from lucene/core/src/java/org/apache/lucene/document/LegacyLongField.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyLongField.java index fa1851fe7e6..3e20b448b96 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LegacyLongField.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyLongField.java @@ -14,9 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.document; +package org.apache.lucene.legacy; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.IndexOptions; @@ -59,7 +61,7 @@ import org.apache.lucene.index.IndexOptions; * long value. * *

To perform range querying or filtering against a - * LegacyLongField, use {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + * LegacyLongField, use {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. * To sort according to a * LegacyLongField, use the normal numeric sort types, eg * {@link org.apache.lucene.search.SortField.Type#LONG}. LegacyLongField @@ -89,11 +91,11 @@ import org.apache.lucene.index.IndexOptions; * but may result in faster range search performance. The * default value, 16, was selected for a reasonable tradeoff * of disk space consumption versus performance. You can - * create a custom {@link FieldType} and invoke the {@link - * FieldType#setNumericPrecisionStep} method if you'd + * create a custom {@link LegacyFieldType} and invoke the {@link + * LegacyFieldType#setNumericPrecisionStep} method if you'd * like to change the value. Note that you must also * specify a congruent value when creating {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. * For low cardinality fields larger precision steps are good. * If the cardinality is < 100, it is fair * to use {@link Integer#MAX_VALUE}, which produces one @@ -101,9 +103,9 @@ import org.apache.lucene.index.IndexOptions; * *

For more information on the internals of numeric trie * indexing, including the precisionStep - * configuration, see {@link org.apache.lucene.search.LegacyNumericRangeQuery}. The format of - * indexed values is described in {@link org.apache.lucene.util.LegacyNumericUtils}. + * href="LegacyNumericRangeQuery.html#precisionStepDesc">precisionStep + * configuration, see {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. The format of + * indexed values is described in {@link org.apache.lucene.legacy.LegacyNumericUtils}. * *

If you only need to sort by numeric value, and never * run range querying/filtering, you can index using a @@ -111,7 +113,7 @@ import org.apache.lucene.index.IndexOptions; * This will minimize disk space consumed. * *

More advanced users can instead use {@link - * org.apache.lucene.analysis.LegacyNumericTokenStream} directly, when indexing numbers. This + * org.apache.lucene.legacy.LegacyNumericTokenStream} directly, when indexing numbers. This * class is a wrapper around this token stream type for * easier, more intuitive usage.

* @@ -121,18 +123,18 @@ import org.apache.lucene.index.IndexOptions; */ @Deprecated -public final class LegacyLongField extends Field { +public final class LegacyLongField extends LegacyField { /** * Type for a LegacyLongField that is not stored: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_NOT_STORED = new FieldType(); + public static final LegacyFieldType TYPE_NOT_STORED = new LegacyFieldType(); static { TYPE_NOT_STORED.setTokenized(true); TYPE_NOT_STORED.setOmitNorms(true); TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_NOT_STORED.setNumericType(FieldType.LegacyNumericType.LONG); + TYPE_NOT_STORED.setNumericType(LegacyNumericType.LONG); TYPE_NOT_STORED.freeze(); } @@ -140,19 +142,19 @@ public final class LegacyLongField extends Field { * Type for a stored LegacyLongField: * normalization factors, frequencies, and positions are omitted. */ - public static final FieldType TYPE_STORED = new FieldType(); + public static final LegacyFieldType TYPE_STORED = new LegacyFieldType(); static { TYPE_STORED.setTokenized(true); TYPE_STORED.setOmitNorms(true); TYPE_STORED.setIndexOptions(IndexOptions.DOCS); - TYPE_STORED.setNumericType(FieldType.LegacyNumericType.LONG); + TYPE_STORED.setNumericType(LegacyNumericType.LONG); TYPE_STORED.setStored(true); TYPE_STORED.freeze(); } /** Creates a stored or un-stored LegacyLongField with the provided value * and default precisionStep {@link - * org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). + * org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). * @param name field name * @param value 64-bit long value * @param stored Store.YES if the content should also be stored @@ -164,17 +166,17 @@ public final class LegacyLongField extends Field { } /** Expert: allows you to customize the {@link - * FieldType}. + * LegacyFieldType}. * @param name field name * @param value 64-bit long value - * @param type customized field type: must have {@link FieldType#numericType()} - * of {@link org.apache.lucene.document.FieldType.LegacyNumericType#LONG}. + * @param type customized field type: must have {@link LegacyFieldType#numericType()} + * of {@link LegacyNumericType#LONG}. * @throws IllegalArgumentException if the field name or type is null, or * if the field type does not have a LONG numericType() */ - public LegacyLongField(String name, long value, FieldType type) { + public LegacyLongField(String name, long value, LegacyFieldType type) { super(name, type); - if (type.numericType() != FieldType.LegacyNumericType.LONG) { + if (type.numericType() != LegacyNumericType.LONG) { throw new IllegalArgumentException("type.numericType() must be LONG but got " + type.numericType()); } fieldsData = Long.valueOf(value); diff --git a/lucene/core/src/java/org/apache/lucene/search/LegacyNumericRangeQuery.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericRangeQuery.java similarity index 89% rename from lucene/core/src/java/org/apache/lucene/search/LegacyNumericRangeQuery.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericRangeQuery.java index fe6c9e24864..f172a200779 100644 --- a/lucene/core/src/java/org/apache/lucene/search/LegacyNumericRangeQuery.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericRangeQuery.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.search; +package org.apache.lucene.legacy; import java.io.IOException; @@ -22,8 +22,6 @@ import java.util.LinkedList; import java.util.Objects; import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FieldType.LegacyNumericType; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; @@ -31,18 +29,21 @@ import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.PointValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.index.Term; // for javadocs /** *

A {@link Query} that matches numeric values within a * specified range. To use this, you must first index the - * numeric values using {@link org.apache.lucene.document.LegacyIntField}, {@link - * org.apache.lucene.document.LegacyFloatField}, {@link org.apache.lucene.document.LegacyLongField} or {@link org.apache.lucene.document.LegacyDoubleField} (expert: {@link - * org.apache.lucene.analysis.LegacyNumericTokenStream}). If your terms are instead textual, + * numeric values using {@link org.apache.lucene.legacy.LegacyIntField}, {@link + * org.apache.lucene.legacy.LegacyFloatField}, {@link org.apache.lucene.legacy.LegacyLongField} or {@link org.apache.lucene.legacy.LegacyDoubleField} (expert: {@link + * org.apache.lucene.legacy.LegacyNumericTokenStream}). If your terms are instead textual, * you should use {@link TermRangeQuery}.

* *

You create a new LegacyNumericRangeQuery with the static @@ -96,7 +97,7 @@ import org.apache.lucene.index.Term; // for javadocs * (all numerical values like doubles, longs, floats, and ints are converted to * lexicographic sortable string representations and stored with different precisions * (for a more detailed description of how the values are stored, - * see {@link org.apache.lucene.util.LegacyNumericUtils}). A range is then divided recursively into multiple intervals for searching: + * see {@link org.apache.lucene.legacy.LegacyNumericUtils}). A range is then divided recursively into multiple intervals for searching: * The center of the range is searched only with the lowest possible precision in the trie, * while the boundaries are matched more exactly. This reduces the number of terms dramatically.

* @@ -112,7 +113,7 @@ import org.apache.lucene.index.Term; // for javadocs *

Precision Step

*

You can choose any precisionStep when encoding values. * Lower step values mean more precisions and so more terms in index (and index gets larger). The number - * of indexed terms per value is (those are generated by {@link org.apache.lucene.analysis.LegacyNumericTokenStream}): + * of indexed terms per value is (those are generated by {@link org.apache.lucene.legacy.LegacyNumericTokenStream}): *

*   indexedTermsPerValue = ceil(bitsPerValue / precisionStep) *

@@ -148,8 +149,8 @@ import org.apache.lucene.index.Term; // for javadocs *
  • Steps ≥64 for long/double and ≥32 for int/float produces one token * per value in the index and querying is as slow as a conventional {@link TermRangeQuery}. But it can be used * to produce fields, that are solely used for sorting (in this case simply use {@link Integer#MAX_VALUE} as - * precisionStep). Using {@link org.apache.lucene.document.LegacyIntField}, - * {@link org.apache.lucene.document.LegacyLongField}, {@link org.apache.lucene.document.LegacyFloatField} or {@link org.apache.lucene.document.LegacyDoubleField} for sorting + * precisionStep). Using {@link org.apache.lucene.legacy.LegacyIntField}, + * {@link org.apache.lucene.legacy.LegacyLongField}, {@link org.apache.lucene.legacy.LegacyFloatField} or {@link org.apache.lucene.legacy.LegacyDoubleField} for sorting * is ideal, because building the field cache is much faster than with text-only numbers. * These fields have one term per value and therefore also work with term enumeration for building distinct lists * (e.g. facets / preselected values to search for). @@ -199,12 +200,12 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newLongRange(final String field, final int precisionStep, Long min, Long max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, precisionStep, FieldType.LegacyNumericType.LONG, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, precisionStep, LegacyNumericType.LONG, min, max, minInclusive, maxInclusive); } /** * Factory that creates a LegacyNumericRangeQuery, that queries a long - * range using the default precisionStep {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). + * range using the default precisionStep {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). * You can have half-open ranges (which are in fact </≤ or >/≥ queries) * by setting the min or max value to null. By setting inclusive to false, it will * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. @@ -212,7 +213,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newLongRange(final String field, Long min, Long max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT, FieldType.LegacyNumericType.LONG, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT, LegacyNumericType.LONG, min, max, minInclusive, maxInclusive); } /** @@ -225,12 +226,12 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newIntRange(final String field, final int precisionStep, Integer min, Integer max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, precisionStep, FieldType.LegacyNumericType.INT, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, precisionStep, LegacyNumericType.INT, min, max, minInclusive, maxInclusive); } /** * Factory that creates a LegacyNumericRangeQuery, that queries a int - * range using the default precisionStep {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). + * range using the default precisionStep {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). * You can have half-open ranges (which are in fact </≤ or >/≥ queries) * by setting the min or max value to null. By setting inclusive to false, it will * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. @@ -238,7 +239,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newIntRange(final String field, Integer min, Integer max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, FieldType.LegacyNumericType.INT, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, LegacyNumericType.INT, min, max, minInclusive, maxInclusive); } /** @@ -253,12 +254,12 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newDoubleRange(final String field, final int precisionStep, Double min, Double max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, precisionStep, FieldType.LegacyNumericType.DOUBLE, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, precisionStep, LegacyNumericType.DOUBLE, min, max, minInclusive, maxInclusive); } /** * Factory that creates a LegacyNumericRangeQuery, that queries a double - * range using the default precisionStep {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). + * range using the default precisionStep {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). * You can have half-open ranges (which are in fact </≤ or >/≥ queries) * by setting the min or max value to null. * {@link Double#NaN} will never match a half-open range, to hit {@code NaN} use a query @@ -268,7 +269,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newDoubleRange(final String field, Double min, Double max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT, FieldType.LegacyNumericType.DOUBLE, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT, LegacyNumericType.DOUBLE, min, max, minInclusive, maxInclusive); } /** @@ -283,12 +284,12 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newFloatRange(final String field, final int precisionStep, Float min, Float max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, precisionStep, FieldType.LegacyNumericType.FLOAT, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, precisionStep, LegacyNumericType.FLOAT, min, max, minInclusive, maxInclusive); } /** * Factory that creates a LegacyNumericRangeQuery, that queries a float - * range using the default precisionStep {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). + * range using the default precisionStep {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT_32} (8). * You can have half-open ranges (which are in fact </≤ or >/≥ queries) * by setting the min or max value to null. * {@link Float#NaN} will never match a half-open range, to hit {@code NaN} use a query @@ -298,7 +299,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu public static LegacyNumericRangeQuery newFloatRange(final String field, Float min, Float max, final boolean minInclusive, final boolean maxInclusive ) { - return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, FieldType.LegacyNumericType.FLOAT, min, max, minInclusive, maxInclusive); + return new LegacyNumericRangeQuery<>(field, LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, LegacyNumericType.FLOAT, min, max, minInclusive, maxInclusive); } @Override @SuppressWarnings("unchecked") @@ -369,7 +370,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu // members (package private, to be also fast accessible by NumericRangeTermEnum) final int precisionStep; - final FieldType.LegacyNumericType dataType; + final LegacyNumericType dataType; final T min, max; final boolean minInclusive,maxInclusive; @@ -389,8 +390,8 @@ public final class LegacyNumericRangeQuery extends MultiTermQu *

    * WARNING: This term enumeration is not guaranteed to be always ordered by * {@link Term#compareTo}. - * The ordering depends on how {@link org.apache.lucene.util.LegacyNumericUtils#splitLongRange} and - * {@link org.apache.lucene.util.LegacyNumericUtils#splitIntRange} generates the sub-ranges. For + * The ordering depends on how {@link org.apache.lucene.legacy.LegacyNumericUtils#splitLongRange} and + * {@link org.apache.lucene.legacy.LegacyNumericUtils#splitIntRange} generates the sub-ranges. For * {@link MultiTermQuery} ordering is not relevant. */ private final class NumericRangeTermsEnum extends FilteredTermsEnum { @@ -406,10 +407,10 @@ public final class LegacyNumericRangeQuery extends MultiTermQu case DOUBLE: { // lower long minBound; - if (dataType == FieldType.LegacyNumericType.LONG) { + if (dataType == LegacyNumericType.LONG) { minBound = (min == null) ? Long.MIN_VALUE : min.longValue(); } else { - assert dataType == FieldType.LegacyNumericType.DOUBLE; + assert dataType == LegacyNumericType.DOUBLE; minBound = (min == null) ? LONG_NEGATIVE_INFINITY : NumericUtils.doubleToSortableLong(min.doubleValue()); } @@ -420,10 +421,10 @@ public final class LegacyNumericRangeQuery extends MultiTermQu // upper long maxBound; - if (dataType == FieldType.LegacyNumericType.LONG) { + if (dataType == LegacyNumericType.LONG) { maxBound = (max == null) ? Long.MAX_VALUE : max.longValue(); } else { - assert dataType == FieldType.LegacyNumericType.DOUBLE; + assert dataType == LegacyNumericType.DOUBLE; maxBound = (max == null) ? LONG_POSITIVE_INFINITY : NumericUtils.doubleToSortableLong(max.doubleValue()); } @@ -446,10 +447,10 @@ public final class LegacyNumericRangeQuery extends MultiTermQu case FLOAT: { // lower int minBound; - if (dataType == FieldType.LegacyNumericType.INT) { + if (dataType == LegacyNumericType.INT) { minBound = (min == null) ? Integer.MIN_VALUE : min.intValue(); } else { - assert dataType == FieldType.LegacyNumericType.FLOAT; + assert dataType == LegacyNumericType.FLOAT; minBound = (min == null) ? INT_NEGATIVE_INFINITY : NumericUtils.floatToSortableInt(min.floatValue()); } @@ -463,7 +464,7 @@ public final class LegacyNumericRangeQuery extends MultiTermQu if (dataType == LegacyNumericType.INT) { maxBound = (max == null) ? Integer.MAX_VALUE : max.intValue(); } else { - assert dataType == FieldType.LegacyNumericType.FLOAT; + assert dataType == LegacyNumericType.FLOAT; maxBound = (max == null) ? INT_POSITIVE_INFINITY : NumericUtils.floatToSortableInt(max.floatValue()); } diff --git a/lucene/core/src/java/org/apache/lucene/analysis/LegacyNumericTokenStream.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericTokenStream.java similarity index 94% rename from lucene/core/src/java/org/apache/lucene/analysis/LegacyNumericTokenStream.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericTokenStream.java index 19f7d37e31f..a2aba19e2ac 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/LegacyNumericTokenStream.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericTokenStream.java @@ -14,11 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.analysis; +package org.apache.lucene.legacy; import java.util.Objects; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; @@ -29,16 +30,15 @@ import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.AttributeReflector; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; /** * Expert: This class provides a {@link TokenStream} * for indexing numeric values that can be used by {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. * - *

    Note that for simple usage, {@link org.apache.lucene.document.LegacyIntField}, {@link - * org.apache.lucene.document.LegacyLongField}, {@link org.apache.lucene.document.LegacyFloatField} or {@link org.apache.lucene.document.LegacyDoubleField} is + *

    Note that for simple usage, {@link org.apache.lucene.legacy.LegacyIntField}, {@link + * org.apache.lucene.legacy.LegacyLongField}, {@link org.apache.lucene.legacy.LegacyFloatField} or {@link org.apache.lucene.legacy.LegacyDoubleField} is * recommended. These fields disable norms and * term freqs, as they are not usually needed during * searching. If you need to change these settings, you @@ -81,9 +81,9 @@ import org.apache.lucene.util.NumericUtils; * than one numeric field, use a separate LegacyNumericTokenStream * instance for each.

    * - *

    See {@link org.apache.lucene.search.LegacyNumericRangeQuery} for more details on the + *

    See {@link org.apache.lucene.legacy.LegacyNumericRangeQuery} for more details on the * precisionStep + * href="LegacyNumericRangeQuery.html#precisionStepDesc">precisionStep * parameter as well as how numeric fields work under the hood.

    * * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead @@ -140,7 +140,7 @@ public final class LegacyNumericTokenStream extends TokenStream { } } - /** Implementation of {@link org.apache.lucene.analysis.LegacyNumericTokenStream.LegacyNumericTermAttribute}. + /** Implementation of {@link org.apache.lucene.legacy.LegacyNumericTokenStream.LegacyNumericTermAttribute}. * @lucene.internal * @since 4.0 */ @@ -240,7 +240,7 @@ public final class LegacyNumericTokenStream extends TokenStream { /** * Creates a token stream for numeric values using the default precisionStep - * {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). The stream is not yet initialized, + * {@link org.apache.lucene.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). The stream is not yet initialized, * before using set a value using the various set???Value() methods. */ public LegacyNumericTokenStream() { diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericType.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericType.java new file mode 100644 index 00000000000..345b4974b02 --- /dev/null +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericType.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + +/** Data type of the numeric value + * @since 3.2 + * + * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead + */ +@Deprecated +public enum LegacyNumericType { + /** 32-bit integer numeric type */ + INT, + /** 64-bit long numeric type */ + LONG, + /** 32-bit float numeric type */ + FLOAT, + /** 64-bit double numeric type */ + DOUBLE +} diff --git a/lucene/core/src/java/org/apache/lucene/util/LegacyNumericUtils.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericUtils.java similarity index 95% rename from lucene/core/src/java/org/apache/lucene/util/LegacyNumericUtils.java rename to lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericUtils.java index 9a26bfa3f2b..e6659d7e102 100644 --- a/lucene/core/src/java/org/apache/lucene/util/LegacyNumericUtils.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/LegacyNumericUtils.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.util; +package org.apache.lucene.legacy; import java.io.IOException; @@ -23,6 +23,8 @@ import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; /** * This is a helper class to generate prefix-encoded representations for numerical values @@ -41,9 +43,9 @@ import org.apache.lucene.index.TermsEnum; * during encoding. * *

    For easy usage, the trie algorithm is implemented for indexing inside - * {@link org.apache.lucene.analysis.LegacyNumericTokenStream} that can index int, long, + * {@link org.apache.lucene.legacy.LegacyNumericTokenStream} that can index int, long, * float, and double. For querying, - * {@link org.apache.lucene.search.LegacyNumericRangeQuery} implements the query part + * {@link org.apache.lucene.legacy.LegacyNumericRangeQuery} implements the query part * for the same data types. * * @lucene.internal @@ -59,15 +61,15 @@ public final class LegacyNumericUtils { private LegacyNumericUtils() {} // no instance! /** - * The default precision step used by {@link org.apache.lucene.document.LegacyLongField}, - * {@link org.apache.lucene.document.LegacyDoubleField}, {@link org.apache.lucene.analysis.LegacyNumericTokenStream}, {@link - * org.apache.lucene.search.LegacyNumericRangeQuery}. + * The default precision step used by {@link org.apache.lucene.legacy.LegacyLongField}, + * {@link org.apache.lucene.legacy.LegacyDoubleField}, {@link org.apache.lucene.legacy.LegacyNumericTokenStream}, {@link + * org.apache.lucene.legacy.LegacyNumericRangeQuery}. */ public static final int PRECISION_STEP_DEFAULT = 16; /** - * The default precision step used by {@link org.apache.lucene.document.LegacyIntField} and - * {@link org.apache.lucene.document.LegacyFloatField}. + * The default precision step used by {@link org.apache.lucene.legacy.LegacyIntField} and + * {@link org.apache.lucene.legacy.LegacyFloatField}. */ public static final int PRECISION_STEP_DEFAULT_32 = 8; @@ -99,7 +101,7 @@ public final class LegacyNumericUtils { /** * Returns prefix coded bits after reducing the precision by shift bits. - * This is method is used by {@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * This is method is used by {@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * After encoding, {@code bytes.offset} will always be 0. * @param val the numeric value * @param shift how many bits to strip from the right @@ -126,7 +128,7 @@ public final class LegacyNumericUtils { /** * Returns prefix coded bits after reducing the precision by shift bits. - * This is method is used by {@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * This is method is used by {@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * After encoding, {@code bytes.offset} will always be 0. * @param val the numeric value * @param shift how many bits to strip from the right @@ -230,7 +232,7 @@ public final class LegacyNumericUtils { * {@link org.apache.lucene.search.BooleanQuery} for each call to its * {@link LongRangeBuilder#addRange(BytesRef,BytesRef)} * method. - *

    This method is used by {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + *

    This method is used by {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. */ public static void splitLongRange(final LongRangeBuilder builder, final int precisionStep, final long minBound, final long maxBound @@ -244,7 +246,7 @@ public final class LegacyNumericUtils { * {@link org.apache.lucene.search.BooleanQuery} for each call to its * {@link IntRangeBuilder#addRange(BytesRef,BytesRef)} * method. - *

    This method is used by {@link org.apache.lucene.search.LegacyNumericRangeQuery}. + *

    This method is used by {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. */ public static void splitIntRange(final IntRangeBuilder builder, final int precisionStep, final int minBound, final int maxBound diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/legacy/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/package-info.java new file mode 100644 index 00000000000..d0167f80023 --- /dev/null +++ b/lucene/backward-codecs/src/java/org/apache/lucene/legacy/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Deprecated stuff! + */ +package org.apache.lucene.legacy; diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index 8226022e6d8..03480d779ed 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -47,8 +47,6 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FloatDocValuesField; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; @@ -57,9 +55,12 @@ import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.BaseDirectoryWrapper; @@ -72,7 +73,6 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.InfoStream; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java new file mode 100644 index 00000000000..65ff0969d9d --- /dev/null +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyField.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + +import java.io.StringReader; + +import org.apache.lucene.analysis.CannedTokenStream; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.document.Field; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; + +public class TestLegacyField extends LuceneTestCase { + + public void testLegacyDoubleField() throws Exception { + Field fields[] = new Field[] { + new LegacyDoubleField("foo", 5d, Field.Store.NO), + new LegacyDoubleField("foo", 5d, Field.Store.YES) + }; + + for (Field field : fields) { + trySetBoost(field); + trySetByteValue(field); + trySetBytesValue(field); + trySetBytesRefValue(field); + field.setDoubleValue(6d); // ok + trySetIntValue(field); + trySetFloatValue(field); + trySetLongValue(field); + trySetReaderValue(field); + trySetShortValue(field); + trySetStringValue(field); + trySetTokenStreamValue(field); + + assertEquals(6d, field.numericValue().doubleValue(), 0.0d); + } + } + + public void testLegacyFloatField() throws Exception { + Field fields[] = new Field[] { + new LegacyFloatField("foo", 5f, Field.Store.NO), + new LegacyFloatField("foo", 5f, Field.Store.YES) + }; + + for (Field field : fields) { + trySetBoost(field); + trySetByteValue(field); + trySetBytesValue(field); + trySetBytesRefValue(field); + trySetDoubleValue(field); + trySetIntValue(field); + field.setFloatValue(6f); // ok + trySetLongValue(field); + trySetReaderValue(field); + trySetShortValue(field); + trySetStringValue(field); + trySetTokenStreamValue(field); + + assertEquals(6f, field.numericValue().floatValue(), 0.0f); + } + } + + public void testLegacyIntField() throws Exception { + Field fields[] = new Field[] { + new LegacyIntField("foo", 5, Field.Store.NO), + new LegacyIntField("foo", 5, Field.Store.YES) + }; + + for (Field field : fields) { + trySetBoost(field); + trySetByteValue(field); + trySetBytesValue(field); + trySetBytesRefValue(field); + trySetDoubleValue(field); + field.setIntValue(6); // ok + trySetFloatValue(field); + trySetLongValue(field); + trySetReaderValue(field); + trySetShortValue(field); + trySetStringValue(field); + trySetTokenStreamValue(field); + + assertEquals(6, field.numericValue().intValue()); + } + } + + public void testLegacyLongField() throws Exception { + Field fields[] = new Field[] { + new LegacyLongField("foo", 5L, Field.Store.NO), + new LegacyLongField("foo", 5L, Field.Store.YES) + }; + + for (Field field : fields) { + trySetBoost(field); + trySetByteValue(field); + trySetBytesValue(field); + trySetBytesRefValue(field); + trySetDoubleValue(field); + trySetIntValue(field); + trySetFloatValue(field); + field.setLongValue(6); // ok + trySetReaderValue(field); + trySetShortValue(field); + trySetStringValue(field); + trySetTokenStreamValue(field); + + assertEquals(6L, field.numericValue().longValue()); + } + } + + private void trySetByteValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setByteValue((byte) 10); + }); + } + + private void trySetBytesValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setBytesValue(new byte[] { 5, 5 }); + }); + } + + private void trySetBytesRefValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setBytesValue(new BytesRef("bogus")); + }); + } + + private void trySetDoubleValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setDoubleValue(Double.MAX_VALUE); + }); + } + + private void trySetIntValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setIntValue(Integer.MAX_VALUE); + }); + } + + private void trySetLongValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setLongValue(Long.MAX_VALUE); + }); + } + + private void trySetFloatValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setFloatValue(Float.MAX_VALUE); + }); + } + + private void trySetReaderValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setReaderValue(new StringReader("BOO!")); + }); + } + + private void trySetShortValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setShortValue(Short.MAX_VALUE); + }); + } + + private void trySetStringValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setStringValue("BOO!"); + }); + } + + private void trySetTokenStreamValue(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setTokenStream(new CannedTokenStream(new Token("foo", 0, 3))); + }); + } + + private void trySetBoost(Field f) { + expectThrows(IllegalArgumentException.class, () -> { + f.setBoost(5.0f); + }); + } +} diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyFieldReuse.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyFieldReuse.java new file mode 100644 index 00000000000..9335290247d --- /dev/null +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyFieldReuse.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + + +import java.io.IOException; + +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.CannedTokenStream; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Field; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyNumericTokenStream; +import org.apache.lucene.legacy.LegacyNumericUtils; +import org.apache.lucene.legacy.LegacyNumericTokenStream.LegacyNumericTermAttribute; + +/** test tokenstream reuse by DefaultIndexingChain */ +public class TestLegacyFieldReuse extends BaseTokenStreamTestCase { + + public void testNumericReuse() throws IOException { + LegacyIntField legacyIntField = new LegacyIntField("foo", 5, Field.Store.NO); + + // passing null + TokenStream ts = legacyIntField.tokenStream(null, null); + assertTrue(ts instanceof LegacyNumericTokenStream); + assertEquals(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, ((LegacyNumericTokenStream)ts).getPrecisionStep()); + assertNumericContents(5, ts); + + // now reuse previous stream + legacyIntField = new LegacyIntField("foo", 20, Field.Store.NO); + TokenStream ts2 = legacyIntField.tokenStream(null, ts); + assertSame(ts, ts2); + assertNumericContents(20, ts); + + // pass a bogus stream and ensure it's still ok + legacyIntField = new LegacyIntField("foo", 2343, Field.Store.NO); + TokenStream bogus = new CannedTokenStream(new Token("bogus", 0, 5)); + ts = legacyIntField.tokenStream(null, bogus); + assertNotSame(bogus, ts); + assertNumericContents(2343, ts); + + // pass another bogus stream (numeric, but different precision step!) + legacyIntField = new LegacyIntField("foo", 42, Field.Store.NO); + assert 3 != LegacyNumericUtils.PRECISION_STEP_DEFAULT; + bogus = new LegacyNumericTokenStream(3); + ts = legacyIntField.tokenStream(null, bogus); + assertNotSame(bogus, ts); + assertNumericContents(42, ts); + } + + private void assertNumericContents(int value, TokenStream ts) throws IOException { + assertTrue(ts instanceof LegacyNumericTokenStream); + LegacyNumericTermAttribute numericAtt = ts.getAttribute(LegacyNumericTermAttribute.class); + ts.reset(); + boolean seen = false; + while (ts.incrementToken()) { + if (numericAtt.getShift() == 0) { + assertEquals(value, numericAtt.getRawValue()); + seen = true; + } + } + ts.end(); + ts.close(); + assertTrue(seen); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/util/TestLegacyNumericUtils.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyNumericUtils.java similarity index 98% rename from lucene/core/src/test/org/apache/lucene/util/TestLegacyNumericUtils.java rename to lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyNumericUtils.java index 2fb20d11db7..8607efdc893 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestLegacyNumericUtils.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyNumericUtils.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.util; +package org.apache.lucene.legacy; import java.util.Arrays; @@ -22,6 +22,13 @@ import java.util.Collections; import java.util.Iterator; import java.util.Random; +import org.apache.lucene.legacy.LegacyNumericUtils; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.LongBitSet; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.NumericUtils; + public class TestLegacyNumericUtils extends LuceneTestCase { public void testLongConversionAndOrdering() throws Exception { diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyTerms.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyTerms.java new file mode 100644 index 00000000000..27fae15e916 --- /dev/null +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestLegacyTerms.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.legacy; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericUtils; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.NumericUtils; + +public class TestLegacyTerms extends LuceneTestCase { + + public void testEmptyIntFieldMinMax() throws Exception { + assertNull(LegacyNumericUtils.getMinInt(EMPTY_TERMS)); + assertNull(LegacyNumericUtils.getMaxInt(EMPTY_TERMS)); + } + + public void testIntFieldMinMax() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir); + int numDocs = atLeast(100); + int minValue = Integer.MAX_VALUE; + int maxValue = Integer.MIN_VALUE; + for(int i=0;iupper) { - int a=lower; lower=upper; upper=a; - } - final BytesRef lowerBytes, upperBytes; - BytesRefBuilder b = new BytesRefBuilder(); - LegacyNumericUtils.intToPrefixCoded(lower, 0, b); - lowerBytes = b.toBytesRef(); - LegacyNumericUtils.intToPrefixCoded(upper, 0, b); - upperBytes = b.toBytesRef(); - - // test inclusive range - LegacyNumericRangeQuery tq= LegacyNumericRangeQuery.newIntRange(field, precisionStep, lower, upper, true, true); - TermRangeQuery cq=new TermRangeQuery(field, lowerBytes, upperBytes, true, true); - TopDocs tTopDocs = searcher.search(tq, 1); - TopDocs cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test exclusive range - tq= LegacyNumericRangeQuery.newIntRange(field, precisionStep, lower, upper, false, false); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, false, false); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test left exclusive range - tq= LegacyNumericRangeQuery.newIntRange(field, precisionStep, lower, upper, false, true); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, false, true); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test right exclusive range - tq= LegacyNumericRangeQuery.newIntRange(field, precisionStep, lower, upper, true, false); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, true, false); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - } - - checkTermCounts(precisionStep, totalTermCountT, totalTermCountC); - if (VERBOSE && precisionStep != Integer.MAX_VALUE) { - System.out.println("Average number of terms during random search on '" + field + "':"); - System.out.println(" Numeric query: " + (((double)totalTermCountT)/(num * 4))); - System.out.println(" Classical query: " + (((double)totalTermCountC)/(num * 4))); - } - } - - @Test - public void testEmptyEnums() throws Exception { - int count=3000; - int lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3); - // test empty enum - assert lower < upper; - assertTrue(0 < countTerms(LegacyNumericRangeQuery.newIntRange("field4", 4, lower, upper, true, true))); - assertEquals(0, countTerms(LegacyNumericRangeQuery.newIntRange("field4", 4, upper, lower, true, true))); - // test empty enum outside of bounds - lower = distance*noDocs+startOffset; - upper = 2 * lower; - assert lower < upper; - assertEquals(0, countTerms(LegacyNumericRangeQuery.newIntRange("field4", 4, lower, upper, true, true))); - } - - private int countTerms(MultiTermQuery q) throws Exception { - final Terms terms = MultiFields.getTerms(reader, q.getField()); - if (terms == null) - return 0; - final TermsEnum termEnum = q.getTermsEnum(terms); - assertNotNull(termEnum); - int count = 0; - BytesRef cur, last = null; - while ((cur = termEnum.next()) != null) { - count++; - if (last != null) { - assertTrue(last.compareTo(cur) < 0); - } - last = BytesRef.deepCopyOf(cur); - } - // LUCENE-3314: the results after next() already returned null are undefined, - // assertNull(termEnum.next()); - return count; - } - - private void checkTermCounts(int precisionStep, int termCountT, int termCountC) { - if (precisionStep == Integer.MAX_VALUE) { - assertEquals("Number of terms should be equal for unlimited precStep", termCountC, termCountT); - } else { - assertTrue("Number of terms for NRQ should be <= compared to classical TRQ", termCountT <= termCountC); - } - } - - @Test - public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception { - testRandomTrieAndClassicRangeQuery(8); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_4bit() throws Exception { - testRandomTrieAndClassicRangeQuery(4); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_2bit() throws Exception { - testRandomTrieAndClassicRangeQuery(2); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_NoTrie() throws Exception { - testRandomTrieAndClassicRangeQuery(Integer.MAX_VALUE); - } - private void testRangeSplit(int precisionStep) throws Exception { String field="ascfield"+precisionStep; // 10 random tests diff --git a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericRangeQuery64.java similarity index 72% rename from lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java rename to lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericRangeQuery64.java index 7f63fbc0136..b3ce55aa66d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericRangeQuery64.java @@ -14,28 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.search; +package org.apache.lucene.legacy; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyDoubleField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryUtils; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.TestLegacyNumericUtils; import org.apache.lucene.util.TestUtil; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -63,37 +61,37 @@ public class TestNumericRangeQuery64 extends LuceneTestCase { .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) .setMergePolicy(newLogMergePolicy())); - final FieldType storedLong = new FieldType(LegacyLongField.TYPE_NOT_STORED); + final LegacyFieldType storedLong = new LegacyFieldType(LegacyLongField.TYPE_NOT_STORED); storedLong.setStored(true); storedLong.freeze(); - final FieldType storedLong8 = new FieldType(storedLong); + final LegacyFieldType storedLong8 = new LegacyFieldType(storedLong); storedLong8.setNumericPrecisionStep(8); - final FieldType storedLong4 = new FieldType(storedLong); + final LegacyFieldType storedLong4 = new LegacyFieldType(storedLong); storedLong4.setNumericPrecisionStep(4); - final FieldType storedLong6 = new FieldType(storedLong); + final LegacyFieldType storedLong6 = new LegacyFieldType(storedLong); storedLong6.setNumericPrecisionStep(6); - final FieldType storedLong2 = new FieldType(storedLong); + final LegacyFieldType storedLong2 = new LegacyFieldType(storedLong); storedLong2.setNumericPrecisionStep(2); - final FieldType storedLongNone = new FieldType(storedLong); + final LegacyFieldType storedLongNone = new LegacyFieldType(storedLong); storedLongNone.setNumericPrecisionStep(Integer.MAX_VALUE); - final FieldType unstoredLong = LegacyLongField.TYPE_NOT_STORED; + final LegacyFieldType unstoredLong = LegacyLongField.TYPE_NOT_STORED; - final FieldType unstoredLong8 = new FieldType(unstoredLong); + final LegacyFieldType unstoredLong8 = new LegacyFieldType(unstoredLong); unstoredLong8.setNumericPrecisionStep(8); - final FieldType unstoredLong6 = new FieldType(unstoredLong); + final LegacyFieldType unstoredLong6 = new LegacyFieldType(unstoredLong); unstoredLong6.setNumericPrecisionStep(6); - final FieldType unstoredLong4 = new FieldType(unstoredLong); + final LegacyFieldType unstoredLong4 = new LegacyFieldType(unstoredLong); unstoredLong4.setNumericPrecisionStep(4); - final FieldType unstoredLong2 = new FieldType(unstoredLong); + final LegacyFieldType unstoredLong2 = new LegacyFieldType(unstoredLong); unstoredLong2.setNumericPrecisionStep(2); LegacyLongField @@ -374,137 +372,6 @@ public class TestNumericRangeQuery64 extends LuceneTestCase { dir.close(); } - private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception { - String field="field"+precisionStep; - int totalTermCountT=0,totalTermCountC=0,termCountT,termCountC; - int num = TestUtil.nextInt(random(), 10, 20); - for (int i = 0; i < num; i++) { - long lower=(long)(random().nextDouble()*noDocs*distance)+startOffset; - long upper=(long)(random().nextDouble()*noDocs*distance)+startOffset; - if (lower>upper) { - long a=lower; lower=upper; upper=a; - } - final BytesRef lowerBytes, upperBytes; - BytesRefBuilder b = new BytesRefBuilder(); - LegacyNumericUtils.longToPrefixCoded(lower, 0, b); - lowerBytes = b.toBytesRef(); - LegacyNumericUtils.longToPrefixCoded(upper, 0, b); - upperBytes = b.toBytesRef(); - - // test inclusive range - LegacyNumericRangeQuery tq= LegacyNumericRangeQuery.newLongRange(field, precisionStep, lower, upper, true, true); - TermRangeQuery cq=new TermRangeQuery(field, lowerBytes, upperBytes, true, true); - TopDocs tTopDocs = searcher.search(tq, 1); - TopDocs cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test exclusive range - tq= LegacyNumericRangeQuery.newLongRange(field, precisionStep, lower, upper, false, false); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, false, false); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test left exclusive range - tq= LegacyNumericRangeQuery.newLongRange(field, precisionStep, lower, upper, false, true); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, false, true); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - // test right exclusive range - tq= LegacyNumericRangeQuery.newLongRange(field, precisionStep, lower, upper, true, false); - cq=new TermRangeQuery(field, lowerBytes, upperBytes, true, false); - tTopDocs = searcher.search(tq, 1); - cTopDocs = searcher.search(cq, 1); - assertEquals("Returned count for LegacyNumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); - totalTermCountT += termCountT = countTerms(tq); - totalTermCountC += termCountC = countTerms(cq); - checkTermCounts(precisionStep, termCountT, termCountC); - } - - checkTermCounts(precisionStep, totalTermCountT, totalTermCountC); - if (VERBOSE && precisionStep != Integer.MAX_VALUE) { - System.out.println("Average number of terms during random search on '" + field + "':"); - System.out.println(" Numeric query: " + (((double)totalTermCountT)/(num * 4))); - System.out.println(" Classical query: " + (((double)totalTermCountC)/(num * 4))); - } - } - - @Test - public void testEmptyEnums() throws Exception { - int count=3000; - long lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3); - // test empty enum - assert lower < upper; - assertTrue(0 < countTerms(LegacyNumericRangeQuery.newLongRange("field4", 4, lower, upper, true, true))); - assertEquals(0, countTerms(LegacyNumericRangeQuery.newLongRange("field4", 4, upper, lower, true, true))); - // test empty enum outside of bounds - lower = distance*noDocs+startOffset; - upper = 2L * lower; - assert lower < upper; - assertEquals(0, countTerms(LegacyNumericRangeQuery.newLongRange("field4", 4, lower, upper, true, true))); - } - - private int countTerms(MultiTermQuery q) throws Exception { - final Terms terms = MultiFields.getTerms(reader, q.getField()); - if (terms == null) - return 0; - final TermsEnum termEnum = q.getTermsEnum(terms); - assertNotNull(termEnum); - int count = 0; - BytesRef cur, last = null; - while ((cur = termEnum.next()) != null) { - count++; - if (last != null) { - assertTrue(last.compareTo(cur) < 0); - } - last = BytesRef.deepCopyOf(cur); - } - // LUCENE-3314: the results after next() already returned null are undefined, - // assertNull(termEnum.next()); - return count; - } - - private void checkTermCounts(int precisionStep, int termCountT, int termCountC) { - if (precisionStep == Integer.MAX_VALUE) { - assertEquals("Number of terms should be equal for unlimited precStep", termCountC, termCountT); - } else { - assertTrue("Number of terms for NRQ should be <= compared to classical TRQ", termCountT <= termCountC); - } - } - - @Test - public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception { - testRandomTrieAndClassicRangeQuery(8); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_6bit() throws Exception { - testRandomTrieAndClassicRangeQuery(6); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_4bit() throws Exception { - testRandomTrieAndClassicRangeQuery(4); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_2bit() throws Exception { - testRandomTrieAndClassicRangeQuery(2); - } - - @Test - public void testRandomTrieAndClassicRangeQuery_NoTrie() throws Exception { - testRandomTrieAndClassicRangeQuery(Integer.MAX_VALUE); - } - private void testRangeSplit(int precisionStep) throws Exception { String field="ascfield"+precisionStep; // 10 random tests diff --git a/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericTokenStream.java similarity index 85% rename from lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java rename to lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericTokenStream.java index dfaa20e5a2e..a507af09e0d 100644 --- a/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/legacy/TestNumericTokenStream.java @@ -14,15 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.analysis; +package org.apache.lucene.legacy; +import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; -import org.apache.lucene.analysis.LegacyNumericTokenStream.LegacyNumericTermAttributeImpl; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; -import org.apache.lucene.analysis.tokenattributes.TestCharTermAttributeImpl; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.apache.lucene.legacy.LegacyNumericTokenStream; +import org.apache.lucene.legacy.LegacyNumericUtils; +import org.apache.lucene.legacy.LegacyNumericTokenStream.LegacyNumericTermAttributeImpl; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl; @@ -150,20 +152,37 @@ public class TestNumericTokenStream extends BaseTokenStreamTestCase { public void testAttributeClone() throws Exception { LegacyNumericTermAttributeImpl att = new LegacyNumericTermAttributeImpl(); att.init(lvalue, 64, 8, 0); // set some value, to make getBytesRef() work - LegacyNumericTermAttributeImpl copy = TestCharTermAttributeImpl.assertCloneIsEqual(att); + LegacyNumericTermAttributeImpl copy = assertCloneIsEqual(att); assertNotSame(att.getBytesRef(), copy.getBytesRef()); - LegacyNumericTermAttributeImpl copy2 = TestCharTermAttributeImpl.assertCopyIsEqual(att); + LegacyNumericTermAttributeImpl copy2 = assertCopyIsEqual(att); assertNotSame(att.getBytesRef(), copy2.getBytesRef()); // LUCENE-7027 test att.init(lvalue, 64, 8, 64); // Exhausted TokenStream -> should return empty BytesRef assertEquals(new BytesRef(), att.getBytesRef()); - copy = TestCharTermAttributeImpl.assertCloneIsEqual(att); + copy = assertCloneIsEqual(att); assertEquals(new BytesRef(), copy.getBytesRef()); assertNotSame(att.getBytesRef(), copy.getBytesRef()); - copy2 = TestCharTermAttributeImpl.assertCopyIsEqual(att); + copy2 = assertCopyIsEqual(att); assertEquals(new BytesRef(), copy2.getBytesRef()); assertNotSame(att.getBytesRef(), copy2.getBytesRef()); } + public static T assertCloneIsEqual(T att) { + @SuppressWarnings("unchecked") + T clone = (T) att.clone(); + assertEquals("Clone must be equal", att, clone); + assertEquals("Clone's hashcode must be equal", att.hashCode(), clone.hashCode()); + return clone; + } + + public static T assertCopyIsEqual(T att) throws Exception { + @SuppressWarnings("unchecked") + T copy = (T) att.getClass().newInstance(); + att.copyTo(copy); + assertEquals("Copied instance must be equal", att, copy); + assertEquals("Copied instance's hashcode must be equal", att.hashCode(), copy.hashCode()); + return copy; + } + } diff --git a/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java b/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java index 6a78e1c0f38..af1e7bd5e9b 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java @@ -22,7 +22,6 @@ import java.io.Closeable; import java.lang.reflect.Modifier; import org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; @@ -176,11 +175,7 @@ public abstract class TokenStream extends AttributeSource implements Closeable { * @throws IOException If an I/O error occurs */ public void end() throws IOException { - clearAttributes(); // LUCENE-3849: don't consume dirty atts - PositionIncrementAttribute posIncAtt = getAttribute(PositionIncrementAttribute.class); - if (posIncAtt != null) { - posIncAtt.setPositionIncrement(0); - } + endAttributes(); // LUCENE-3849: don't consume dirty atts } /** diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java index a84d7b70c26..aaa3316b576 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java @@ -138,6 +138,17 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl startOffset = endOffset = 0; type = DEFAULT_TYPE; } + + /** Resets the attributes at end + */ + @Override + public void end() { + super.end(); + positionIncrement = 0; + positionLength = 1; + startOffset = endOffset = 0; + type = DEFAULT_TYPE; + } @Override public PackedTokenAttributeImpl clone() { diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java index 283f481f760..4d63d6fb36b 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java @@ -46,6 +46,11 @@ public class PositionIncrementAttributeImpl extends AttributeImpl implements Pos this.positionIncrement = 1; } + @Override + public void end() { + this.positionIncrement = 0; + } + @Override public boolean equals(Object other) { if (other == this) { diff --git a/lucene/core/src/java/org/apache/lucene/document/Field.java b/lucene/core/src/java/org/apache/lucene/document/Field.java index 87986101f3b..8f5f8692d66 100644 --- a/lucene/core/src/java/org/apache/lucene/document/Field.java +++ b/lucene/core/src/java/org/apache/lucene/document/Field.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.io.Reader; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.LegacyNumericTokenStream; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.BytesTermAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -426,9 +425,6 @@ public class Field implements IndexableField { if (type.indexOptions() == IndexOptions.NONE || !type.tokenized()) { throw new IllegalArgumentException("TokenStream fields must be indexed and tokenized"); } - if (type.numericType() != null) { - throw new IllegalArgumentException("cannot set private TokenStream on numeric fields"); - } this.tokenStream = tokenStream; } @@ -511,35 +507,6 @@ public class Field implements IndexableField { return null; } - final FieldType.LegacyNumericType numericType = fieldType().numericType(); - if (numericType != null) { - if (!(reuse instanceof LegacyNumericTokenStream && ((LegacyNumericTokenStream)reuse).getPrecisionStep() == type.numericPrecisionStep())) { - // lazy init the TokenStream as it is heavy to instantiate - // (attributes,...) if not needed (stored field loading) - reuse = new LegacyNumericTokenStream(type.numericPrecisionStep()); - } - final LegacyNumericTokenStream nts = (LegacyNumericTokenStream) reuse; - // initialize value in TokenStream - final Number val = (Number) fieldsData; - switch (numericType) { - case INT: - nts.setIntValue(val.intValue()); - break; - case LONG: - nts.setLongValue(val.longValue()); - break; - case FLOAT: - nts.setFloatValue(val.floatValue()); - break; - case DOUBLE: - nts.setDoubleValue(val.doubleValue()); - break; - default: - throw new AssertionError("Should never get here"); - } - return reuse; - } - if (!fieldType().tokenized()) { if (stringValue() != null) { if (!(reuse instanceof StringTokenStream)) { diff --git a/lucene/core/src/java/org/apache/lucene/document/FieldType.java b/lucene/core/src/java/org/apache/lucene/document/FieldType.java index e0f058f520e..6f206a49ca7 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FieldType.java +++ b/lucene/core/src/java/org/apache/lucene/document/FieldType.java @@ -22,30 +22,12 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.PointValues; -import org.apache.lucene.util.LegacyNumericUtils; /** * Describes the properties of a field. */ public class FieldType implements IndexableFieldType { - /** Data type of the numeric value - * @since 3.2 - * - * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead - */ - @Deprecated - public enum LegacyNumericType { - /** 32-bit integer numeric type */ - INT, - /** 64-bit long numeric type */ - LONG, - /** 32-bit float numeric type */ - FLOAT, - /** 64-bit double numeric type */ - DOUBLE - } - private boolean stored; private boolean tokenized = true; private boolean storeTermVectors; @@ -54,9 +36,7 @@ public class FieldType implements IndexableFieldType { private boolean storeTermVectorPayloads; private boolean omitNorms; private IndexOptions indexOptions = IndexOptions.NONE; - private LegacyNumericType numericType; private boolean frozen; - private int numericPrecisionStep = LegacyNumericUtils.PRECISION_STEP_DEFAULT; private DocValuesType docValuesType = DocValuesType.NONE; private int dimensionCount; private int dimensionNumBytes; @@ -73,8 +53,6 @@ public class FieldType implements IndexableFieldType { this.storeTermVectorPayloads = ref.storeTermVectorPayloads(); this.omitNorms = ref.omitNorms(); this.indexOptions = ref.indexOptions(); - this.numericType = ref.numericType(); - this.numericPrecisionStep = ref.numericPrecisionStep(); this.docValuesType = ref.docValuesType(); this.dimensionCount = ref.dimensionCount; this.dimensionNumBytes = ref.dimensionNumBytes; @@ -297,70 +275,6 @@ public class FieldType implements IndexableFieldType { this.indexOptions = value; } - /** - * Specifies the field's numeric type. - * @param type numeric type, or null if the field has no numeric type. - * @throws IllegalStateException if this FieldType is frozen against - * future modifications. - * @see #numericType() - * - * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead - */ - @Deprecated - public void setNumericType(LegacyNumericType type) { - checkIfFrozen(); - numericType = type; - } - - /** - * LegacyNumericType: if non-null then the field's value will be indexed - * numerically so that {@link org.apache.lucene.search.LegacyNumericRangeQuery} can be used at - * search time. - *

    - * The default is null (no numeric type) - * @see #setNumericType(org.apache.lucene.document.FieldType.LegacyNumericType) - * - * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead - */ - @Deprecated - public LegacyNumericType numericType() { - return numericType; - } - - /** - * Sets the numeric precision step for the field. - * @param precisionStep numeric precision step for the field - * @throws IllegalArgumentException if precisionStep is less than 1. - * @throws IllegalStateException if this FieldType is frozen against - * future modifications. - * @see #numericPrecisionStep() - * - * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead - */ - @Deprecated - public void setNumericPrecisionStep(int precisionStep) { - checkIfFrozen(); - if (precisionStep < 1) { - throw new IllegalArgumentException("precisionStep must be >= 1 (got " + precisionStep + ")"); - } - this.numericPrecisionStep = precisionStep; - } - - /** - * Precision step for numeric field. - *

    - * This has no effect if {@link #numericType()} returns null. - *

    - * The default is {@link org.apache.lucene.util.LegacyNumericUtils#PRECISION_STEP_DEFAULT} - * @see #setNumericPrecisionStep(int) - * - * @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead - */ - @Deprecated - public int numericPrecisionStep() { - return numericPrecisionStep; - } - /** * Enables points indexing. */ @@ -403,7 +317,7 @@ public class FieldType implements IndexableFieldType { /** Prints a Field for human consumption. */ @Override - public final String toString() { + public String toString() { StringBuilder result = new StringBuilder(); if (stored()) { result.append("stored"); @@ -434,12 +348,6 @@ public class FieldType implements IndexableFieldType { result.append(",indexOptions="); result.append(indexOptions); } - if (numericType != null) { - result.append(",numericType="); - result.append(numericType); - result.append(",numericPrecisionStep="); - result.append(numericPrecisionStep); - } } if (dimensionCount != 0) { if (result.length() > 0) { @@ -495,8 +403,6 @@ public class FieldType implements IndexableFieldType { result = prime * result + dimensionNumBytes; result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode()); result = prime * result + indexOptions.hashCode(); - result = prime * result + numericPrecisionStep; - result = prime * result + ((numericType == null) ? 0 : numericType.hashCode()); result = prime * result + (omitNorms ? 1231 : 1237); result = prime * result + (storeTermVectorOffsets ? 1231 : 1237); result = prime * result + (storeTermVectorPayloads ? 1231 : 1237); @@ -517,8 +423,6 @@ public class FieldType implements IndexableFieldType { if (dimensionNumBytes != other.dimensionNumBytes) return false; if (docValuesType != other.docValuesType) return false; if (indexOptions != other.indexOptions) return false; - if (numericPrecisionStep != other.numericPrecisionStep) return false; - if (numericType != other.numericType) return false; if (omitNorms != other.omitNorms) return false; if (storeTermVectorOffsets != other.storeTermVectorOffsets) return false; if (storeTermVectorPayloads != other.storeTermVectorPayloads) return false; diff --git a/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java b/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java index c8fddf728c7..a8200c6edd3 100644 --- a/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java +++ b/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java @@ -186,4 +186,33 @@ public class Rectangle { return new Rectangle(minLat, maxLat, minLon, maxLon); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Rectangle rectangle = (Rectangle) o; + + if (Double.compare(rectangle.minLat, minLat) != 0) return false; + if (Double.compare(rectangle.minLon, minLon) != 0) return false; + if (Double.compare(rectangle.maxLat, maxLat) != 0) return false; + return Double.compare(rectangle.maxLon, maxLon) == 0; + + } + + @Override + public int hashCode() { + int result; + long temp; + temp = Double.doubleToLongBits(minLat); + result = (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(minLon); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(maxLat); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(maxLon); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + return result; + } } diff --git a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java index 7ad208ff22f..44c04e54b7c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java +++ b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java @@ -405,6 +405,7 @@ public class LRUQueryCache implements QueryCache, Accountable { lock.lock(); try { cache.clear(); + // Note that this also clears the uniqueQueries map since mostRecentlyUsedQueries is the uniqueQueries.keySet view: mostRecentlyUsedQueries.clear(); onClear(); } finally { diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBufferGuard.java b/lucene/core/src/java/org/apache/lucene/store/ByteBufferGuard.java new file mode 100644 index 00000000000..95fa17d5ea0 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/store/ByteBufferGuard.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A guard that is created for every {@link ByteBufferIndexInput} that tries on best effort + * to reject any access to the {@link ByteBuffer} behind, once it is unmapped. A single instance + * of this is used for the original and all clones, so once the original is closed and unmapped + * all clones also throw {@link AlreadyClosedException}, triggered by a {@link NullPointerException}. + *

    + * This code tries to hopefully flush any CPU caches using a store-store barrier. It also yields the + * current thread to give other threads a chance to finish in-flight requests... + */ +final class ByteBufferGuard { + + /** + * Pass in an implementation of this interface to cleanup ByteBuffers. + * MMapDirectory implements this to allow unmapping of bytebuffers with private Java APIs. + */ + @FunctionalInterface + static interface BufferCleaner { + void freeBuffer(String resourceDescription, ByteBuffer b) throws IOException; + } + + private final String resourceDescription; + private final BufferCleaner cleaner; + + /** Not volatile; see comments on visibility below! */ + private boolean invalidated = false; + + /** Used as a store-store barrier; see comments below! */ + private final AtomicInteger barrier = new AtomicInteger(); + + /** + * Creates an instance to be used for a single {@link ByteBufferIndexInput} which + * must be shared by all of its clones. + */ + public ByteBufferGuard(String resourceDescription, BufferCleaner cleaner) { + this.resourceDescription = resourceDescription; + this.cleaner = cleaner; + } + + /** + * Invalidates this guard and unmaps (if supported). + */ + public void invalidateAndUnmap(ByteBuffer... bufs) throws IOException { + if (cleaner != null) { + invalidated = true; + // This call should hopefully flush any CPU caches and as a result make + // the "invalidated" field update visible to other threads. We specifically + // don't make "invalidated" field volatile for performance reasons, hoping the + // JVM won't optimize away reads of that field and hardware should ensure + // caches are in sync after this call. This isn't entirely "fool-proof" + // (see LUCENE-7409 discussion), but it has been shown to work in practice + // and we count on this behavior. + barrier.lazySet(0); + // we give other threads a bit of time to finish reads on their ByteBuffer...: + Thread.yield(); + // finally unmap the ByteBuffers: + for (ByteBuffer b : bufs) { + cleaner.freeBuffer(resourceDescription, b); + } + } + } + + private void ensureValid() { + if (invalidated) { + // this triggers an AlreadyClosedException in ByteBufferIndexInput: + throw new NullPointerException(); + } + } + + public void getBytes(ByteBuffer receiver, byte[] dst, int offset, int length) { + ensureValid(); + receiver.get(dst, offset, length); + } + + public byte getByte(ByteBuffer receiver) { + ensureValid(); + return receiver.get(); + } + + public short getShort(ByteBuffer receiver) { + ensureValid(); + return receiver.getShort(); + } + + public int getInt(ByteBuffer receiver) { + ensureValid(); + return receiver.getInt(); + } + + public long getLong(ByteBuffer receiver) { + ensureValid(); + return receiver.getLong(); + } + + public byte getByte(ByteBuffer receiver, int pos) { + ensureValid(); + return receiver.get(pos); + } + + public short getShort(ByteBuffer receiver, int pos) { + ensureValid(); + return receiver.getShort(pos); + } + + public int getInt(ByteBuffer receiver, int pos) { + ensureValid(); + return receiver.getInt(pos); + } + + public long getLong(ByteBuffer receiver, int pos) { + ensureValid(); + return receiver.getLong(pos); + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java index 8e8ef90655a..0f6c733410b 100644 --- a/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java @@ -21,9 +21,6 @@ import java.io.EOFException; import java.io.IOException; import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; -import java.util.Iterator; - -import org.apache.lucene.util.WeakIdentityMap; /** * Base IndexInput implementation that uses an array @@ -37,35 +34,32 @@ import org.apache.lucene.util.WeakIdentityMap; * are a power-of-two (chunkSizePower). */ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessInput { - protected final BufferCleaner cleaner; protected final long length; protected final long chunkSizeMask; protected final int chunkSizePower; + protected final ByteBufferGuard guard; protected ByteBuffer[] buffers; protected int curBufIndex = -1; protected ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex] protected boolean isClone = false; - protected final WeakIdentityMap clones; - public static ByteBufferIndexInput newInstance(String resourceDescription, ByteBuffer[] buffers, long length, int chunkSizePower, BufferCleaner cleaner, boolean trackClones) { - final WeakIdentityMap clones = trackClones ? WeakIdentityMap.newConcurrentHashMap() : null; + public static ByteBufferIndexInput newInstance(String resourceDescription, ByteBuffer[] buffers, long length, int chunkSizePower, ByteBufferGuard guard) { if (buffers.length == 1) { - return new SingleBufferImpl(resourceDescription, buffers[0], length, chunkSizePower, cleaner, clones); + return new SingleBufferImpl(resourceDescription, buffers[0], length, chunkSizePower, guard); } else { - return new MultiBufferImpl(resourceDescription, buffers, 0, length, chunkSizePower, cleaner, clones); + return new MultiBufferImpl(resourceDescription, buffers, 0, length, chunkSizePower, guard); } } - ByteBufferIndexInput(String resourceDescription, ByteBuffer[] buffers, long length, int chunkSizePower, BufferCleaner cleaner, WeakIdentityMap clones) { + ByteBufferIndexInput(String resourceDescription, ByteBuffer[] buffers, long length, int chunkSizePower, ByteBufferGuard guard) { super(resourceDescription); this.buffers = buffers; this.length = length; this.chunkSizePower = chunkSizePower; this.chunkSizeMask = (1L << chunkSizePower) - 1L; - this.clones = clones; - this.cleaner = cleaner; + this.guard = guard; assert chunkSizePower >= 0 && chunkSizePower <= 30; assert (length >>> chunkSizePower) < Integer.MAX_VALUE; } @@ -73,7 +67,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public final byte readByte() throws IOException { try { - return curBuf.get(); + return guard.getByte(curBuf); } catch (BufferUnderflowException e) { do { curBufIndex++; @@ -83,7 +77,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn curBuf = buffers[curBufIndex]; curBuf.position(0); } while (!curBuf.hasRemaining()); - return curBuf.get(); + return guard.getByte(curBuf); } catch (NullPointerException npe) { throw new AlreadyClosedException("Already closed: " + this); } @@ -92,11 +86,11 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public final void readBytes(byte[] b, int offset, int len) throws IOException { try { - curBuf.get(b, offset, len); + guard.getBytes(curBuf, b, offset, len); } catch (BufferUnderflowException e) { int curAvail = curBuf.remaining(); while (len > curAvail) { - curBuf.get(b, offset, curAvail); + guard.getBytes(curBuf, b, offset, curAvail); len -= curAvail; offset += curAvail; curBufIndex++; @@ -107,7 +101,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn curBuf.position(0); curAvail = curBuf.remaining(); } - curBuf.get(b, offset, len); + guard.getBytes(curBuf, b, offset, len); } catch (NullPointerException npe) { throw new AlreadyClosedException("Already closed: " + this); } @@ -116,7 +110,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public final short readShort() throws IOException { try { - return curBuf.getShort(); + return guard.getShort(curBuf); } catch (BufferUnderflowException e) { return super.readShort(); } catch (NullPointerException npe) { @@ -127,7 +121,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public final int readInt() throws IOException { try { - return curBuf.getInt(); + return guard.getInt(curBuf); } catch (BufferUnderflowException e) { return super.readInt(); } catch (NullPointerException npe) { @@ -138,7 +132,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public final long readLong() throws IOException { try { - return curBuf.getLong(); + return guard.getLong(curBuf); } catch (BufferUnderflowException e) { return super.readLong(); } catch (NullPointerException npe) { @@ -181,7 +175,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn public byte readByte(long pos) throws IOException { try { final int bi = (int) (pos >> chunkSizePower); - return buffers[bi].get((int) (pos & chunkSizeMask)); + return guard.getByte(buffers[bi], (int) (pos & chunkSizeMask)); } catch (IndexOutOfBoundsException ioobe) { throw new EOFException("seek past EOF: " + this); } catch (NullPointerException npe) { @@ -207,7 +201,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn public short readShort(long pos) throws IOException { final int bi = (int) (pos >> chunkSizePower); try { - return buffers[bi].getShort((int) (pos & chunkSizeMask)); + return guard.getShort(buffers[bi], (int) (pos & chunkSizeMask)); } catch (IndexOutOfBoundsException ioobe) { // either it's a boundary, or read past EOF, fall back: setPos(pos, bi); @@ -221,7 +215,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn public int readInt(long pos) throws IOException { final int bi = (int) (pos >> chunkSizePower); try { - return buffers[bi].getInt((int) (pos & chunkSizeMask)); + return guard.getInt(buffers[bi], (int) (pos & chunkSizeMask)); } catch (IndexOutOfBoundsException ioobe) { // either it's a boundary, or read past EOF, fall back: setPos(pos, bi); @@ -235,7 +229,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn public long readLong(long pos) throws IOException { final int bi = (int) (pos >> chunkSizePower); try { - return buffers[bi].getLong((int) (pos & chunkSizeMask)); + return guard.getLong(buffers[bi], (int) (pos & chunkSizeMask)); } catch (IndexOutOfBoundsException ioobe) { // either it's a boundary, or read past EOF, fall back: setPos(pos, bi); @@ -285,11 +279,6 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn final ByteBufferIndexInput clone = newCloneInstance(getFullSliceDescription(sliceDescription), newBuffers, ofs, length); clone.isClone = true; - - // register the new clone in our clone list to clean it up on closing: - if (clones != null) { - this.clones.put(clone, Boolean.TRUE); - } return clone; } @@ -299,9 +288,9 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn protected ByteBufferIndexInput newCloneInstance(String newResourceDescription, ByteBuffer[] newBuffers, int offset, long length) { if (newBuffers.length == 1) { newBuffers[0].position(offset); - return new SingleBufferImpl(newResourceDescription, newBuffers[0].slice(), length, chunkSizePower, this.cleaner, this.clones); + return new SingleBufferImpl(newResourceDescription, newBuffers[0].slice(), length, chunkSizePower, this.guard); } else { - return new MultiBufferImpl(newResourceDescription, newBuffers, offset, length, chunkSizePower, cleaner, clones); + return new MultiBufferImpl(newResourceDescription, newBuffers, offset, length, chunkSizePower, guard); } } @@ -335,25 +324,11 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn // make local copy, then un-set early final ByteBuffer[] bufs = buffers; unsetBuffers(); - if (clones != null) { - clones.remove(this); - } if (isClone) return; - // for extra safety unset also all clones' buffers: - if (clones != null) { - for (Iterator it = this.clones.keyIterator(); it.hasNext();) { - final ByteBufferIndexInput clone = it.next(); - assert clone.isClone; - clone.unsetBuffers(); - } - this.clones.clear(); - } - - for (final ByteBuffer b : bufs) { - freeBuffer(b); - } + // tell the guard to invalidate and later unmap the bytebuffers (if supported): + guard.invalidateAndUnmap(bufs); } finally { unsetBuffers(); } @@ -367,31 +342,12 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn curBuf = null; curBufIndex = 0; } - - /** - * Called when the contents of a buffer will be no longer needed. - */ - private void freeBuffer(ByteBuffer b) throws IOException { - if (cleaner != null) { - cleaner.freeBuffer(this, b); - } - } - - /** - * Pass in an implementation of this interface to cleanup ByteBuffers. - * MMapDirectory implements this to allow unmapping of bytebuffers with private Java APIs. - */ - @FunctionalInterface - static interface BufferCleaner { - void freeBuffer(ByteBufferIndexInput parent, ByteBuffer b) throws IOException; - } /** Optimization of ByteBufferIndexInput for when there is only one buffer */ static final class SingleBufferImpl extends ByteBufferIndexInput { - SingleBufferImpl(String resourceDescription, ByteBuffer buffer, long length, int chunkSizePower, - BufferCleaner cleaner, WeakIdentityMap clones) { - super(resourceDescription, new ByteBuffer[] { buffer }, length, chunkSizePower, cleaner, clones); + SingleBufferImpl(String resourceDescription, ByteBuffer buffer, long length, int chunkSizePower, ByteBufferGuard guard) { + super(resourceDescription, new ByteBuffer[] { buffer }, length, chunkSizePower, guard); this.curBufIndex = 0; this.curBuf = buffer; buffer.position(0); @@ -426,7 +382,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public byte readByte(long pos) throws IOException { try { - return curBuf.get((int) pos); + return guard.getByte(curBuf, (int) pos); } catch (IllegalArgumentException e) { if (pos < 0) { throw new IllegalArgumentException("Seeking to negative position: " + this, e); @@ -441,7 +397,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public short readShort(long pos) throws IOException { try { - return curBuf.getShort((int) pos); + return guard.getShort(curBuf, (int) pos); } catch (IllegalArgumentException e) { if (pos < 0) { throw new IllegalArgumentException("Seeking to negative position: " + this, e); @@ -456,7 +412,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public int readInt(long pos) throws IOException { try { - return curBuf.getInt((int) pos); + return guard.getInt(curBuf, (int) pos); } catch (IllegalArgumentException e) { if (pos < 0) { throw new IllegalArgumentException("Seeking to negative position: " + this, e); @@ -471,7 +427,7 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn @Override public long readLong(long pos) throws IOException { try { - return curBuf.getLong((int) pos); + return guard.getLong(curBuf, (int) pos); } catch (IllegalArgumentException e) { if (pos < 0) { throw new IllegalArgumentException("Seeking to negative position: " + this, e); @@ -489,8 +445,8 @@ abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessIn private final int offset; MultiBufferImpl(String resourceDescription, ByteBuffer[] buffers, int offset, long length, int chunkSizePower, - BufferCleaner cleaner, WeakIdentityMap clones) { - super(resourceDescription, buffers, length, chunkSizePower, cleaner, clones); + ByteBufferGuard guard) { + super(resourceDescription, buffers, length, chunkSizePower, guard); this.offset = offset; try { seek(0L); diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index 60ca103a047..c0e35197f0e 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -36,7 +36,7 @@ import java.util.concurrent.Future; import java.lang.invoke.MethodHandle; import java.lang.reflect.Method; -import org.apache.lucene.store.ByteBufferIndexInput.BufferCleaner; +import org.apache.lucene.store.ByteBufferGuard.BufferCleaner; import org.apache.lucene.util.Constants; import org.apache.lucene.util.SuppressForbidden; @@ -240,7 +240,7 @@ public class MMapDirectory extends FSDirectory { final boolean useUnmap = getUseUnmap(); return ByteBufferIndexInput.newInstance(resourceDescription, map(resourceDescription, c, 0, c.size()), - c.size(), chunkSizePower, useUnmap ? CLEANER : null, useUnmap); + c.size(), chunkSizePower, new ByteBufferGuard(resourceDescription, useUnmap ? CLEANER : null)); } } @@ -370,7 +370,7 @@ public class MMapDirectory extends FSDirectory { final MethodHandle unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)) .asType(methodType(void.class, ByteBuffer.class)); - return (BufferCleaner) (ByteBufferIndexInput parent, ByteBuffer buffer) -> { + return (BufferCleaner) (String resourceDescription, ByteBuffer buffer) -> { if (directBufferClass.isInstance(buffer)) { final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { try { @@ -381,7 +381,7 @@ public class MMapDirectory extends FSDirectory { } }); if (error != null) { - throw new IOException("Unable to unmap the mapped buffer: " + parent.toString(), error); + throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, error); } } }; diff --git a/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java b/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java index b7226612ef7..8a7c5429d05 100644 --- a/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java @@ -32,6 +32,17 @@ public abstract class AttributeImpl implements Cloneable, Attribute { */ public abstract void clear(); + /** + * Clears the values in this AttributeImpl and resets it to its value + * at the end of the field. If this implementation implements more than one Attribute interface + * it clears all. + *

    + * The default implementation simply calls {@link #clear()} + */ + public void end() { + clear(); + } + /** * This method returns the current attribute values as a string in the following format * by calling the {@link #reflectWith(AttributeReflector)} method: diff --git a/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java b/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java index 6c39025ba4a..e962fedc1de 100644 --- a/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java +++ b/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java @@ -270,6 +270,16 @@ public class AttributeSource { state.attribute.clear(); } } + + /** + * Resets all Attributes in this AttributeSource by calling + * {@link AttributeImpl#end()} on each Attribute implementation. + */ + public final void endAttributes() { + for (State state = getCurrentState(); state != null; state = state.next) { + state.attribute.end(); + } + } /** * Removes all attributes and their implementations from this AttributeSource. diff --git a/lucene/core/src/java/org/apache/lucene/util/Version.java b/lucene/core/src/java/org/apache/lucene/util/Version.java index 3bd817ea8a5..478f41f16ff 100644 --- a/lucene/core/src/java/org/apache/lucene/util/Version.java +++ b/lucene/core/src/java/org/apache/lucene/util/Version.java @@ -59,6 +59,13 @@ public final class Version { @Deprecated public static final Version LUCENE_6_2_0 = new Version(6, 2, 0); + /** + * Match settings and bugs in Lucene's 6.3.0 release. + * @deprecated Use latest + */ + @Deprecated + public static final Version LUCENE_6_3_0 = new Version(6, 3, 0); + /** * Match settings and bugs in Lucene's 7.0.0 release. *

    diff --git a/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestGrowableByteArrayDataOutput.java b/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestGrowableByteArrayDataOutput.java index fb90d925eb9..3820733ecca 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestGrowableByteArrayDataOutput.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestGrowableByteArrayDataOutput.java @@ -58,7 +58,7 @@ public class TestGrowableByteArrayDataOutput extends LuceneTestCase { public void testWriteLargeStrings() throws Exception { int minSizeForDoublePass = GrowableByteArrayDataOutput.MIN_UTF8_SIZE_TO_ENABLE_DOUBLE_PASS_ENCODING; - int num = atLeast(1000); + int num = atLeast(100); for (int i = 0; i < num; i++) { String unicode = TestUtil.randomRealisticUnicodeString(random(), minSizeForDoublePass, 10 * minSizeForDoublePass); byte[] utf8 = new byte[unicode.length() * UnicodeUtil.MAX_UTF8_BYTES_PER_CHAR]; diff --git a/lucene/core/src/test/org/apache/lucene/document/TestField.java b/lucene/core/src/test/org/apache/lucene/document/TestField.java index 92d6a832141..4ef7ffbcdcc 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestField.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestField.java @@ -79,29 +79,7 @@ public class TestField extends LuceneTestCase { assertEquals("DoublePoint ", field.toString()); } - public void testLegacyDoubleField() throws Exception { - Field fields[] = new Field[] { - new LegacyDoubleField("foo", 5d, Field.Store.NO), - new LegacyDoubleField("foo", 5d, Field.Store.YES) - }; - for (Field field : fields) { - trySetBoost(field); - trySetByteValue(field); - trySetBytesValue(field); - trySetBytesRefValue(field); - field.setDoubleValue(6d); // ok - trySetIntValue(field); - trySetFloatValue(field); - trySetLongValue(field); - trySetReaderValue(field); - trySetShortValue(field); - trySetStringValue(field); - trySetTokenStreamValue(field); - - assertEquals(6d, field.numericValue().doubleValue(), 0.0d); - } - } public void testDoubleDocValuesField() throws Exception { DoubleDocValuesField field = new DoubleDocValuesField("foo", 5d); @@ -185,30 +163,6 @@ public class TestField extends LuceneTestCase { assertEquals("FloatPoint ", field.toString()); } - public void testLegacyFloatField() throws Exception { - Field fields[] = new Field[] { - new LegacyFloatField("foo", 5f, Field.Store.NO), - new LegacyFloatField("foo", 5f, Field.Store.YES) - }; - - for (Field field : fields) { - trySetBoost(field); - trySetByteValue(field); - trySetBytesValue(field); - trySetBytesRefValue(field); - trySetDoubleValue(field); - trySetIntValue(field); - field.setFloatValue(6f); // ok - trySetLongValue(field); - trySetReaderValue(field); - trySetShortValue(field); - trySetStringValue(field); - trySetTokenStreamValue(field); - - assertEquals(6f, field.numericValue().floatValue(), 0.0f); - } - } - public void testIntPoint() throws Exception { Field field = new IntPoint("foo", 5); @@ -253,30 +207,6 @@ public class TestField extends LuceneTestCase { assertEquals("IntPoint ", field.toString()); } - public void testLegacyIntField() throws Exception { - Field fields[] = new Field[] { - new LegacyIntField("foo", 5, Field.Store.NO), - new LegacyIntField("foo", 5, Field.Store.YES) - }; - - for (Field field : fields) { - trySetBoost(field); - trySetByteValue(field); - trySetBytesValue(field); - trySetBytesRefValue(field); - trySetDoubleValue(field); - field.setIntValue(6); // ok - trySetFloatValue(field); - trySetLongValue(field); - trySetReaderValue(field); - trySetShortValue(field); - trySetStringValue(field); - trySetTokenStreamValue(field); - - assertEquals(6, field.numericValue().intValue()); - } - } - public void testNumericDocValuesField() throws Exception { NumericDocValuesField field = new NumericDocValuesField("foo", 5L); @@ -340,30 +270,6 @@ public class TestField extends LuceneTestCase { assertEquals("LongPoint ", field.toString()); } - public void testLegacyLongField() throws Exception { - Field fields[] = new Field[] { - new LegacyLongField("foo", 5L, Field.Store.NO), - new LegacyLongField("foo", 5L, Field.Store.YES) - }; - - for (Field field : fields) { - trySetBoost(field); - trySetByteValue(field); - trySetBytesValue(field); - trySetBytesRefValue(field); - trySetDoubleValue(field); - trySetIntValue(field); - trySetFloatValue(field); - field.setLongValue(6); // ok - trySetReaderValue(field); - trySetShortValue(field); - trySetStringValue(field); - trySetTokenStreamValue(field); - - assertEquals(6L, field.numericValue().longValue()); - } - } - public void testSortedBytesDocValuesField() throws Exception { SortedDocValuesField field = new SortedDocValuesField("foo", new BytesRef("bar")); diff --git a/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java b/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java index da76f40fe71..9214cb9f844 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java @@ -18,7 +18,6 @@ package org.apache.lucene.document; import java.lang.reflect.Method; -import org.apache.lucene.document.FieldType.LegacyNumericType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.PointValues; @@ -58,14 +57,6 @@ public class TestFieldType extends LuceneTestCase { ft7.setOmitNorms(true); assertFalse(ft7.equals(ft)); - FieldType ft8 = new FieldType(); - ft8.setNumericType(LegacyNumericType.DOUBLE); - assertFalse(ft8.equals(ft)); - - FieldType ft9 = new FieldType(); - ft9.setNumericPrecisionStep(3); - assertFalse(ft9.equals(ft)); - FieldType ft10 = new FieldType(); ft10.setStoreTermVectors(true); assertFalse(ft10.equals(ft)); diff --git a/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java b/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java index e75ae85b8a6..2cfb2f84002 100644 --- a/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java +++ b/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java @@ -30,7 +30,7 @@ public class TestGeoUtils extends LuceneTestCase { // We rely heavily on GeoUtils.circleToBBox so we test it here: public void testRandomCircleToBBox() throws Exception { - int iters = atLeast(1000); + int iters = atLeast(100); for(int iter=0;iter 0) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java index b36cfefa9ff..977df3da618 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java @@ -24,16 +24,12 @@ import java.util.Collections; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.CannedTokenStream; -import org.apache.lucene.analysis.LegacyNumericTokenStream.LegacyNumericTermAttribute; -import org.apache.lucene.analysis.LegacyNumericTokenStream; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.StringField; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; /** test tokenstream reuse by DefaultIndexingChain */ public class TestFieldReuse extends BaseTokenStreamTestCase { @@ -61,7 +57,7 @@ public class TestFieldReuse extends BaseTokenStreamTestCase { // pass a bogus stream and ensure it's still ok stringField = new StringField("foo", "beer", Field.Store.NO); - TokenStream bogus = new LegacyNumericTokenStream(); + TokenStream bogus = new CannedTokenStream(); ts = stringField.tokenStream(null, bogus); assertNotSame(ts, bogus); assertTokenStreamContents(ts, @@ -71,37 +67,6 @@ public class TestFieldReuse extends BaseTokenStreamTestCase { ); } - public void testNumericReuse() throws IOException { - LegacyIntField legacyIntField = new LegacyIntField("foo", 5, Field.Store.NO); - - // passing null - TokenStream ts = legacyIntField.tokenStream(null, null); - assertTrue(ts instanceof LegacyNumericTokenStream); - assertEquals(LegacyNumericUtils.PRECISION_STEP_DEFAULT_32, ((LegacyNumericTokenStream)ts).getPrecisionStep()); - assertNumericContents(5, ts); - - // now reuse previous stream - legacyIntField = new LegacyIntField("foo", 20, Field.Store.NO); - TokenStream ts2 = legacyIntField.tokenStream(null, ts); - assertSame(ts, ts2); - assertNumericContents(20, ts); - - // pass a bogus stream and ensure it's still ok - legacyIntField = new LegacyIntField("foo", 2343, Field.Store.NO); - TokenStream bogus = new CannedTokenStream(new Token("bogus", 0, 5)); - ts = legacyIntField.tokenStream(null, bogus); - assertNotSame(bogus, ts); - assertNumericContents(2343, ts); - - // pass another bogus stream (numeric, but different precision step!) - legacyIntField = new LegacyIntField("foo", 42, Field.Store.NO); - assert 3 != LegacyNumericUtils.PRECISION_STEP_DEFAULT; - bogus = new LegacyNumericTokenStream(3); - ts = legacyIntField.tokenStream(null, bogus); - assertNotSame(bogus, ts); - assertNumericContents(42, ts); - } - static class MyField implements IndexableField { TokenStream lastSeen; TokenStream lastReturned; @@ -163,20 +128,4 @@ public class TestFieldReuse extends BaseTokenStreamTestCase { iw.close(); dir.close(); } - - private void assertNumericContents(int value, TokenStream ts) throws IOException { - assertTrue(ts instanceof LegacyNumericTokenStream); - LegacyNumericTermAttribute numericAtt = ts.getAttribute(LegacyNumericTermAttribute.class); - ts.reset(); - boolean seen = false; - while (ts.incrementToken()) { - if (numericAtt.getShift() == 0) { - assertEquals(value, numericAtt.getRawValue()); - seen = true; - } - } - ts.end(); - ts.close(); - assertTrue(seen); - } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java index 363ccb2ce59..5045396e43e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java @@ -1217,7 +1217,7 @@ public class TestIndexSorting extends LuceneTestCase { if (TEST_NIGHTLY) { numDocs = atLeast(100000); } else { - numDocs = atLeast(10000); + numDocs = atLeast(1000); } List docs = new ArrayList<>(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java index 8d4c5c7903f..55aa6e036d8 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java @@ -72,6 +72,7 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase { dir.close(); } + @Slow public void testStressUpdateSameID() throws Exception { int iters = atLeast(100); for(int iter=0;iter writerRef = new AtomicReference<>(); + writerRef.set(new IndexWriter(dir, newIndexWriterConfig())); + + AtomicReference mgrRef = new AtomicReference<>(); + mgrRef.set(new SearcherManager(writerRef.get(), null)); + final AtomicBoolean stop = new AtomicBoolean(); + + Thread indexThread = new Thread() { + @Override + public void run() { + try { + LineFileDocs docs = new LineFileDocs(random()); + long runTimeSec = TEST_NIGHTLY ? atLeast(10) : atLeast(2); + long endTime = System.nanoTime() + runTimeSec * 1000000000; + while (System.nanoTime() < endTime) { + IndexWriter w = writerRef.get(); + w.addDocument(docs.nextDoc()); + if (random().nextInt(1000) == 17) { + if (random().nextBoolean()) { + w.close(); + } else { + w.rollback(); + } + writerRef.set(new IndexWriter(dir, newIndexWriterConfig())); + } + } + docs.close(); + stop.set(true); + if (VERBOSE) { + System.out.println("TEST: index count=" + writerRef.get().maxDoc()); + } + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + }; + + Thread searchThread = new Thread() { + @Override + public void run() { + try { + long totCount = 0; + while (stop.get() == false) { + SearcherManager mgr = mgrRef.get(); + if (mgr != null) { + IndexSearcher searcher; + try { + searcher = mgr.acquire(); + } catch (AlreadyClosedException ace) { + // ok + continue; + } + totCount += searcher.getIndexReader().maxDoc(); + mgr.release(searcher); + } + } + if (VERBOSE) { + System.out.println("TEST: search totCount=" + totCount); + } + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + }; + + Thread refreshThread = new Thread() { + @Override + public void run() { + try { + int refreshCount = 0; + int aceCount = 0; + while (stop.get() == false) { + SearcherManager mgr = mgrRef.get(); + if (mgr != null) { + refreshCount++; + try { + mgr.maybeRefreshBlocking(); + } catch (AlreadyClosedException ace) { + // ok + aceCount++; + continue; + } + } + } + if (VERBOSE) { + System.out.println("TEST: refresh count=" + refreshCount + " aceCount=" + aceCount); + } + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + }; + + Thread closeThread = new Thread() { + @Override + public void run() { + try { + int closeCount = 0; + int aceCount = 0; + while (stop.get() == false) { + SearcherManager mgr = mgrRef.get(); + assert mgr != null; + mgr.close(); + closeCount++; + while (stop.get() == false) { + try { + mgrRef.set(new SearcherManager(writerRef.get(), null)); + break; + } catch (AlreadyClosedException ace) { + // ok + aceCount++; + } + } + } + if (VERBOSE) { + System.out.println("TEST: close count=" + closeCount + " aceCount=" + aceCount); + } + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + }; + + indexThread.start(); + searchThread.start(); + refreshThread.start(); + closeThread.start(); + + indexThread.join(); + searchThread.join(); + refreshThread.join(); + closeThread.join(); + + mgrRef.get().close(); + writerRef.get().close(); + dir.close(); + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanationsWithFillerDocs.java b/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanationsWithFillerDocs.java index 0a79ae0f495..9f506688c52 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanationsWithFillerDocs.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanationsWithFillerDocs.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.Term; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.util.TestUtil; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.junit.BeforeClass; import org.junit.Assume; @@ -34,6 +35,7 @@ import org.junit.Assume; * all use terms from same set of source data as our regular docs (to emphasis the DocFreq factor in scoring), * in which case the queries will be wrapped so they can be excluded. */ +@Slow // can this be sped up to be non-slow? filler docs make it quite a bit slower and many test methods... public class TestSimpleExplanationsWithFillerDocs extends TestSimpleExplanations { /** num of empty docs injected between every doc in the index */ diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java index 153cc5e6d30..098fd44519b 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java @@ -19,6 +19,10 @@ package org.apache.lucene.store; import java.io.IOException; import java.nio.file.Path; +import java.util.Random; +import java.util.concurrent.CountDownLatch; + +import org.junit.Ignore; /** * Tests MMapDirectory @@ -39,4 +43,38 @@ public class TestMmapDirectory extends BaseDirectoryTestCase { MMapDirectory.UNMAP_SUPPORTED); } + @Ignore("This test is for JVM testing purposes. There are no guarantees that it may not fail with SIGSEGV!") + public void testAceWithThreads() throws Exception { + for (int iter = 0; iter < 10; iter++) { + Directory dir = getDirectory(createTempDir("testAceWithThreads")); + IndexOutput out = dir.createOutput("test", IOContext.DEFAULT); + Random random = random(); + for (int i = 0; i < 8 * 1024 * 1024; i++) { + out.writeInt(random.nextInt()); + } + out.close(); + IndexInput in = dir.openInput("test", IOContext.DEFAULT); + IndexInput clone = in.clone(); + final byte accum[] = new byte[32 * 1024 * 1024]; + final CountDownLatch shotgun = new CountDownLatch(1); + Thread t1 = new Thread(() -> { + try { + shotgun.await(); + for (int i = 0; i < 10; i++) { + clone.seek(0); + clone.readBytes(accum, 0, accum.length); + } + } catch (IOException | AlreadyClosedException ok) { + // OK + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + t1.start(); + shotgun.countDown(); + in.close(); + t1.join(); + dir.close(); + } + } } diff --git a/lucene/join/build.xml b/lucene/join/build.xml index b5360c4c7b9..b6878b809c6 100644 --- a/lucene/join/build.xml +++ b/lucene/join/build.xml @@ -26,6 +26,7 @@ + @@ -34,13 +35,14 @@ - + - + diff --git a/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java index a9b11ed7008..4bb692a50da 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java @@ -19,8 +19,6 @@ package org.apache.lucene.search.join; import java.io.IOException; import java.util.function.LongConsumer; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FieldType.LegacyNumericType; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; @@ -28,10 +26,11 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.legacy.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; abstract class DocValuesTermsCollector extends SimpleCollector { @@ -85,13 +84,13 @@ abstract class DocValuesTermsCollector extends SimpleCollector { return (l) -> LegacyNumericUtils.longToPrefixCoded(l, 0, bytes); default: throw new IllegalArgumentException("Unsupported "+type+ - ". Only "+ LegacyNumericType.INT+" and "+ FieldType.LegacyNumericType.LONG+" are supported." + ". Only "+ LegacyNumericType.INT+" and "+ LegacyNumericType.LONG+" are supported." + "Field "+fieldName ); } } /** this adapter is quite weird. ords are per doc index, don't use ords across different docs*/ - static Function sortedNumericAsSortedSetDocValues(String field, FieldType.LegacyNumericType numTyp) { + static Function sortedNumericAsSortedSetDocValues(String field, LegacyNumericType numTyp) { return (ctx) -> { final SortedNumericDocValues numerics = DocValues.getSortedNumeric(ctx, field); final BytesRefBuilder bytes = new BytesRefBuilder(); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java index b0133e570da..49423947383 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java @@ -26,7 +26,7 @@ import java.util.function.BiConsumer; import java.util.function.LongFunction; import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; @@ -123,8 +123,8 @@ public final class JoinUtil { * @param multipleValuesPerDocument Whether the from field has multiple terms per document * when true fromField might be {@link DocValuesType#SORTED_NUMERIC}, * otherwise fromField should be {@link DocValuesType#NUMERIC} - * @param toField The to field to join to, should be {@link org.apache.lucene.document.LegacyIntField} or {@link org.apache.lucene.document.LegacyLongField} - * @param numericType either {@link org.apache.lucene.document.FieldType.LegacyNumericType#INT} or {@link org.apache.lucene.document.FieldType.LegacyNumericType#LONG}, it should correspond to fromField and toField types + * @param toField The to field to join to, should be {@link org.apache.lucene.legacy.LegacyIntField} or {@link org.apache.lucene.legacy.LegacyLongField} + * @param numericType either {@link LegacyNumericType#INT} or {@link LegacyNumericType#LONG}, it should correspond to fromField and toField types * @param fromQuery The query to match documents on the from side * @param fromSearcher The searcher that executed the specified fromQuery * @param scoreMode Instructs how scores from the fromQuery are mapped to the returned query diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java index 3b03bd39c38..a39c25fcc93 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; @@ -37,7 +38,6 @@ import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash; import org.apache.lucene.util.FixedBitSet; -import org.apache.lucene.util.LegacyNumericUtils; class TermsIncludingScoreQuery extends Query { diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index b29e9ff0e9e..6d9eb2ab339 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -37,12 +37,9 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleDocValuesField; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType.LegacyNumericType; import org.apache.lucene.document.FloatDocValuesField; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; @@ -59,6 +56,9 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.MultiDocValues.OrdinalMap; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.NumericDocValues; diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java index 45e95510dc5..a7857207a39 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java @@ -45,7 +45,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; @@ -457,9 +456,6 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase { Document doc = new Document(); long randomLong = random().nextLong(); doc.add(new NumericDocValuesField("numeric", randomLong)); - if (random().nextBoolean()) { - doc.add(new LegacyLongField("numeric", randomLong, Field.Store.NO)); - } int numValues = atLeast(5); for (int i = 0; i < numValues; i++) { randomLong = random().nextLong(); @@ -468,9 +464,6 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase { // randomly duplicate field/value doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong)); } - if (random().nextBoolean()) { - doc.add(new LegacyLongField("numeric", randomLong, Field.Store.NO)); - } } BytesRef randomTerm = new BytesRef(randomTerm()); doc.add(new BinaryDocValuesField("binary", randomTerm)); diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java index da9fdc5dc5d..05a3b239959 100644 --- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java +++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java @@ -21,11 +21,10 @@ import java.util.HashMap; import java.util.Map; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatDocValuesField; -import org.apache.lucene.document.LegacyFloatField; import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInvertState; @@ -331,7 +330,7 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase { new BytesRef("")); Field weeksAtNumberOneField = new FloatDocValuesField("weeksAtNumberOne", 0.0F); - Field weeksStoredField = new LegacyFloatField("weeks", 0.0F, Store.YES); + Field weeksStoredField = new StoredField("weeks", 0.0F); Field idField = newStringField("id", "", Field.Store.YES); Field songField = newTextField("song", "", Field.Store.NO); Field storedArtistField = newTextField("artistName", "", Field.Store.NO); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java index 89722db075f..32a610bf8a9 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java @@ -265,6 +265,7 @@ public class TestMoreLikeThis extends LuceneTestCase { return writer.numDocs() - 1; } + @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-7161") public void testMultiFieldShouldReturnPerFieldBooleanQuery() throws Exception { IndexReader reader = null; Directory dir = newDirectory(); diff --git a/lucene/queryparser/build.xml b/lucene/queryparser/build.xml index b6e43c2ce26..f1d59a34a4c 100644 --- a/lucene/queryparser/build.xml +++ b/lucene/queryparser/build.xml @@ -25,15 +25,17 @@ + - + - + diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java index 8ae7d5e2e43..0781afb533c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java @@ -16,7 +16,8 @@ */ package org.apache.lucene.queryparser.flexible.standard.builders; -import org.apache.lucene.document.FieldType; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; @@ -25,12 +26,11 @@ import org.apache.lucene.queryparser.flexible.messages.MessageImpl; import org.apache.lucene.queryparser.flexible.standard.config.LegacyNumericConfig; import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericRangeQueryNode; -import org.apache.lucene.search.LegacyNumericRangeQuery; /** - * Builds {@link org.apache.lucene.search.LegacyNumericRangeQuery}s out of {@link LegacyNumericRangeQueryNode}s. + * Builds {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}s out of {@link LegacyNumericRangeQueryNode}s. * - * @see org.apache.lucene.search.LegacyNumericRangeQuery + * @see org.apache.lucene.legacy.LegacyNumericRangeQuery * @see LegacyNumericRangeQueryNode * @deprecated Index with points and use {@link PointRangeQueryNodeBuilder} instead. */ @@ -56,7 +56,7 @@ public class LegacyNumericRangeQueryNodeBuilder implements StandardQueryBuilder Number upperNumber = upperNumericNode.getValue(); LegacyNumericConfig numericConfig = numericRangeNode.getNumericConfig(); - FieldType.LegacyNumericType numberType = numericConfig.getType(); + LegacyNumericType numberType = numericConfig.getType(); String field = StringUtils.toString(numericRangeNode.getField()); boolean minInclusive = numericRangeNode.isLowerInclusive(); boolean maxInclusive = numericRangeNode.isUpperInclusive(); diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java index 6cd3c490e57..038023e65ae 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java @@ -19,14 +19,13 @@ package org.apache.lucene.queryparser.flexible.standard.config; import java.text.NumberFormat; import java.util.Objects; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericType; /** * This class holds the configuration used to parse numeric queries and create - * {@link org.apache.lucene.search.LegacyNumericRangeQuery}s. + * {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}s. * - * @see org.apache.lucene.search.LegacyNumericRangeQuery + * @see org.apache.lucene.legacy.LegacyNumericRangeQuery * @see NumberFormat * @deprecated Index with Points instead and use {@link PointsConfig} */ @@ -37,7 +36,7 @@ public class LegacyNumericConfig { private NumberFormat format; - private FieldType.LegacyNumericType type; + private LegacyNumericType type; /** * Constructs a {@link LegacyNumericConfig} object. @@ -52,7 +51,7 @@ public class LegacyNumericConfig { * * @see LegacyNumericConfig#setPrecisionStep(int) * @see LegacyNumericConfig#setNumberFormat(NumberFormat) - * @see #setType(org.apache.lucene.document.FieldType.LegacyNumericType) + * @see #setType(LegacyNumericType) */ public LegacyNumericConfig(int precisionStep, NumberFormat format, LegacyNumericType type) { @@ -67,7 +66,7 @@ public class LegacyNumericConfig { * * @return the precision used to index the numeric values * - * @see org.apache.lucene.search.LegacyNumericRangeQuery#getPrecisionStep() + * @see org.apache.lucene.legacy.LegacyNumericRangeQuery#getPrecisionStep() */ public int getPrecisionStep() { return precisionStep; @@ -79,7 +78,7 @@ public class LegacyNumericConfig { * @param precisionStep * the precision used to index the numeric values * - * @see org.apache.lucene.search.LegacyNumericRangeQuery#getPrecisionStep() + * @see org.apache.lucene.legacy.LegacyNumericRangeQuery#getPrecisionStep() */ public void setPrecisionStep(int precisionStep) { this.precisionStep = precisionStep; diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java index 27c285eb34c..20cde351ce7 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java @@ -16,8 +16,7 @@ */ package org.apache.lucene.queryparser.flexible.standard.nodes; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; import org.apache.lucene.queryparser.flexible.messages.MessageImpl; @@ -57,13 +56,13 @@ public class LegacyNumericRangeQueryNode extends private static LegacyNumericType getNumericDataType(Number number) throws QueryNodeException { if (number instanceof Long) { - return FieldType.LegacyNumericType.LONG; + return LegacyNumericType.LONG; } else if (number instanceof Integer) { - return FieldType.LegacyNumericType.INT; + return LegacyNumericType.INT; } else if (number instanceof Double) { return LegacyNumericType.DOUBLE; } else if (number instanceof Float) { - return FieldType.LegacyNumericType.FLOAT; + return LegacyNumericType.FLOAT; } else { throw new QueryNodeException( new MessageImpl( diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java index f7aef3f477b..9f4505f056c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java @@ -16,19 +16,19 @@ */ package org.apache.lucene.queryparser.xml.builders; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.util.LegacyNumericUtils; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queryparser.xml.DOMUtils; import org.apache.lucene.queryparser.xml.ParserException; import org.apache.lucene.queryparser.xml.QueryBuilder; import org.w3c.dom.Element; /** - * Creates a {@link org.apache.lucene.search.LegacyNumericRangeQuery}. The table below specifies the required + * Creates a {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}. The table below specifies the required * attributes and the defaults if optional attributes are omitted. For more * detail on what each of the attributes actually do, consult the documentation - * for {@link org.apache.lucene.search.LegacyNumericRangeQuery}: + * for {@link org.apache.lucene.legacy.LegacyNumericRangeQuery}: * * * diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java index c6ab7f5ffff..398923e299f 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java @@ -32,15 +32,15 @@ import java.util.TimeZone; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType.LegacyNumericType; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax; import org.apache.lucene.queryparser.flexible.standard.config.NumberDateFormat; @@ -179,7 +179,7 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { ; randomNumberMap.put(LegacyNumericType.LONG.name(), randomLong); - randomNumberMap.put(FieldType.LegacyNumericType.INT.name(), randomInt); + randomNumberMap.put(LegacyNumericType.INT.name(), randomInt); randomNumberMap.put(LegacyNumericType.FLOAT.name(), randomFloat); randomNumberMap.put(LegacyNumericType.DOUBLE.name(), randomDouble); randomNumberMap.put(DATE_FIELD_NAME, randomDate); @@ -201,7 +201,7 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { numericConfigMap.put(type.name(), new LegacyNumericConfig(PRECISION_STEP, NUMBER_FORMAT, type)); - FieldType ft = new FieldType(LegacyIntField.TYPE_NOT_STORED); + LegacyFieldType ft = new LegacyFieldType(LegacyIntField.TYPE_NOT_STORED); ft.setNumericType(type); ft.setStored(true); ft.setNumericPrecisionStep(PRECISION_STEP); @@ -231,7 +231,7 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { numericConfigMap.put(DATE_FIELD_NAME, new LegacyNumericConfig(PRECISION_STEP, DATE_FORMAT, LegacyNumericType.LONG)); - FieldType ft = new FieldType(LegacyLongField.TYPE_NOT_STORED); + LegacyFieldType ft = new LegacyFieldType(LegacyLongField.TYPE_NOT_STORED); ft.setStored(true); ft.setNumericPrecisionStep(PRECISION_STEP); LegacyLongField dateField = new LegacyLongField(DATE_FIELD_NAME, 0l, ft); @@ -268,10 +268,10 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { || DATE_FIELD_NAME.equals(fieldName)) { number = -number.longValue(); - } else if (FieldType.LegacyNumericType.DOUBLE.name().equals(fieldName)) { + } else if (LegacyNumericType.DOUBLE.name().equals(fieldName)) { number = -number.doubleValue(); - } else if (FieldType.LegacyNumericType.FLOAT.name().equals(fieldName)) { + } else if (LegacyNumericType.FLOAT.name().equals(fieldName)) { number = -number.floatValue(); } else if (LegacyNumericType.INT.name().equals(fieldName)) { @@ -299,16 +299,16 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { numericFieldMap.get(LegacyNumericType.DOUBLE.name()).setDoubleValue( number.doubleValue()); - number = getNumberType(numberType, FieldType.LegacyNumericType.INT.name()); - numericFieldMap.get(FieldType.LegacyNumericType.INT.name()).setIntValue( + number = getNumberType(numberType, LegacyNumericType.INT.name()); + numericFieldMap.get(LegacyNumericType.INT.name()).setIntValue( number.intValue()); number = getNumberType(numberType, LegacyNumericType.LONG.name()); - numericFieldMap.get(FieldType.LegacyNumericType.LONG.name()).setLongValue( + numericFieldMap.get(LegacyNumericType.LONG.name()).setLongValue( number.longValue()); - number = getNumberType(numberType, FieldType.LegacyNumericType.FLOAT.name()); - numericFieldMap.get(FieldType.LegacyNumericType.FLOAT.name()).setFloatValue( + number = getNumberType(numberType, LegacyNumericType.FLOAT.name()); + numericFieldMap.get(LegacyNumericType.FLOAT.name()).setFloatValue( number.floatValue()); number = getNumberType(numberType, DATE_FIELD_NAME); @@ -456,7 +456,7 @@ public class TestLegacyNumericQueryParser extends LuceneTestCase { StringBuilder sb = new StringBuilder(); - for (LegacyNumericType type : FieldType.LegacyNumericType.values()) { + for (LegacyNumericType type : LegacyNumericType.values()) { String boundStr = numberToString(getNumberType(boundType, type.name())); sb.append("+").append(type.name()).append(operator).append('"').append(boundStr).append('"').append(' '); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java index 71b627e74cd..4763005d985 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java @@ -20,10 +20,10 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.legacy.LegacyIntField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java index 8fc0641e4e7..0bc019595be 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java @@ -16,9 +16,9 @@ */ package org.apache.lucene.queryparser.xml.builders; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; import org.apache.lucene.queryparser.xml.ParserException; import org.w3c.dom.Document; import org.xml.sax.SAXException; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/FloatRangeField.java b/lucene/sandbox/src/java/org/apache/lucene/document/FloatRangeField.java new file mode 100644 index 00000000000..e138ae2057d --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/document/FloatRangeField.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.document.RangeFieldQuery.QueryType; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; + +/** + * An indexed Float Range field. + *

    + * This field indexes dimensional ranges defined as min/max pairs. It supports + * up to a maximum of 4 dimensions (indexed as 8 numeric values). With 1 dimension representing a single float range, + * 2 dimensions representing a bounding box, 3 dimensions a bounding cube, and 4 dimensions a tesseract. + *

    + * Multiple values for the same field in one document is supported, and open ended ranges can be defined using + * {@code Float.NEGATIVE_INFINITY} and {@code Float.POSITIVE_INFINITY}. + * + *

    + * This field defines the following static factory methods for common search operations over float ranges: + *

      + *
    • {@link #newIntersectsQuery newIntersectsQuery()} matches ranges that intersect the defined search range. + *
    • {@link #newWithinQuery newWithinQuery()} matches ranges that are within the defined search range. + *
    • {@link #newContainsQuery newContainsQuery()} matches ranges that contain the defined search range. + *
    + */ +public class FloatRangeField extends Field { + /** stores float values so number of bytes is 4 */ + public static final int BYTES = Float.BYTES; + + /** + * Create a new FloatRangeField type, from min/max parallel arrays + * + * @param name field name. must not be null. + * @param min range min values; each entry is the min value for the dimension + * @param max range max values; each entry is the max value for the dimension + */ + public FloatRangeField(String name, final float[] min, final float[] max) { + super(name, getType(min.length)); + setRangeValues(min, max); + } + + /** set the field type */ + private static FieldType getType(int dimensions) { + if (dimensions > 4) { + throw new IllegalArgumentException("FloatRangeField does not support greater than 4 dimensions"); + } + + FieldType ft = new FieldType(); + // dimensions is set as 2*dimension size (min/max per dimension) + ft.setDimensions(dimensions*2, BYTES); + ft.freeze(); + return ft; + } + + /** + * Changes the values of the field. + * @param min array of min values. (accepts {@code Float.NEGATIVE_INFINITY}) + * @param max array of max values. (accepts {@code Float.POSITIVE_INFINITY}) + * @throws IllegalArgumentException if {@code min} or {@code max} is invalid + */ + public void setRangeValues(float[] min, float[] max) { + checkArgs(min, max); + if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) { + throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2 + + " dimensions; cannot change to (incoming) " + min.length + " dimensions"); + } + + final byte[] bytes; + if (fieldsData == null) { + bytes = new byte[BYTES*2*min.length]; + fieldsData = new BytesRef(bytes); + } else { + bytes = ((BytesRef)fieldsData).bytes; + } + verifyAndEncode(min, max, bytes); + } + + /** validate the arguments */ + private static void checkArgs(final float[] min, final float[] max) { + if (min == null || max == null || min.length == 0 || max.length == 0) { + throw new IllegalArgumentException("min/max range values cannot be null or empty"); + } + if (min.length != max.length) { + throw new IllegalArgumentException("min/max ranges must agree"); + } + if (min.length > 4) { + throw new IllegalArgumentException("FloatRangeField does not support greater than 4 dimensions"); + } + } + + /** + * Encodes the min, max ranges into a byte array + */ + private static byte[] encode(float[] min, float[] max) { + checkArgs(min, max); + byte[] b = new byte[BYTES*2*min.length]; + verifyAndEncode(min, max, b); + return b; + } + + /** + * encode the ranges into a sortable byte array ({@code Float.NaN} not allowed) + *

    + * example for 4 dimensions (8 bytes per dimension value): + * minD1 ... minD4 | maxD1 ... maxD4 + */ + static void verifyAndEncode(float[] min, float[] max, byte[] bytes) { + for (int d=0,i=0,j=min.length*BYTES; d max[d]) { + throw new IllegalArgumentException("min value (" + min[d] + ") is greater than max value (" + max[d] + ")"); + } + encode(min[d], bytes, i); + encode(max[d], bytes, j); + } + } + + /** encode the given value into the byte array at the defined offset */ + private static void encode(float val, byte[] bytes, int offset) { + NumericUtils.intToSortableBytes(NumericUtils.floatToSortableInt(val), bytes, offset); + } + + /** + * Get the min value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded min value + */ + public float getMin(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMin(((BytesRef)fieldsData).bytes, dimension); + } + + /** + * Get the max value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded max value + */ + public float getMax(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMax(((BytesRef)fieldsData).bytes, dimension); + } + + /** decodes the min value (for the defined dimension) from the encoded input byte array */ + static float decodeMin(byte[] b, int dimension) { + int offset = dimension*BYTES; + return NumericUtils.sortableIntToFloat(NumericUtils.sortableBytesToInt(b, offset)); + } + + /** decodes the max value (for the defined dimension) from the encoded input byte array */ + static float decodeMax(byte[] b, int dimension) { + int offset = b.length/2 + dimension*BYTES; + return NumericUtils.sortableIntToFloat(NumericUtils.sortableBytesToInt(b, offset)); + } + + /** + * Create a query for matching indexed ranges that intersect the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Float.NEGATIVE_INFINITY}) + * @param max array of max values. (accepts {@code Float.MAX_VALUE}) + * @return query for matching intersecting ranges (overlap, within, or contains) + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newIntersectsQuery(String field, final float[] min, final float[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.INTERSECTS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return FloatRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed float ranges that contain the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Float.NEGATIVE_INFINITY}) + * @param max array of max values. (accepts {@code Float.POSITIVE_INFINITY}) + * @return query for matching ranges that contain the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newContainsQuery(String field, final float[] min, final float[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.CONTAINS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return FloatRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed ranges that are within the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Float.NEGATIVE_INFINITY}) + * @param max array of max values. (accepts {@code Float.POSITIVE_INFINITY}) + * @return query for matching ranges within the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newWithinQuery(String field, final float[] min, final float[] max) { + checkArgs(min, max); + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.WITHIN) { + @Override + protected String toString(byte[] ranges, int dimension) { + return FloatRangeField.toString(ranges, dimension); + } + }; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append(" <"); + sb.append(name); + sb.append(':'); + byte[] b = ((BytesRef)fieldsData).bytes; + toString(b, 0); + for (int d=1; d'); + + return sb.toString(); + } + + /** + * Returns the String representation for the range at the given dimension + * @param ranges the encoded ranges, never null + * @param dimension the dimension of interest + * @return The string representation for the range at the provided dimension + */ + private static String toString(byte[] ranges, int dimension) { + return "[" + Float.toString(decodeMin(ranges, dimension)) + " : " + + Float.toString(decodeMax(ranges, dimension)) + "]"; + } +} diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/IntRangeField.java b/lucene/sandbox/src/java/org/apache/lucene/document/IntRangeField.java new file mode 100644 index 00000000000..c0ce61d85e3 --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/document/IntRangeField.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.document.RangeFieldQuery.QueryType; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; + +/** + * An indexed Integer Range field. + *

    + * This field indexes dimensional ranges defined as min/max pairs. It supports + * up to a maximum of 4 dimensions (indexed as 8 numeric values). With 1 dimension representing a single integer range, + * 2 dimensions representing a bounding box, 3 dimensions a bounding cube, and 4 dimensions a tesseract. + *

    + * Multiple values for the same field in one document is supported, and open ended ranges can be defined using + * {@code Integer.MIN_VALUE} and {@code Integer.MAX_VALUE}. + * + *

    + * This field defines the following static factory methods for common search operations over integer ranges: + *

      + *
    • {@link #newIntersectsQuery newIntersectsQuery()} matches ranges that intersect the defined search range. + *
    • {@link #newWithinQuery newWithinQuery()} matches ranges that are within the defined search range. + *
    • {@link #newContainsQuery newContainsQuery()} matches ranges that contain the defined search range. + *
    + */ +public class IntRangeField extends Field { + /** stores integer values so number of bytes is 4 */ + public static final int BYTES = Integer.BYTES; + + /** + * Create a new IntRangeField type, from min/max parallel arrays + * + * @param name field name. must not be null. + * @param min range min values; each entry is the min value for the dimension + * @param max range max values; each entry is the max value for the dimension + */ + public IntRangeField(String name, final int[] min, final int[] max) { + super(name, getType(min.length)); + setRangeValues(min, max); + } + + /** set the field type */ + private static FieldType getType(int dimensions) { + if (dimensions > 4) { + throw new IllegalArgumentException("IntRangeField does not support greater than 4 dimensions"); + } + + FieldType ft = new FieldType(); + // dimensions is set as 2*dimension size (min/max per dimension) + ft.setDimensions(dimensions*2, BYTES); + ft.freeze(); + return ft; + } + + /** + * Changes the values of the field. + * @param min array of min values. (accepts {@code Integer.NEGATIVE_INFINITY}) + * @param max array of max values. (accepts {@code Integer.POSITIVE_INFINITY}) + * @throws IllegalArgumentException if {@code min} or {@code max} is invalid + */ + public void setRangeValues(int[] min, int[] max) { + checkArgs(min, max); + if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) { + throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2 + + " dimensions; cannot change to (incoming) " + min.length + " dimensions"); + } + + final byte[] bytes; + if (fieldsData == null) { + bytes = new byte[BYTES*2*min.length]; + fieldsData = new BytesRef(bytes); + } else { + bytes = ((BytesRef)fieldsData).bytes; + } + verifyAndEncode(min, max, bytes); + } + + /** validate the arguments */ + private static void checkArgs(final int[] min, final int[] max) { + if (min == null || max == null || min.length == 0 || max.length == 0) { + throw new IllegalArgumentException("min/max range values cannot be null or empty"); + } + if (min.length != max.length) { + throw new IllegalArgumentException("min/max ranges must agree"); + } + if (min.length > 4) { + throw new IllegalArgumentException("IntRangeField does not support greater than 4 dimensions"); + } + } + + /** + * Encodes the min, max ranges into a byte array + */ + private static byte[] encode(int[] min, int[] max) { + checkArgs(min, max); + byte[] b = new byte[BYTES*2*min.length]; + verifyAndEncode(min, max, b); + return b; + } + + /** + * encode the ranges into a sortable byte array ({@code Double.NaN} not allowed) + *

    + * example for 4 dimensions (8 bytes per dimension value): + * minD1 ... minD4 | maxD1 ... maxD4 + */ + static void verifyAndEncode(int[] min, int[] max, byte[] bytes) { + for (int d=0,i=0,j=min.length*BYTES; d max[d]) { + throw new IllegalArgumentException("min value (" + min[d] + ") is greater than max value (" + max[d] + ")"); + } + encode(min[d], bytes, i); + encode(max[d], bytes, j); + } + } + + /** encode the given value into the byte array at the defined offset */ + private static void encode(int val, byte[] bytes, int offset) { + NumericUtils.intToSortableBytes(val, bytes, offset); + } + + /** + * Get the min value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded min value + */ + public int getMin(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMin(((BytesRef)fieldsData).bytes, dimension); + } + + /** + * Get the max value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded max value + */ + public int getMax(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMax(((BytesRef)fieldsData).bytes, dimension); + } + + /** decodes the min value (for the defined dimension) from the encoded input byte array */ + static int decodeMin(byte[] b, int dimension) { + int offset = dimension*BYTES; + return NumericUtils.sortableBytesToInt(b, offset); + } + + /** decodes the max value (for the defined dimension) from the encoded input byte array */ + static int decodeMax(byte[] b, int dimension) { + int offset = b.length/2 + dimension*BYTES; + return NumericUtils.sortableBytesToInt(b, offset); + } + + /** + * Create a query for matching indexed ranges that intersect the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Integer.MIN_VALUE}) + * @param max array of max values. (accepts {@code Integer.MAX_VALUE}) + * @return query for matching intersecting ranges (overlap, within, or contains) + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newIntersectsQuery(String field, final int[] min, final int[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.INTERSECTS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return IntRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed ranges that contain the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Integer.MIN_VALUE}) + * @param max array of max values. (accepts {@code Integer.MAX_VALUE}) + * @return query for matching ranges that contain the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newContainsQuery(String field, final int[] min, final int[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.CONTAINS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return IntRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed ranges that are within the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Integer.MIN_VALUE}) + * @param max array of max values. (accepts {@code Integer.MAX_VALUE}) + * @return query for matching ranges within the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newWithinQuery(String field, final int[] min, final int[] max) { + checkArgs(min, max); + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.WITHIN) { + @Override + protected String toString(byte[] ranges, int dimension) { + return IntRangeField.toString(ranges, dimension); + } + }; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append(" <"); + sb.append(name); + sb.append(':'); + byte[] b = ((BytesRef)fieldsData).bytes; + toString(b, 0); + for (int d=1; d'); + + return sb.toString(); + } + + /** + * Returns the String representation for the range at the given dimension + * @param ranges the encoded ranges, never null + * @param dimension the dimension of interest + * @return The string representation for the range at the provided dimension + */ + private static String toString(byte[] ranges, int dimension) { + return "[" + Integer.toString(decodeMin(ranges, dimension)) + " : " + + Integer.toString(decodeMax(ranges, dimension)) + "]"; + } +} diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LongRangeField.java b/lucene/sandbox/src/java/org/apache/lucene/document/LongRangeField.java new file mode 100644 index 00000000000..b9298b9d8d3 --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LongRangeField.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.document.RangeFieldQuery.QueryType; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; + +/** + * An indexed Long Range field. + *

    + * This field indexes dimensional ranges defined as min/max pairs. It supports + * up to a maximum of 4 dimensions (indexed as 8 numeric values). With 1 dimension representing a single long range, + * 2 dimensions representing a bounding box, 3 dimensions a bounding cube, and 4 dimensions a tesseract. + *

    + * Multiple values for the same field in one document is supported, and open ended ranges can be defined using + * {@code Long.MIN_VALUE} and {@code Long.MAX_VALUE}. + * + *

    + * This field defines the following static factory methods for common search operations over long ranges: + *

      + *
    • {@link #newIntersectsQuery newIntersectsQuery()} matches ranges that intersect the defined search range. + *
    • {@link #newWithinQuery newWithinQuery()} matches ranges that are within the defined search range. + *
    • {@link #newContainsQuery newContainsQuery()} matches ranges that contain the defined search range. + *
    + */ +public class LongRangeField extends Field { + /** stores long values so number of bytes is 8 */ + public static final int BYTES = Long.BYTES; + + /** + * Create a new LongRangeField type, from min/max parallel arrays + * + * @param name field name. must not be null. + * @param min range min values; each entry is the min value for the dimension + * @param max range max values; each entry is the max value for the dimension + */ + public LongRangeField(String name, final long[] min, final long[] max) { + super(name, getType(min.length)); + setRangeValues(min, max); + } + + /** set the field type */ + private static FieldType getType(int dimensions) { + if (dimensions > 4) { + throw new IllegalArgumentException("LongRangeField does not support greater than 4 dimensions"); + } + + FieldType ft = new FieldType(); + // dimensions is set as 2*dimension size (min/max per dimension) + ft.setDimensions(dimensions*2, BYTES); + ft.freeze(); + return ft; + } + + /** + * Changes the values of the field. + * @param min array of min values. (accepts {@code Long.MIN_VALUE}) + * @param max array of max values. (accepts {@code Long.MAX_VALUE}) + * @throws IllegalArgumentException if {@code min} or {@code max} is invalid + */ + public void setRangeValues(long[] min, long[] max) { + checkArgs(min, max); + if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) { + throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2 + + " dimensions; cannot change to (incoming) " + min.length + " dimensions"); + } + + final byte[] bytes; + if (fieldsData == null) { + bytes = new byte[BYTES*2*min.length]; + fieldsData = new BytesRef(bytes); + } else { + bytes = ((BytesRef)fieldsData).bytes; + } + verifyAndEncode(min, max, bytes); + } + + /** validate the arguments */ + private static void checkArgs(final long[] min, final long[] max) { + if (min == null || max == null || min.length == 0 || max.length == 0) { + throw new IllegalArgumentException("min/max range values cannot be null or empty"); + } + if (min.length != max.length) { + throw new IllegalArgumentException("min/max ranges must agree"); + } + if (min.length > 4) { + throw new IllegalArgumentException("LongRangeField does not support greater than 4 dimensions"); + } + } + + /** Encodes the min, max ranges into a byte array */ + private static byte[] encode(long[] min, long[] max) { + checkArgs(min, max); + byte[] b = new byte[BYTES*2*min.length]; + verifyAndEncode(min, max, b); + return b; + } + + /** + * encode the ranges into a sortable byte array ({@code Double.NaN} not allowed) + *

    + * example for 4 dimensions (8 bytes per dimension value): + * minD1 ... minD4 | maxD1 ... maxD4 + */ + static void verifyAndEncode(long[] min, long[] max, byte[] bytes) { + for (int d=0,i=0,j=min.length*BYTES; d max[d]) { + throw new IllegalArgumentException("min value (" + min[d] + ") is greater than max value (" + max[d] + ")"); + } + encode(min[d], bytes, i); + encode(max[d], bytes, j); + } + } + + /** encode the given value into the byte array at the defined offset */ + private static void encode(long val, byte[] bytes, int offset) { + NumericUtils.longToSortableBytes(val, bytes, offset); + } + + /** + * Get the min value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded min value + */ + public long getMin(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMin(((BytesRef)fieldsData).bytes, dimension); + } + + /** + * Get the max value for the given dimension + * @param dimension the dimension, always positive + * @return the decoded max value + */ + public long getMax(int dimension) { + if (dimension < 0 || dimension >= type.pointDimensionCount()/2) { + throw new IllegalArgumentException("dimension request (" + dimension + + ") out of bounds for field (name=" + name + " dimensions=" + type.pointDimensionCount()/2 + "). "); + } + return decodeMax(((BytesRef)fieldsData).bytes, dimension); + } + + /** decodes the min value (for the defined dimension) from the encoded input byte array */ + static long decodeMin(byte[] b, int dimension) { + int offset = dimension*BYTES; + return NumericUtils.sortableBytesToLong(b, offset); + } + + /** decodes the max value (for the defined dimension) from the encoded input byte array */ + static long decodeMax(byte[] b, int dimension) { + int offset = b.length/2 + dimension*BYTES; + return NumericUtils.sortableBytesToLong(b, offset); + } + + /** + * Create a query for matching indexed ranges that intersect the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Long.MIN_VALUE}) + * @param max array of max values. (accepts {@code Long.MAX_VALUE}) + * @return query for matching intersecting ranges (overlap, within, or contains) + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newIntersectsQuery(String field, final long[] min, final long[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.INTERSECTS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return LongRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed ranges that contain the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Long.MIN_VALUE}) + * @param max array of max values. (accepts {@code Long.MAX_VALUE}) + * @return query for matching ranges that contain the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newContainsQuery(String field, final long[] min, final long[] max) { + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.CONTAINS) { + @Override + protected String toString(byte[] ranges, int dimension) { + return LongRangeField.toString(ranges, dimension); + } + }; + } + + /** + * Create a query for matching indexed ranges that are within the defined range. + * @param field field name. must not be null. + * @param min array of min values. (accepts {@code Long.MIN_VALUE}) + * @param max array of max values. (accepts {@code Long.MAX_VALUE}) + * @return query for matching ranges within the defined range + * @throws IllegalArgumentException if {@code field} is null, {@code min} or {@code max} is invalid + */ + public static Query newWithinQuery(String field, final long[] min, final long[] max) { + checkArgs(min, max); + return new RangeFieldQuery(field, encode(min, max), min.length, QueryType.WITHIN) { + @Override + protected String toString(byte[] ranges, int dimension) { + return LongRangeField.toString(ranges, dimension); + } + }; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append(" <"); + sb.append(name); + sb.append(':'); + byte[] b = ((BytesRef)fieldsData).bytes; + toString(b, 0); + for (int d=1; d'); + + return sb.toString(); + } + + /** + * Returns the String representation for the range at the given dimension + * @param ranges the encoded ranges, never null + * @param dimension the dimension of interest + * @return The string representation for the range at the provided dimension + */ + private static String toString(byte[] ranges, int dimension) { + return "[" + Long.toString(decodeMin(ranges, dimension)) + " : " + + Long.toString(decodeMax(ranges, dimension)) + "]"; + } +} diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java b/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java index d9cb830c120..9d293305c70 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java @@ -17,7 +17,6 @@ package org.apache.lucene.search; import java.io.IOException; -import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -41,16 +40,18 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; /** - * Abstract class to do basic tests for a RangeField query. + * Abstract class to do basic tests for a RangeField query. Testing rigor inspired by {@code BaseGeoPointTestCase} */ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase { - protected abstract Field newRangeField(double[] min, double[] max); + protected abstract Field newRangeField(Range box); - protected abstract Query newIntersectsQuery(double[] min, double[] max); + protected abstract Query newIntersectsQuery(Range box); - protected abstract Query newContainsQuery(double[] min, double[] max); + protected abstract Query newContainsQuery(Range box); - protected abstract Query newWithinQuery(double[] min, double[] max); + protected abstract Query newWithinQuery(Range box); + + protected abstract Range nextRange(int dimensions); protected int dimension() { return random().nextInt(4) + 1; @@ -82,18 +83,18 @@ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase { System.out.println("TEST: numDocs=" + numDocs); } - Box[][] boxes = new Box[numDocs][]; + Range[][] ranges = new Range[numDocs][]; boolean haveRealDoc = true; nextdoc: for (int id=0; id 0 && x < 9 && haveRealDoc) { int oldID; int i=0; - // don't step on missing boxes: + // don't step on missing ranges: while (true) { oldID = random().nextInt(id); - if (Double.isNaN(boxes[oldID][0].min[0]) == false) { + if (ranges[oldID][0].isMissing == false) { break; } else if (++i > id) { continue nextdoc; @@ -125,11 +126,11 @@ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase { if (x == dimensions*2) { // Fully identical box (use first box in case current is multivalued but old is not) for (int d=0; d 50000) { + if (ranges.length > 50000) { dir = newFSDirectory(createTempDir(getClass().getSimpleName())); } else { dir = newDirectory(); @@ -173,13 +174,13 @@ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase { Set deleted = new HashSet<>(); IndexWriter w = new IndexWriter(dir, iwc); - for (int id=0; id < boxes.length; ++id) { + for (int id=0; id < ranges.length; ++id) { Document doc = new Document(); doc.add(newStringField("id", ""+id, Field.Store.NO)); doc.add(new NumericDocValuesField("id", id)); - if (Double.isNaN(boxes[id][0].min[0]) == false) { - for (int n=0; n 1) ? "es=" : "=" ) + boxes[id][0]); - for (int n=1; n 1) ? "es=" : "=" ) + ranges[id][0]); + for (int n=1; n other.max[d] || this.max[d] < other.min[d]) { - // disjoint: - return null; - } - } - - // check within - boolean within = true; - for (int d=0; d= other.min[d] && this.max[d] <= other.max[d]) == false) { - // not within: - within = false; - break; - } - } - if (within == true) { + protected QueryType relate(Range other) { + if (isDisjoint(other)) { + // if disjoint; return null: + return null; + } else if (isWithin(other)) { return QueryType.WITHIN; - } - - // check contains - boolean contains = true; - for (int d=0; d= other.max[d]) == false) { - // not contains: - contains = false; - break; - } - } - if (contains == true) { + } else if (contains(other)) { return QueryType.CONTAINS; } return QueryType.INTERSECTS; } - - @Override - public String toString() { - StringBuilder b = new StringBuilder(); - b.append("Box("); - b.append(min[0]); - b.append(" TO "); - b.append(max[0]); - for (int d=1; d 0 && max.length > 0 + : "test box: min/max cannot be null or empty"; + assert min.length == max.length : "test box: min/max length do not agree"; + this.min = new double[min.length]; + this.max = new double[max.length]; + for (int d=0; d max[d]) { + // swap if max < min: + double temp = min[d]; + min[d] = max[d]; + max[d] = temp; + } + } + } + + @Override + protected int numDimensions() { + return min.length; + } + + @Override + protected Double getMin(int dim) { + return min[dim]; + } + + @Override + protected void setMin(int dim, Object val) { + min[dim] = (Double)val; + } + + @Override + protected Double getMax(int dim) { + return max[dim]; + } + + @Override + protected void setMax(int dim, Object val) { + max[dim] = (Double)val; + } + + @Override + protected boolean isEqual(Range other) { + DoubleRange o = (DoubleRange)other; + return Arrays.equals(min, o.min) && Arrays.equals(max, o.max); + } + + @Override + protected boolean isDisjoint(Range o) { + DoubleRange other = (DoubleRange)o; + for (int d=0; d other.max[d] || this.max[d] < other.min[d]) { + // disjoint: + return true; + } + } + return false; + } + + @Override + protected boolean isWithin(Range o) { + DoubleRange other = (DoubleRange)o; + for (int d=0; d= other.min[d] && this.max[d] <= other.max[d]) == false) { + // not within: + return false; + } + } + return true; + } + + @Override + protected boolean contains(Range o) { + DoubleRange other = (DoubleRange) o; + for (int d=0; d= other.max[d]) == false) { + // not contains: + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("Box("); + b.append(min[0]); + b.append(" TO "); + b.append(max[0]); + for (int d=1; d 0 && max.length > 0 + : "test box: min/max cannot be null or empty"; + assert min.length == max.length : "test box: min/max length do not agree"; + this.min = new float[min.length]; + this.max = new float[max.length]; + for (int d=0; d max[d]) { + // swap if max < min: + float temp = min[d]; + min[d] = max[d]; + max[d] = temp; + } + } + } + + @Override + protected int numDimensions() { + return min.length; + } + + @Override + protected Float getMin(int dim) { + return min[dim]; + } + + @Override + protected void setMin(int dim, Object val) { + min[dim] = (Float)val; + } + + @Override + protected Float getMax(int dim) { + return max[dim]; + } + + @Override + protected void setMax(int dim, Object val) { + max[dim] = (Float)val; + } + + @Override + protected boolean isEqual(Range other) { + FloatRange o = (FloatRange)other; + return Arrays.equals(min, o.min) && Arrays.equals(max, o.max); + } + + @Override + protected boolean isDisjoint(Range o) { + FloatRange other = (FloatRange)o; + for (int d=0; d other.max[d] || this.max[d] < other.min[d]) { + // disjoint: + return true; + } + } + return false; + } + + @Override + protected boolean isWithin(Range o) { + FloatRange other = (FloatRange)o; + for (int d=0; d= other.min[d] && this.max[d] <= other.max[d]) == false) { + // not within: + return false; + } + } + return true; + } + + @Override + protected boolean contains(Range o) { + FloatRange other = (FloatRange) o; + for (int d=0; d= other.max[d]) == false) { + // not contains: + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("Box("); + b.append(min[0]); + b.append(" TO "); + b.append(max[0]); + for (int d=1; d 0 && max.length > 0 + : "test box: min/max cannot be null or empty"; + assert min.length == max.length : "test box: min/max length do not agree"; + this.min = new int[min.length]; + this.max = new int[max.length]; + for (int d=0; d max[d]) { + // swap if max < min: + int temp = min[d]; + min[d] = max[d]; + max[d] = temp; + } + } + } + + @Override + protected int numDimensions() { + return min.length; + } + + @Override + protected Integer getMin(int dim) { + return min[dim]; + } + + @Override + protected void setMin(int dim, Object val) { + min[dim] = (Integer)val; + } + + @Override + protected Integer getMax(int dim) { + return max[dim]; + } + + @Override + protected void setMax(int dim, Object val) { + max[dim] = (Integer)val; + } + + @Override + protected boolean isEqual(Range other) { + IntRange o = (IntRange)other; + return Arrays.equals(min, o.min) && Arrays.equals(max, o.max); + } + + @Override + protected boolean isDisjoint(Range o) { + IntRange other = (IntRange)o; + for (int d=0; d other.max[d] || this.max[d] < other.min[d]) { + // disjoint: + return true; + } + } + return false; + } + + @Override + protected boolean isWithin(Range o) { + IntRange other = (IntRange)o; + for (int d=0; d= other.min[d] && this.max[d] <= other.max[d]) == false) { + // not within: + return false; + } + } + return true; + } + + @Override + protected boolean contains(Range o) { + IntRange other = (IntRange) o; + for (int d=0; d= other.max[d]) == false) { + // not contains: + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("Box("); + b.append(min[0]); + b.append(" TO "); + b.append(max[0]); + for (int d=1; d 0 && max.length > 0 + : "test box: min/max cannot be null or empty"; + assert min.length == max.length : "test box: min/max length do not agree"; + this.min = new long[min.length]; + this.max = new long[max.length]; + for (int d=0; d max[d]) { + // swap if max < min: + long temp = min[d]; + min[d] = max[d]; + max[d] = temp; + } + } + } + + @Override + protected int numDimensions() { + return min.length; + } + + @Override + protected Long getMin(int dim) { + return min[dim]; + } + + @Override + protected void setMin(int dim, Object val) { + min[dim] = (Long)val; + } + + @Override + protected Long getMax(int dim) { + return max[dim]; + } + + @Override + protected void setMax(int dim, Object val) { + max[dim] = (Long)val; + } + + @Override + protected boolean isEqual(Range other) { + LongRange o = (LongRange)other; + return Arrays.equals(min, o.min) && Arrays.equals(max, o.max); + } + + @Override + protected boolean isDisjoint(Range o) { + LongRange other = (LongRange)o; + for (int d=0; d other.max[d] || this.max[d] < other.min[d]) { + // disjoint: + return true; + } + } + return false; + } + + @Override + protected boolean isWithin(Range o) { + LongRange other = (LongRange)o; + for (int d=0; d= other.min[d] && this.max[d] <= other.max[d]) == false) { + // not within: + return false; + } + } + return true; + } + + @Override + protected boolean contains(Range o) { + LongRange other = (LongRange) o; + for (int d=0; d= other.max[d]) == false) { + // not contains: + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("Box("); + b.append(min[0]); + b.append(" TO "); + b.append(max[0]); + for (int d=1; d + @@ -42,16 +43,17 @@ - + - + - \ No newline at end of file + diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java index 63a113839b5..90e36d835db 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java @@ -20,17 +20,20 @@ import org.apache.lucene.document.DoubleDocValuesField; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.spatial.SpatialStrategy; @@ -39,7 +42,6 @@ import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.query.UnsupportedSpatialOperation; import org.apache.lucene.spatial.util.DistanceToShapeValueSource; import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; import org.locationtech.spatial4j.context.SpatialContext; import org.locationtech.spatial4j.shape.Point; @@ -87,7 +89,7 @@ public class BBoxStrategy extends SpatialStrategy { public static FieldType DEFAULT_FIELDTYPE; @Deprecated - public static FieldType LEGACY_FIELDTYPE; + public static LegacyFieldType LEGACY_FIELDTYPE; static { // Default: pointValues + docValues FieldType type = new FieldType(); @@ -97,14 +99,14 @@ public class BBoxStrategy extends SpatialStrategy { type.freeze(); DEFAULT_FIELDTYPE = type; // Legacy default: legacyNumerics + docValues - type = new FieldType(); - type.setIndexOptions(IndexOptions.DOCS); - type.setNumericType(FieldType.LegacyNumericType.DOUBLE); - type.setNumericPrecisionStep(8);// same as solr default - type.setDocValuesType(DocValuesType.NUMERIC);//docValues - type.setStored(false); - type.freeze(); - LEGACY_FIELDTYPE = type; + LegacyFieldType legacyType = new LegacyFieldType(); + legacyType.setIndexOptions(IndexOptions.DOCS); + legacyType.setNumericType(LegacyNumericType.DOUBLE); + legacyType.setNumericPrecisionStep(8);// same as solr default + legacyType.setDocValuesType(DocValuesType.NUMERIC);//docValues + legacyType.setStored(false); + legacyType.freeze(); + LEGACY_FIELDTYPE = legacyType; } public static final String SUFFIX_MINX = "__minX"; @@ -130,7 +132,7 @@ public class BBoxStrategy extends SpatialStrategy { private final boolean hasDocVals; private final boolean hasPointVals; // equiv to "hasLegacyNumerics": - private final FieldType legacyNumericFieldType; // not stored; holds precision step. + private final LegacyFieldType legacyNumericFieldType; // not stored; holds precision step. private final FieldType xdlFieldType; /** @@ -177,16 +179,17 @@ public class BBoxStrategy extends SpatialStrategy { if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) { numQuads++; } - if (fieldType.indexOptions() != IndexOptions.NONE && fieldType.numericType() != null) { + if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) { if (hasPointVals) { throw new IllegalArgumentException("pointValues and LegacyNumericType are mutually exclusive"); } - if (fieldType.numericType() != FieldType.LegacyNumericType.DOUBLE) { - throw new IllegalArgumentException(getClass() + " does not support " + fieldType.numericType()); + final LegacyFieldType legacyType = (LegacyFieldType) fieldType; + if (legacyType.numericType() != LegacyNumericType.DOUBLE) { + throw new IllegalArgumentException(getClass() + " does not support " + legacyType.numericType()); } numQuads++; - legacyNumericFieldType = new FieldType(LegacyDoubleField.TYPE_NOT_STORED); - legacyNumericFieldType.setNumericPrecisionStep(fieldType.numericPrecisionStep()); + legacyNumericFieldType = new LegacyFieldType(LegacyDoubleField.TYPE_NOT_STORED); + legacyNumericFieldType.setNumericPrecisionStep(legacyType.numericPrecisionStep()); legacyNumericFieldType.freeze(); } else { legacyNumericFieldType = null; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java index e724ab05fe6..757e2bd38f7 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java @@ -26,7 +26,7 @@ import org.apache.lucene.util.BytesRefIterator; /** * A TokenStream used internally by {@link org.apache.lucene.spatial.prefix.PrefixTreeStrategy}. * - * This is modelled after {@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * This is modelled after {@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * * @lucene.internal */ diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java index 197547c1d56..59aff490916 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java @@ -20,16 +20,18 @@ import org.apache.lucene.document.DoubleDocValuesField; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queries.function.FunctionRangeQuery; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.SpatialStrategy; import org.apache.lucene.spatial.query.SpatialArgs; @@ -85,7 +87,7 @@ public class PointVectorStrategy extends SpatialStrategy { public static FieldType DEFAULT_FIELDTYPE; @Deprecated - public static FieldType LEGACY_FIELDTYPE; + public static LegacyFieldType LEGACY_FIELDTYPE; static { // Default: pointValues + docValues FieldType type = new FieldType(); @@ -95,14 +97,14 @@ public class PointVectorStrategy extends SpatialStrategy { type.freeze(); DEFAULT_FIELDTYPE = type; // Legacy default: legacyNumerics - type = new FieldType(); - type.setIndexOptions(IndexOptions.DOCS); - type.setNumericType(FieldType.LegacyNumericType.DOUBLE); - type.setNumericPrecisionStep(8);// same as solr default - type.setDocValuesType(DocValuesType.NONE);//no docValues! - type.setStored(false); - type.freeze(); - LEGACY_FIELDTYPE = type; + LegacyFieldType legacyType = new LegacyFieldType(); + legacyType.setIndexOptions(IndexOptions.DOCS); + legacyType.setNumericType(LegacyNumericType.DOUBLE); + legacyType.setNumericPrecisionStep(8);// same as solr default + legacyType.setDocValuesType(DocValuesType.NONE);//no docValues! + legacyType.setStored(false); + legacyType.freeze(); + LEGACY_FIELDTYPE = legacyType; } public static final String SUFFIX_X = "__x"; @@ -116,7 +118,7 @@ public class PointVectorStrategy extends SpatialStrategy { private final boolean hasDocVals; private final boolean hasPointVals; // equiv to "hasLegacyNumerics": - private final FieldType legacyNumericFieldType; // not stored; holds precision step. + private final LegacyFieldType legacyNumericFieldType; // not stored; holds precision step. /** * Create a new {@link PointVectorStrategy} instance that uses {@link DoublePoint} and {@link DoublePoint#newRangeQuery} @@ -157,16 +159,17 @@ public class PointVectorStrategy extends SpatialStrategy { if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) { numPairs++; } - if (fieldType.indexOptions() != IndexOptions.NONE && fieldType.numericType() != null) { + if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) { if (hasPointVals) { throw new IllegalArgumentException("pointValues and LegacyNumericType are mutually exclusive"); } - if (fieldType.numericType() != FieldType.LegacyNumericType.DOUBLE) { - throw new IllegalArgumentException(getClass() + " does not support " + fieldType.numericType()); + final LegacyFieldType legacyType = (LegacyFieldType) fieldType; + if (legacyType.numericType() != LegacyNumericType.DOUBLE) { + throw new IllegalArgumentException(getClass() + " does not support " + legacyType.numericType()); } numPairs++; - legacyNumericFieldType = new FieldType(LegacyDoubleField.TYPE_NOT_STORED); - legacyNumericFieldType.setNumericPrecisionStep(fieldType.numericPrecisionStep()); + legacyNumericFieldType = new LegacyFieldType(LegacyDoubleField.TYPE_NOT_STORED); + legacyNumericFieldType.setNumericPrecisionStep(legacyType.numericPrecisionStep()); legacyNumericFieldType.freeze(); } else { legacyNumericFieldType = null; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java index 01e925926d2..20df7305cbe 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java @@ -22,6 +22,7 @@ import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.legacy.LegacyFieldType; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.SpatialMatchConcern; import org.apache.lucene.spatial.prefix.RandomSpatialOpStrategyTestCase; @@ -100,7 +101,12 @@ public class TestBBoxStrategy extends RandomSpatialOpStrategyTestCase { } //test we can disable docValues for predicate tests if (random().nextBoolean()) { - FieldType fieldType = new FieldType(((BBoxStrategy)strategy).getFieldType()); + FieldType fieldType = ((BBoxStrategy)strategy).getFieldType(); + if (fieldType instanceof LegacyFieldType) { + fieldType = new LegacyFieldType((LegacyFieldType)fieldType); + } else { + fieldType = new FieldType(fieldType); + } fieldType.setDocValuesType(DocValuesType.NONE); strategy = new BBoxStrategy(ctx, strategy.getFieldName(), fieldType); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index 1ff9470fb97..d15476ae7f3 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -838,7 +838,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { } // RuntimeException instead of IOException because // super() does not throw IOException currently: - throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause); + throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still " + openFiles.size() + " open files: " + openFiles, cause); } if (openLocks.size() > 0) { Exception cause = null; diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java index b63216085b3..19fcb3bfffb 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java @@ -177,11 +177,14 @@ public final class TestUtil { assert hasNext; T v = iterator.next(); assert allowNull || v != null; - try { - iterator.remove(); - throw new AssertionError("broken iterator (supports remove): " + iterator); - } catch (UnsupportedOperationException expected) { - // ok + // for the first element, check that remove is not supported + if (i == 0) { + try { + iterator.remove(); + throw new AssertionError("broken iterator (supports remove): " + iterator); + } catch (UnsupportedOperationException expected) { + // ok + } } } assert !iterator.hasNext(); diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy index f1d8f106dc2..2a623b70cea 100644 --- a/lucene/tools/junit4/tests.policy +++ b/lucene/tools/junit4/tests.policy @@ -28,10 +28,6 @@ grant { // should be enclosed within common.dir, but just in case: permission java.io.FilePermission "${junit4.childvm.cwd}", "read"; - // jenkins wants to read outside its sandbox, to use a special linedocs file. - // this is best effort and not really supported. - permission java.io.FilePermission "/home/jenkins/lucene-data/enwiki.random.lines.txt", "read"; - // write only to sandbox permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp", "read,write,delete"; permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp${/}-", "read,write,delete"; diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 68834576ad4..519472616db 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -50,6 +50,23 @@ Optimizations check on every request and move connection lifecycle management towards the client. (Ryan Zezeski, Mark Miller, Shawn Heisey, Steve Davids) +================== 6.3.0 ================== + +Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. + +Versions of Major Components +--------------------- +Apache Tika 1.13 +Carrot2 3.12.0 +Velocity 1.7 and Velocity Tools 2.0 +Apache UIMA 2.3.1 +Apache ZooKeeper 3.4.6 +Jetty 9.3.8.v20160314 + + +(No Changes) + + ================== 6.2.0 ================== Versions of Major Components @@ -119,6 +136,11 @@ New Features * SOLR-6465: CDCR: fall back to whole-index replication when tlogs are insufficient. (Noble Paul, Renaud Delbru, shalin) +* SOLR-9320: A REPLACENODE command to decommission an existing node with another new node + (noble, Nitin Sharma, Varun Thacker) + +* SOLR-9318: A DELETENODE command to delete all replicas in that node (noble, Nitin Sharma, Varun Thacker) + Bug Fixes ---------------------- @@ -190,10 +212,32 @@ Bug Fixes * SOLR-8379: UI Cloud->Tree view now shows .txt files correctly (Alexandre Rafalovitch via janhoy) +* SOLR-9003: New Admin UI's Dataimport screen now correctly displays DIH Debug output (Alexandre Rafalovitch) + * SOLR-9308: Fix distributed RTG to forward request params, fixes fq and non-default fl params (hossman) * SOLR-9179: NPE in IndexSchema using IBM JDK (noble, Colvin Cowie) +* SOLR-9397: Config API does not support adding caches (noble) + +* SOLR-9405: ConcurrentModificationException in ZkStateReader.getStateWatchers. + (Alan Woodward, Edward Ribeiro, shalin) + +* SOLR-9232: Admin UI now fully implements Swap Cores interface (Alexandre Rafalovitch) + +* SOLR-8715: Admin UI's Schema screen now works for fields with stored=false and some content indexed (Alexandre Rafalovitch) + +* SOLR-8911: In Admin UI, enable scrolling for overflowing Versions and JVM property values (Alexandre Rafalovitch) + +* SOLR-9002: Admin UI now correctly displays json and text files in the collection/Files screen (Upayavira, Alexandre Rafalovitch) + +* SOLR-8993: Admin UI now correctly supports multiple DIH handler end-points (Upayavira, Alexandre Rafalovitch) + +* SOLR-9032: Admin UI now correctly implements Create Alias command (Upayavira, Alexandre Rafalovitch) + +* SOLR-9391: LBHttpSolrClient.request now correctly returns Rsp.server when + previously skipped servers were successfully tried. (Christine Poerschke) + Optimizations ---------------------- @@ -249,6 +293,21 @@ Other Changes * SOLR-9367: Improved TestInjection's randomization logic to use LuceneTestCase.random() (hossman) +* SOLR-9331: Remove ReRankQuery's length constructor argument and member. (Christine Poerschke) + +* SOLR-9092: For the delete replica command we attempt to send the core admin delete request only + if that node is actually up. (Jessica Cheng Mallet, Varun Thacker) + +* SOLR-9410: Make ReRankQParserPlugin's private ReRankWeight a public class of its own. (Christine Poerschke) + +* SOLR-9404: Refactor move/renames in JSON FacetProcessor and FacetFieldProcessor. (David Smiley) + +* SOLR-9421: Refactored out OverseerCollectionMessageHandler to smaller classes (noble) + +* SOLR-8643: BlockJoinFacetComponent is substituted by BlockJoinFacetDocSetComponent. It doesn't need to change solrconfig.xml (Mikhail Khludnev) + +* SOLR-8644: Test asserts that block join facets work with parent level fq exclusions. (Dr. Oleg Savrasov via Mikhail Khludnev) + ================== 6.1.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java index 7a7e697d059..aadb9e2d4ce 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java @@ -20,8 +20,8 @@ import java.io.IOException; import java.time.Instant; import java.util.Arrays; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.TrieDateField; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java index 4d66e0025bc..22dde4c00ab 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java @@ -24,12 +24,12 @@ import java.util.Map; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.LongDocValues; import org.apache.lucene.queries.function.valuesource.LongFieldSource; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueDate; diff --git a/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java new file mode 100644 index 00000000000..6bb33508edb --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + + +import java.lang.invoke.MethodHandles; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.component.ShardHandler; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.cloud.Assign.getNodesForNewReplicas; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SKIP_CREATE_REPLICA_IN_CLUSTER_STATE; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; + +public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final OverseerCollectionMessageHandler ocmh; + + public AddReplicaCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + addReplica(ocmh.zkStateReader.getClusterState(), message, results, null); + } + + ZkNodeProps addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete) + throws KeeperException, InterruptedException { + log.info("addReplica() : {}", Utils.toJSONString(message)); + String collection = message.getStr(COLLECTION_PROP); + String node = message.getStr(CoreAdminParams.NODE); + String shard = message.getStr(SHARD_ID_PROP); + String coreName = message.getStr(CoreAdminParams.NAME); + boolean parallel = message.getBool("parallel", false); + if (StringUtils.isBlank(coreName)) { + coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME); + } + + final String asyncId = message.getStr(ASYNC); + + DocCollection coll = clusterState.getCollection(collection); + if (coll == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collection + " does not exist"); + } + if (coll.getSlice(shard) == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Collection: " + collection + " shard: " + shard + " does not exist"); + } + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false); + + // Kind of unnecessary, but it does put the logic of whether to override maxShardsPerNode in one place. + if (!skipCreateReplicaInClusterState) { + node = getNodesForNewReplicas(clusterState, collection, shard, 1, node, + ocmh.overseer.getZkController().getCoreContainer()).get(0).nodeName; + } + log.info("Node Identified {} for creating new replica", node); + + if (!clusterState.liveNodesContain(node)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Node: " + node + " is not live"); + } + if (coreName == null) { + coreName = Assign.buildCoreName(coll, shard); + } else if (!skipCreateReplicaInClusterState) { + //Validate that the core name is unique in that collection + for (Slice slice : coll.getSlices()) { + for (Replica replica : slice.getReplicas()) { + String replicaCoreName = replica.getStr(CORE_NAME_PROP); + if (coreName.equals(replicaCoreName)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Another replica with the same core name already exists" + + " for this collection"); + } + } + } + } + ModifiableSolrParams params = new ModifiableSolrParams(); + + ZkStateReader zkStateReader = ocmh.zkStateReader; + if (!Overseer.isLegacy(zkStateReader)) { + if (!skipCreateReplicaInClusterState) { + ZkNodeProps props = new ZkNodeProps( + Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(), + ZkStateReader.COLLECTION_PROP, collection, + ZkStateReader.SHARD_ID_PROP, shard, + ZkStateReader.CORE_NAME_PROP, coreName, + ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), + ZkStateReader.BASE_URL_PROP, zkStateReader.getBaseUrlForNodeName(node), + ZkStateReader.NODE_NAME_PROP, node); + Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(props)); + } + params.set(CoreAdminParams.CORE_NODE_NAME, + ocmh.waitToSeeReplicasInState(collection, Collections.singletonList(coreName)).get(coreName).getName()); + } + + String configName = zkStateReader.readConfigName(collection); + String routeKey = message.getStr(ShardParams._ROUTE_); + String dataDir = message.getStr(CoreAdminParams.DATA_DIR); + String instanceDir = message.getStr(CoreAdminParams.INSTANCE_DIR); + + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATE.toString()); + params.set(CoreAdminParams.NAME, coreName); + params.set(COLL_CONF, configName); + params.set(CoreAdminParams.COLLECTION, collection); + if (shard != null) { + params.set(CoreAdminParams.SHARD, shard); + } else if (routeKey != null) { + Collection slices = coll.getRouter().getSearchSlicesSingle(routeKey, null, coll); + if (slices.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No active shard serving _route_=" + routeKey + " found"); + } else { + params.set(CoreAdminParams.SHARD, slices.iterator().next().getName()); + } + } else { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Specify either 'shard' or _route_ param"); + } + if (dataDir != null) { + params.set(CoreAdminParams.DATA_DIR, dataDir); + } + if (instanceDir != null) { + params.set(CoreAdminParams.INSTANCE_DIR, instanceDir); + } + ocmh.addPropertyParams(message, params); + + // For tracking async calls. + Map requestMap = new HashMap<>(); + ocmh.sendShardRequest(node, params, shardHandler, asyncId, requestMap); + + final String fnode = node; + final String fcoreName = coreName; + + Runnable runnable = () -> { + ocmh.processResponses(results, shardHandler, true, "ADDREPLICA failed to create replica", asyncId, requestMap); + ocmh.waitForCoreNodeName(collection, fnode, fcoreName); + if (onComplete != null) onComplete.run(); + }; + + if (!parallel) { + runnable.run(); + } else { + ocmh.tpe.submit(runnable); + } + + + return new ZkNodeProps( + ZkStateReader.COLLECTION_PROP, collection, + ZkStateReader.SHARD_ID_PROP, shard, + ZkStateReader.CORE_NAME_PROP, coreName, + ZkStateReader.NODE_NAME_PROP, node + ); + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/BackupCmd.java new file mode 100644 index 00000000000..679cb07ac2e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/BackupCmd.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.backup.BackupManager; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.handler.component.ShardHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CommonParams.NAME; + +public class BackupCmd implements OverseerCollectionMessageHandler.Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final OverseerCollectionMessageHandler ocmh; + + public BackupCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + String collectionName = message.getStr(COLLECTION_PROP); + String backupName = message.getStr(NAME); + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + String asyncId = message.getStr(ASYNC); + String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); + String location = message.getStr(CoreAdminParams.BACKUP_LOCATION); + + Map requestMap = new HashMap<>(); + Instant startTime = Instant.now(); + + CoreContainer cc = ocmh.overseer.getZkController().getCoreContainer(); + BackupRepository repository = cc.newBackupRepository(Optional.ofNullable(repo)); + BackupManager backupMgr = new BackupManager(repository, ocmh.zkStateReader, collectionName); + + // Backup location + URI backupPath = repository.createURI(location, backupName); + + //Validating if the directory already exists. + if (repository.exists(backupPath)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupPath); + } + + // Create a directory to store backup details. + repository.createDirectory(backupPath); + + log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName, + backupPath); + + for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) { + Replica replica = slice.getLeader(); + + String coreName = replica.getStr(CORE_NAME_PROP); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString()); + params.set(NAME, slice.getName()); + params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); + params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.getPath()); // note: index dir will be here then the "snapshot." + slice name + params.set(CORE_NAME_PROP, coreName); + + ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap); + log.debug("Sent backup request to core={} for backupName={}", coreName, backupName); + } + log.debug("Sent backup requests to all shard leaders for backupName={}", backupName); + + ocmh.processResponses(results, shardHandler, true, "Could not backup all replicas", asyncId, requestMap); + + log.info("Starting to backup ZK data for backupName={}", backupName); + + //Download the configs + String configName = ocmh.zkStateReader.readConfigName(collectionName); + backupMgr.downloadConfigDir(location, backupName, configName); + + //Save the collection's state. Can be part of the monolithic clusterstate.json or a individual state.json + //Since we don't want to distinguish we extract the state and back it up as a separate json + DocCollection collectionState = ocmh.zkStateReader.getClusterState().getCollection(collectionName); + backupMgr.writeCollectionState(location, backupName, collectionName, collectionState); + + Properties properties = new Properties(); + + properties.put(BackupManager.BACKUP_NAME_PROP, backupName); + properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName); + properties.put(COLL_CONF, configName); + properties.put(BackupManager.START_TIME_PROP, startTime.toString()); + //TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same. + //if they are not the same then we can throw an error or have an 'overwriteConfig' flag + //TODO save numDocs for the shardLeader. We can use it to sanity check the restore. + + backupMgr.writeBackupProperties(location, backupName, properties); + + log.info("Completed backing up ZK data for backupName={}", backupName); + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java new file mode 100644 index 00000000000..b966ebdd769 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java @@ -0,0 +1,101 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.Aliases; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Utils; +import org.apache.solr.util.TimeOut; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.params.CommonParams.NAME; + + +public class CreateAliasCmd implements Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final OverseerCollectionMessageHandler ocmh; + + public CreateAliasCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState state, ZkNodeProps message, NamedList results) + throws Exception { + String aliasName = message.getStr(NAME); + String collections = message.getStr("collections"); + + Map> newAliasesMap = new HashMap<>(); + Map newCollectionAliasesMap = new HashMap<>(); + ZkStateReader zkStateReader = ocmh.zkStateReader; + Map prevColAliases = zkStateReader.getAliases().getCollectionAliasMap(); + if (prevColAliases != null) { + newCollectionAliasesMap.putAll(prevColAliases); + } + newCollectionAliasesMap.put(aliasName, collections); + newAliasesMap.put("collection", newCollectionAliasesMap); + Aliases newAliases = new Aliases(newAliasesMap); + byte[] jsonBytes = null; + if (newAliases.collectionAliasSize() > 0) { // only sub map right now + jsonBytes = Utils.toJSON(newAliases.getAliasMap()); + } + try { + zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, jsonBytes, true); + + checkForAlias(aliasName, collections); + // some fudge for other nodes + Thread.sleep(100); + } catch (KeeperException e) { + log.error("", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } catch (InterruptedException e) { + log.warn("", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + } + + private void checkForAlias(String name, String value) { + + TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); + boolean success = false; + Aliases aliases; + while (!timeout.hasTimedOut()) { + aliases = ocmh.zkStateReader.getAliases(); + String collections = aliases.getCollectionAlias(name); + if (collections != null && collections.equals(value)) { + success = true; + break; + } + } + if (!success) { + log.warn("Timeout waiting to be notified of Alias change..."); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java new file mode 100644 index 00000000000..7f28600498f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; +import org.apache.solr.cloud.overseer.ClusterStateMutator; +import org.apache.solr.cloud.rule.ReplicaAssigner; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocRouter; +import org.apache.solr.common.cloud.ImplicitDocRouter; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.ZkConfigManager; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.component.ShardHandler; +import org.apache.solr.handler.component.ShardRequest; +import org.apache.solr.util.TimeOut; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.RANDOM; +import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.common.util.StrUtils.formatString; + +public class CreateCollectionCmd implements Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final OverseerCollectionMessageHandler ocmh; + + public CreateCollectionCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception { + final String collectionName = message.getStr(NAME); + log.info("Create collection {}", collectionName); + if (clusterState.hasCollection(collectionName)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "collection already exists: " + collectionName); + } + + String configName = getConfigName(collectionName, message); + if (configName == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No config set found to associate with the collection."); + } + + ocmh.validateConfigOrThrowSolrException(configName); + + + try { + // look at the replication factor and see if it matches reality + // if it does not, find best nodes to create more cores + + int repFactor = message.getInt(REPLICATION_FACTOR, 1); + + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + final String async = message.getStr(ASYNC); + + Integer numSlices = message.getInt(NUM_SLICES, null); + String router = message.getStr("router.name", DocRouter.DEFAULT_NAME); + List shardNames = new ArrayList<>(); + if(ImplicitDocRouter.NAME.equals(router)){ + ClusterStateMutator.getShardNames(shardNames, message.getStr("shards", null)); + numSlices = shardNames.size(); + } else { + if (numSlices == null ) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NUM_SLICES + " is a required param (when using CompositeId router)."); + } + ClusterStateMutator.getShardNames(numSlices, shardNames); + } + + int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1); + + if (repFactor <= 0) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, REPLICATION_FACTOR + " must be greater than 0"); + } + + if (numSlices <= 0) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NUM_SLICES + " must be > 0"); + } + + // we need to look at every node and see how many cores it serves + // add our new cores to existing nodes serving the least number of cores + // but (for now) require that each core goes on a distinct node. + + final List nodeList = OverseerCollectionMessageHandler.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, RANDOM); + Map positionVsNodes; + if (nodeList.isEmpty()) { + log.warn("It is unusual to create a collection ("+collectionName+") without cores."); + + positionVsNodes = new HashMap<>(); + } else { + if (repFactor > nodeList.size()) { + log.warn("Specified " + + REPLICATION_FACTOR + + " of " + + repFactor + + " on collection " + + collectionName + + " is higher than or equal to the number of Solr instances currently live or live and part of your " + CREATE_NODE_SET + "(" + + nodeList.size() + + "). It's unusual to run two replica of the same slice on the same Solr-instance."); + } + + int maxShardsAllowedToCreate = maxShardsPerNode * nodeList.size(); + int requestedShardsToCreate = numSlices * repFactor; + if (maxShardsAllowedToCreate < requestedShardsToCreate) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot create collection " + collectionName + ". Value of " + + MAX_SHARDS_PER_NODE + " is " + maxShardsPerNode + + ", and the number of nodes currently live or live and part of your "+CREATE_NODE_SET+" is " + nodeList.size() + + ". This allows a maximum of " + maxShardsAllowedToCreate + + " to be created. Value of " + NUM_SLICES + " is " + numSlices + + " and value of " + REPLICATION_FACTOR + " is " + repFactor + + ". This requires " + requestedShardsToCreate + + " shards to be created (higher than the allowed number)"); + } + + positionVsNodes = ocmh.identifyNodes(clusterState, nodeList, message, shardNames, repFactor); + } + + ZkStateReader zkStateReader = ocmh.zkStateReader; + boolean isLegacyCloud = Overseer.isLegacy(zkStateReader); + + ocmh.createConfNode(configName, collectionName, isLegacyCloud); + + Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message)); + + // wait for a while until we don't see the collection + TimeOut waitUntil = new TimeOut(30, TimeUnit.SECONDS); + boolean created = false; + while (! waitUntil.hasTimedOut()) { + Thread.sleep(100); + created = zkStateReader.getClusterState().hasCollection(collectionName); + if(created) break; + } + if (!created) + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Could not fully create collection: " + collectionName); + + if (nodeList.isEmpty()) { + log.info("Finished create command for collection: {}", collectionName); + return; + } + + // For tracking async calls. + Map requestMap = new HashMap<>(); + + + log.info(formatString("Creating SolrCores for new collection {0}, shardNames {1} , replicationFactor : {2}", + collectionName, shardNames, repFactor)); + Map coresToCreate = new LinkedHashMap<>(); + for (Map.Entry e : positionVsNodes.entrySet()) { + ReplicaAssigner.Position position = e.getKey(); + String nodeName = e.getValue(); + String coreName = collectionName + "_" + position.shard + "_replica" + (position.index + 1); + log.info(formatString("Creating core {0} as part of shard {1} of collection {2} on {3}" + , coreName, position.shard, collectionName, nodeName)); + + + String baseUrl = zkStateReader.getBaseUrlForNodeName(nodeName); + //in the new mode, create the replica in clusterstate prior to creating the core. + // Otherwise the core creation fails + if (!isLegacyCloud) { + ZkNodeProps props = new ZkNodeProps( + Overseer.QUEUE_OPERATION, ADDREPLICA.toString(), + ZkStateReader.COLLECTION_PROP, collectionName, + ZkStateReader.SHARD_ID_PROP, position.shard, + ZkStateReader.CORE_NAME_PROP, coreName, + ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), + ZkStateReader.BASE_URL_PROP, baseUrl); + Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(props)); + } + + // Need to create new params for each request + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATE.toString()); + + params.set(CoreAdminParams.NAME, coreName); + params.set(COLL_CONF, configName); + params.set(CoreAdminParams.COLLECTION, collectionName); + params.set(CoreAdminParams.SHARD, position.shard); + params.set(ZkStateReader.NUM_SHARDS_PROP, numSlices); + + if (async != null) { + String coreAdminAsyncId = async + Math.abs(System.nanoTime()); + params.add(ASYNC, coreAdminAsyncId); + requestMap.put(nodeName, coreAdminAsyncId); + } + ocmh.addPropertyParams(message, params); + + ShardRequest sreq = new ShardRequest(); + sreq.nodeName = nodeName; + params.set("qt", ocmh.adminPath); + sreq.purpose = 1; + sreq.shards = new String[]{baseUrl}; + sreq.actualShards = sreq.shards; + sreq.params = params; + + if (isLegacyCloud) { + shardHandler.submit(sreq, sreq.shards[0], sreq.params); + } else { + coresToCreate.put(coreName, sreq); + } + } + + if(!isLegacyCloud) { + // wait for all replica entries to be created + Map replicas = ocmh.waitToSeeReplicasInState(collectionName, coresToCreate.keySet()); + for (Map.Entry e : coresToCreate.entrySet()) { + ShardRequest sreq = e.getValue(); + sreq.params.set(CoreAdminParams.CORE_NODE_NAME, replicas.get(e.getKey()).getName()); + shardHandler.submit(sreq, sreq.shards[0], sreq.params); + } + } + + ocmh.processResponses(results, shardHandler, false, null, async, requestMap, Collections.emptySet()); + if(results.get("failure") != null && ((SimpleOrderedMap)results.get("failure")).size() > 0) { + // Let's cleanup as we hit an exception + // We shouldn't be passing 'results' here for the cleanup as the response would then contain 'success' + // element, which may be interpreted by the user as a positive ack + ocmh.cleanupCollection(collectionName, new NamedList()); + log.info("Cleaned up artifacts for failed create collection for [" + collectionName + "]"); + } else { + log.debug("Finished create command on all shards for collection: " + + collectionName); + } + } catch (SolrException ex) { + throw ex; + } catch (Exception ex) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, ex); + } + } + String getConfigName(String coll, ZkNodeProps message) throws KeeperException, InterruptedException { + String configName = message.getStr(COLL_CONF); + + if (configName == null) { + // if there is only one conf, use that + List configNames = null; + try { + configNames = ocmh.zkStateReader.getZkClient().getChildren(ZkConfigManager.CONFIGS_ZKNODE, null, true); + if (configNames != null && configNames.size() == 1) { + configName = configNames.get(0); + // no config set named, but there is only 1 - use it + log.info("Only one config set found in zk - using it:" + configName); + } else if (configNames.contains(coll)) { + configName = coll; + } + } catch (KeeperException.NoNodeException e) { + + } + } + return configName; + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java new file mode 100644 index 00000000000..3d5aa4151ba --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + + +import java.lang.invoke.MethodHandles; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.component.ShardHandler; +import org.apache.solr.util.TimeOut; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.cloud.Assign.getNodesForNewReplicas; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; +import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CommonParams.NAME; + +public class CreateShardCmd implements Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final OverseerCollectionMessageHandler ocmh; + + public CreateShardCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception { + String collectionName = message.getStr(COLLECTION_PROP); + String sliceName = message.getStr(SHARD_ID_PROP); + + log.info("Create shard invoked: {}", message); + if (collectionName == null || sliceName == null) + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'collection' and 'shard' are required parameters"); + int numSlices = 1; + + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + DocCollection collection = clusterState.getCollection(collectionName); + int repFactor = message.getInt(REPLICATION_FACTOR, collection.getInt(REPLICATION_FACTOR, 1)); + String createNodeSetStr = message.getStr(OverseerCollectionMessageHandler.CREATE_NODE_SET); + List sortedNodeList = getNodesForNewReplicas(clusterState, collectionName, sliceName, repFactor, + createNodeSetStr, ocmh.overseer.getZkController().getCoreContainer()); + + ZkStateReader zkStateReader = ocmh.zkStateReader; + Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message)); + // wait for a while until we see the shard + TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); + boolean created = false; + while (!timeout.hasTimedOut()) { + Thread.sleep(100); + created = zkStateReader.getClusterState().getCollection(collectionName).getSlice(sliceName) != null; + if (created) break; + } + if (!created) + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Could not fully create shard: " + message.getStr(NAME)); + + String configName = message.getStr(COLL_CONF); + + String async = message.getStr(ASYNC); + Map requestMap = null; + if (async != null) { + requestMap = new HashMap<>(repFactor, 1.0f); + } + + for (int j = 1; j <= repFactor; j++) { + String nodeName = sortedNodeList.get(((j - 1)) % sortedNodeList.size()).nodeName; + String shardName = collectionName + "_" + sliceName + "_replica" + j; + log.info("Creating shard " + shardName + " as part of slice " + sliceName + " of collection " + collectionName + + " on " + nodeName); + + // Need to create new params for each request + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATE.toString()); + params.set(CoreAdminParams.NAME, shardName); + params.set(COLL_CONF, configName); + params.set(CoreAdminParams.COLLECTION, collectionName); + params.set(CoreAdminParams.SHARD, sliceName); + params.set(ZkStateReader.NUM_SHARDS_PROP, numSlices); + ocmh.addPropertyParams(message, params); + + ocmh.sendShardRequest(nodeName, params, shardHandler, async, requestMap); + } + + ocmh.processResponses(results, shardHandler, true, "Failed to create shard", async, requestMap, Collections.emptySet()); + + log.info("Finished create command on all shards for collection: " + collectionName); + + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/DeleteAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/DeleteAliasCmd.java new file mode 100644 index 00000000000..7b1993ce4a0 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/DeleteAliasCmd.java @@ -0,0 +1,95 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.Aliases; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Utils; +import org.apache.solr.util.TimeOut; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.params.CommonParams.NAME; + +public class DeleteAliasCmd implements OverseerCollectionMessageHandler.Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final OverseerCollectionMessageHandler ocmh; + + public DeleteAliasCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + String aliasName = message.getStr(NAME); + + Map> newAliasesMap = new HashMap<>(); + Map newCollectionAliasesMap = new HashMap<>(); + ZkStateReader zkStateReader = ocmh.zkStateReader; + newCollectionAliasesMap.putAll(zkStateReader.getAliases().getCollectionAliasMap()); + newCollectionAliasesMap.remove(aliasName); + newAliasesMap.put("collection", newCollectionAliasesMap); + Aliases newAliases = new Aliases(newAliasesMap); + byte[] jsonBytes = null; + if (newAliases.collectionAliasSize() > 0) { // only sub map right now + jsonBytes = Utils.toJSON(newAliases.getAliasMap()); + } + try { + zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, + jsonBytes, true); + checkForAliasAbsence(aliasName); + // some fudge for other nodes + Thread.sleep(100); + } catch (KeeperException e) { + log.error("", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } catch (InterruptedException e) { + log.warn("", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + + } + private void checkForAliasAbsence(String name) { + + TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); + boolean success = false; + Aliases aliases = null; + while (! timeout.hasTimedOut()) { + aliases = ocmh.zkStateReader.getAliases(); + String collections = aliases.getCollectionAlias(name); + if (collections == null) { + success = true; + break; + } + } + if (!success) { + log.warn("Timeout waiting to be notified of Alias change..."); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/DeleteCollectionCmd.java new file mode 100644 index 00000000000..4c5ae007794 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/DeleteCollectionCmd.java @@ -0,0 +1,121 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.common.NonExistentCoreException; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Utils; +import org.apache.solr.util.TimeOut; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CommonParams.NAME; + +public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final OverseerCollectionMessageHandler ocmh; + + public DeleteCollectionCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + ZkStateReader zkStateReader = ocmh.zkStateReader; + final String collection = message.getStr(NAME); + try { + if (zkStateReader.getClusterState().getCollectionOrNull(collection) == null) { + if (zkStateReader.getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection, true)) { + // if the collection is not in the clusterstate, but is listed in zk, do nothing, it will just + // be removed in the finally - we cannot continue, because the below code will error if the collection + // is not in the clusterstate + return; + } + } + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.UNLOAD.toString()); + params.set(CoreAdminParams.DELETE_INSTANCE_DIR, true); + params.set(CoreAdminParams.DELETE_DATA_DIR, true); + + String asyncId = message.getStr(ASYNC); + Map requestMap = null; + if (asyncId != null) { + requestMap = new HashMap<>(); + } + + Set okayExceptions = new HashSet<>(1); + okayExceptions.add(NonExistentCoreException.class.getName()); + + ocmh.collectionCmd(message, params, results, null, asyncId, requestMap, okayExceptions); + + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, collection); + Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(m)); + + // wait for a while until we don't see the collection + TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); + boolean removed = false; + while (! timeout.hasTimedOut()) { + Thread.sleep(100); + removed = !zkStateReader.getClusterState().hasCollection(collection); + if (removed) { + Thread.sleep(500); // just a bit of time so it's more likely other + // readers see on return + break; + } + } + if (!removed) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Could not fully remove collection: " + collection); + } + + } finally { + + try { + if (zkStateReader.getZkClient().exists( + ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection, true)) { + zkStateReader.getZkClient().clean( + ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection); + } + } catch (InterruptedException e) { + SolrException.log(log, "Cleaning up collection in zk was interrupted:" + + collection, e); + Thread.currentThread().interrupt(); + } catch (KeeperException e) { + SolrException.log(log, "Problem cleaning up collection in zk:" + + collection, e); + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java new file mode 100644 index 00000000000..b3c505557ca --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + + +import java.lang.invoke.MethodHandles; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.util.NamedList; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA; + +public class DeleteNodeCmd implements OverseerCollectionMessageHandler.Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final OverseerCollectionMessageHandler ocmh; + + public DeleteNodeCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + ocmh.checkRequired(message, "node"); + String node = message.getStr("node"); + if (!state.liveNodesContain(node)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source Node: " + node + " is not live"); + } + List sourceReplicas = ReplaceNodeCmd.getReplicasOfNode(node, state); + cleanupReplicas(results, state, sourceReplicas, ocmh, node); + } + + static void cleanupReplicas(NamedList results, + ClusterState clusterState, + List sourceReplicas, + OverseerCollectionMessageHandler ocmh, String node) throws InterruptedException { + CountDownLatch cleanupLatch = new CountDownLatch(sourceReplicas.size()); + for (ZkNodeProps sourceReplica : sourceReplicas) { + log.info("Deleting replica for collection={} shard={} on node={}", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), node); + NamedList deleteResult = new NamedList(); + try { + ((DeleteReplicaCmd)ocmh.commandMap.get(DELETEREPLICA)).deleteReplica(clusterState, sourceReplica.plus("parallel", "true"), deleteResult, () -> { + cleanupLatch.countDown(); + if (deleteResult.get("failure") != null) { + synchronized (results) { + results.add("failure", String.format(Locale.ROOT, "Failed to delete replica for collection=%s shard=%s" + + " on node=%s", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), node)); + } + } + }); + } catch (KeeperException e) { + log.warn("Error deleting ", e); + cleanupLatch.countDown(); + } catch (Exception e) { + log.warn("Error deleting ", e); + cleanupLatch.countDown(); + throw e; + } + } + log.debug("Waiting for delete node action to complete"); + cleanupLatch.await(5, TimeUnit.MINUTES); + } + + +} diff --git a/solr/core/src/java/org/apache/solr/cloud/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/DeleteReplicaCmd.java new file mode 100644 index 00000000000..6f5fc6223db --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/DeleteReplicaCmd.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.handler.component.ShardHandler; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.ONLY_IF_DOWN; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; + + +public class DeleteReplicaCmd implements Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final OverseerCollectionMessageHandler ocmh; + + public DeleteReplicaCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + @SuppressWarnings("unchecked") + + public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception { + deleteReplica(clusterState, message, results,null); + } + + @SuppressWarnings("unchecked") + void deleteReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete) + throws KeeperException, InterruptedException { + ocmh.checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP); + String collectionName = message.getStr(COLLECTION_PROP); + String shard = message.getStr(SHARD_ID_PROP); + String replicaName = message.getStr(REPLICA_PROP); + boolean parallel = message.getBool("parallel", false); + + DocCollection coll = clusterState.getCollection(collectionName); + Slice slice = coll.getSlice(shard); + if (slice == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Invalid shard name : " + shard + " in collection : " + collectionName); + } + Replica replica = slice.getReplica(replicaName); + if (replica == null) { + ArrayList l = new ArrayList<>(); + for (Replica r : slice.getReplicas()) + l.add(r.getName()); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid replica : " + replicaName + " in shard/collection : " + + shard + "/" + collectionName + " available replicas are " + StrUtils.join(l, ',')); + } + + // If users are being safe and only want to remove a shard if it is down, they can specify onlyIfDown=true + // on the command. + if (Boolean.parseBoolean(message.getStr(ONLY_IF_DOWN)) && replica.getState() != Replica.State.DOWN) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Attempted to remove replica : " + collectionName + "/" + shard + "/" + replicaName + + " with onlyIfDown='true', but state is '" + replica.getStr(ZkStateReader.STATE_PROP) + "'"); + } + + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + String core = replica.getStr(ZkStateReader.CORE_NAME_PROP); + String asyncId = message.getStr(ASYNC); + AtomicReference> requestMap = new AtomicReference<>(null); + if (asyncId != null) { + requestMap.set(new HashMap<>(1, 1.0f)); + } + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.UNLOAD.toString()); + params.add(CoreAdminParams.CORE, core); + + params.set(CoreAdminParams.DELETE_INDEX, message.getBool(CoreAdminParams.DELETE_INDEX, true)); + params.set(CoreAdminParams.DELETE_INSTANCE_DIR, message.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, true)); + params.set(CoreAdminParams.DELETE_DATA_DIR, message.getBool(CoreAdminParams.DELETE_DATA_DIR, true)); + + boolean isLive = ocmh.zkStateReader.getClusterState().getLiveNodes().contains(replica.getNodeName()); + if (isLive) { + ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap.get()); + } + + Callable callable = () -> { + try { + if (isLive) { + ocmh.processResponses(results, shardHandler, false, null, asyncId, requestMap.get()); + + //check if the core unload removed the corenode zk entry + if (ocmh.waitForCoreNodeGone(collectionName, shard, replicaName, 5000)) return Boolean.TRUE; + } + + // try and ensure core info is removed from cluster state + ocmh.deleteCoreNode(collectionName, replicaName, replica, core); + if (ocmh.waitForCoreNodeGone(collectionName, shard, replicaName, 30000)) return Boolean.TRUE; + return Boolean.FALSE; + } catch (Exception e) { + results.add("failure", "Could not complete delete " + e.getMessage()); + throw e; + } finally { + if (onComplete != null) onComplete.run(); + } + }; + + if (!parallel) { + try { + if (!callable.call()) + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Could not remove replica : " + collectionName + "/" + shard + "/" + replicaName); + } catch (InterruptedException | KeeperException e) { + throw e; + } catch (Exception ex) { + throw new SolrException(SolrException.ErrorCode.UNKNOWN, "Error waiting for corenode gone", ex); + } + + } else { + ocmh.tpe.submit(callable); + } + } + + } diff --git a/solr/core/src/java/org/apache/solr/cloud/DeleteShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/DeleteShardCmd.java new file mode 100644 index 00000000000..f2ae5ca6a48 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/DeleteShardCmd.java @@ -0,0 +1,126 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; +import java.lang.invoke.MethodHandles; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.component.ShardHandler; +import org.apache.solr.util.TimeOut; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; + +public class DeleteShardCmd implements Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final OverseerCollectionMessageHandler ocmh; + + public DeleteShardCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception { + String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP); + String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP); + + log.info("Delete shard invoked"); + Slice slice = clusterState.getSlice(collectionName, sliceId); + + if (slice == null) { + if (clusterState.hasCollection(collectionName)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "No shard with name " + sliceId + " exists for collection " + collectionName); + } else { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No collection with the specified name exists: " + collectionName); + } + } + // For now, only allow for deletions of Inactive slices or custom hashes (range==null). + // TODO: Add check for range gaps on Slice deletion + final Slice.State state = slice.getState(); + if (!(slice.getRange() == null || state == Slice.State.INACTIVE || state == Slice.State.RECOVERY + || state == Slice.State.CONSTRUCTION)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The slice: " + slice.getName() + " is currently " + state + + ". Only non-active (or custom-hashed) slices can be deleted."); + } + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + + String asyncId = message.getStr(ASYNC); + Map requestMap = null; + if (asyncId != null) { + requestMap = new HashMap<>(slice.getReplicas().size(), 1.0f); + } + + try { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.UNLOAD.toString()); + params.set(CoreAdminParams.DELETE_INDEX, message.getBool(CoreAdminParams.DELETE_INDEX, true)); + params.set(CoreAdminParams.DELETE_INSTANCE_DIR, message.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, true)); + params.set(CoreAdminParams.DELETE_DATA_DIR, message.getBool(CoreAdminParams.DELETE_DATA_DIR, true)); + + ocmh.sliceCmd(clusterState, params, null, slice, shardHandler, asyncId, requestMap); + + ocmh.processResponses(results, shardHandler, true, "Failed to delete shard", asyncId, requestMap, Collections.emptySet()); + + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, DELETESHARD.toLower(), ZkStateReader.COLLECTION_PROP, + collectionName, ZkStateReader.SHARD_ID_PROP, sliceId); + ZkStateReader zkStateReader = ocmh.zkStateReader; + Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(m)); + + // wait for a while until we don't see the shard + TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); + boolean removed = false; + while (! timeout.hasTimedOut()) { + Thread.sleep(100); + DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName); + removed = collection.getSlice(sliceId) == null; + if (removed) { + Thread.sleep(100); // just a bit of time so it's more likely other readers see on return + break; + } + } + if (!removed) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Could not fully remove collection: " + collectionName + " shard: " + sliceId); + } + + log.info("Successfully deleted collection: " + collectionName + ", shard: " + sliceId); + + } catch (SolrException e) { + throw e; + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Error executing delete operation for collection: " + collectionName + " shard: " + sliceId, e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/MigrateCmd.java new file mode 100644 index 00000000000..7b1ad2c2b86 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/MigrateCmd.java @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.lang.invoke.MethodHandles; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.cloud.overseer.OverseerAction; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.CompositeIdRouter; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.DocRouter; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.RoutingRule; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.component.ShardHandler; +import org.apache.solr.handler.component.ShardHandlerFactory; +import org.apache.solr.update.SolrIndexSplitter; +import org.apache.solr.util.TimeOut; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_PROP_PREFIX; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; +import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.common.util.Utils.makeMap; + +public class MigrateCmd implements OverseerCollectionMessageHandler.Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final OverseerCollectionMessageHandler ocmh; + + public MigrateCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + + @Override + public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception { + String sourceCollectionName = message.getStr("collection"); + String splitKey = message.getStr("split.key"); + String targetCollectionName = message.getStr("target.collection"); + int timeout = message.getInt("forward.timeout", 10 * 60) * 1000; + + DocCollection sourceCollection = clusterState.getCollection(sourceCollectionName); + if (sourceCollection == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown source collection: " + sourceCollectionName); + } + DocCollection targetCollection = clusterState.getCollection(targetCollectionName); + if (targetCollection == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown target collection: " + sourceCollectionName); + } + if (!(sourceCollection.getRouter() instanceof CompositeIdRouter)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source collection must use a compositeId router"); + } + if (!(targetCollection.getRouter() instanceof CompositeIdRouter)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Target collection must use a compositeId router"); + } + CompositeIdRouter sourceRouter = (CompositeIdRouter) sourceCollection.getRouter(); + CompositeIdRouter targetRouter = (CompositeIdRouter) targetCollection.getRouter(); + Collection sourceSlices = sourceRouter.getSearchSlicesSingle(splitKey, null, sourceCollection); + if (sourceSlices.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "No active slices available in source collection: " + sourceCollection + "for given split.key: " + splitKey); + } + Collection targetSlices = targetRouter.getSearchSlicesSingle(splitKey, null, targetCollection); + if (targetSlices.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "No active slices available in target collection: " + targetCollection + "for given split.key: " + splitKey); + } + + String asyncId = null; + if (message.containsKey(ASYNC) && message.get(ASYNC) != null) + asyncId = message.getStr(ASYNC); + + for (Slice sourceSlice : sourceSlices) { + for (Slice targetSlice : targetSlices) { + log.info("Migrating source shard: {} to target shard: {} for split.key = " + splitKey, sourceSlice, targetSlice); + migrateKey(clusterState, sourceCollection, sourceSlice, targetCollection, targetSlice, splitKey, + timeout, results, asyncId, message); + } + } + } + + private void migrateKey(ClusterState clusterState, DocCollection sourceCollection, Slice sourceSlice, + DocCollection targetCollection, Slice targetSlice, + String splitKey, int timeout, + NamedList results, String asyncId, ZkNodeProps message) throws Exception { + String tempSourceCollectionName = "split_" + sourceSlice.getName() + "_temp_" + targetSlice.getName(); + ZkStateReader zkStateReader = ocmh.zkStateReader; + if (clusterState.hasCollection(tempSourceCollectionName)) { + log.info("Deleting temporary collection: " + tempSourceCollectionName); + Map props = makeMap( + Overseer.QUEUE_OPERATION, DELETE.toLower(), + NAME, tempSourceCollectionName); + + try { + ocmh.commandMap.get(DELETE).call(zkStateReader.getClusterState(), new ZkNodeProps(props), results); + clusterState = zkStateReader.getClusterState(); + } catch (Exception e) { + log.warn("Unable to clean up existing temporary collection: " + tempSourceCollectionName, e); + } + } + + CompositeIdRouter sourceRouter = (CompositeIdRouter) sourceCollection.getRouter(); + DocRouter.Range keyHashRange = sourceRouter.keyHashRange(splitKey); + + ShardHandlerFactory shardHandlerFactory = ocmh.shardHandlerFactory; + ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); + + log.info("Hash range for split.key: {} is: {}", splitKey, keyHashRange); + // intersect source range, keyHashRange and target range + // this is the range that has to be split from source and transferred to target + DocRouter.Range splitRange = ocmh.intersect(targetSlice.getRange(), ocmh.intersect(sourceSlice.getRange(), keyHashRange)); + if (splitRange == null) { + log.info("No common hashes between source shard: {} and target shard: {}", sourceSlice.getName(), targetSlice.getName()); + return; + } + log.info("Common hash range between source shard: {} and target shard: {} = " + splitRange, sourceSlice.getName(), targetSlice.getName()); + + Replica targetLeader = zkStateReader.getLeaderRetry(targetCollection.getName(), targetSlice.getName(), 10000); + // For tracking async calls. + Map requestMap = new HashMap<>(); + + log.info("Asking target leader node: " + targetLeader.getNodeName() + " core: " + + targetLeader.getStr("core") + " to buffer updates"); + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.REQUESTBUFFERUPDATES.toString()); + params.set(CoreAdminParams.NAME, targetLeader.getStr("core")); + + ocmh.sendShardRequest(targetLeader.getNodeName(), params, shardHandler, asyncId, requestMap); + + ocmh.processResponses(results, shardHandler, true, "MIGRATE failed to request node to buffer updates", asyncId, requestMap); + + ZkNodeProps m = new ZkNodeProps( + Overseer.QUEUE_OPERATION, OverseerAction.ADDROUTINGRULE.toLower(), + COLLECTION_PROP, sourceCollection.getName(), + SHARD_ID_PROP, sourceSlice.getName(), + "routeKey", SolrIndexSplitter.getRouteKey(splitKey) + "!", + "range", splitRange.toString(), + "targetCollection", targetCollection.getName(), + "expireAt", RoutingRule.makeExpiryAt(timeout)); + log.info("Adding routing rule: " + m); + Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(m)); + + // wait for a while until we see the new rule + log.info("Waiting to see routing rule updated in clusterstate"); + TimeOut waitUntil = new TimeOut(60, TimeUnit.SECONDS); + boolean added = false; + while (!waitUntil.hasTimedOut()) { + Thread.sleep(100); + sourceCollection = zkStateReader.getClusterState().getCollection(sourceCollection.getName()); + sourceSlice = sourceCollection.getSlice(sourceSlice.getName()); + Map rules = sourceSlice.getRoutingRules(); + if (rules != null) { + RoutingRule rule = rules.get(SolrIndexSplitter.getRouteKey(splitKey) + "!"); + if (rule != null && rule.getRouteRanges().contains(splitRange)) { + added = true; + break; + } + } + } + if (!added) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Could not add routing rule: " + m); + } + + log.info("Routing rule added successfully"); + + // Create temp core on source shard + Replica sourceLeader = zkStateReader.getLeaderRetry(sourceCollection.getName(), sourceSlice.getName(), 10000); + + // create a temporary collection with just one node on the shard leader + String configName = zkStateReader.readConfigName(sourceCollection.getName()); + Map props = makeMap( + Overseer.QUEUE_OPERATION, CREATE.toLower(), + NAME, tempSourceCollectionName, + REPLICATION_FACTOR, 1, + NUM_SLICES, 1, + COLL_CONF, configName, + CREATE_NODE_SET, sourceLeader.getNodeName()); + if (asyncId != null) { + String internalAsyncId = asyncId + Math.abs(System.nanoTime()); + props.put(ASYNC, internalAsyncId); + } + + log.info("Creating temporary collection: " + props); + ocmh.commandMap.get(CREATE).call(clusterState, new ZkNodeProps(props), results); + // refresh cluster state + clusterState = zkStateReader.getClusterState(); + Slice tempSourceSlice = clusterState.getCollection(tempSourceCollectionName).getSlices().iterator().next(); + Replica tempSourceLeader = zkStateReader.getLeaderRetry(tempSourceCollectionName, tempSourceSlice.getName(), 120000); + + String tempCollectionReplica1 = tempSourceCollectionName + "_" + tempSourceSlice.getName() + "_replica1"; + String coreNodeName = ocmh.waitForCoreNodeName(tempSourceCollectionName, + sourceLeader.getNodeName(), tempCollectionReplica1); + // wait for the replicas to be seen as active on temp source leader + log.info("Asking source leader to wait for: " + tempCollectionReplica1 + " to be alive on: " + sourceLeader.getNodeName()); + CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState(); + cmd.setCoreName(tempCollectionReplica1); + cmd.setNodeName(sourceLeader.getNodeName()); + cmd.setCoreNodeName(coreNodeName); + cmd.setState(Replica.State.ACTIVE); + cmd.setCheckLive(true); + cmd.setOnlyIfLeader(true); + // we don't want this to happen asynchronously + ocmh.sendShardRequest(tempSourceLeader.getNodeName(), new ModifiableSolrParams(cmd.getParams()), shardHandler, null, null); + + ocmh.processResponses(results, shardHandler, true, "MIGRATE failed to create temp collection leader" + + " or timed out waiting for it to come up", asyncId, requestMap); + + log.info("Asking source leader to split index"); + params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.SPLIT.toString()); + params.set(CoreAdminParams.CORE, sourceLeader.getStr("core")); + params.add(CoreAdminParams.TARGET_CORE, tempSourceLeader.getStr("core")); + params.set(CoreAdminParams.RANGES, splitRange.toString()); + params.set("split.key", splitKey); + + String tempNodeName = sourceLeader.getNodeName(); + + ocmh.sendShardRequest(tempNodeName, params, shardHandler, asyncId, requestMap); + ocmh.processResponses(results, shardHandler, true, "MIGRATE failed to invoke SPLIT core admin command", asyncId, requestMap); + + log.info("Creating a replica of temporary collection: {} on the target leader node: {}", + tempSourceCollectionName, targetLeader.getNodeName()); + String tempCollectionReplica2 = tempSourceCollectionName + "_" + tempSourceSlice.getName() + "_replica2"; + props = new HashMap<>(); + props.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower()); + props.put(COLLECTION_PROP, tempSourceCollectionName); + props.put(SHARD_ID_PROP, tempSourceSlice.getName()); + props.put("node", targetLeader.getNodeName()); + props.put(CoreAdminParams.NAME, tempCollectionReplica2); + // copy over property params: + for (String key : message.keySet()) { + if (key.startsWith(COLL_PROP_PREFIX)) { + props.put(key, message.getStr(key)); + } + } + // add async param + if (asyncId != null) { + props.put(ASYNC, asyncId); + } + ((AddReplicaCmd)ocmh.commandMap.get(ADDREPLICA)).addReplica(clusterState, new ZkNodeProps(props), results, null); + + ocmh.processResponses(results, shardHandler, true, "MIGRATE failed to create replica of " + + "temporary collection in target leader node.", asyncId, requestMap); + + coreNodeName = ocmh.waitForCoreNodeName(tempSourceCollectionName, + targetLeader.getNodeName(), tempCollectionReplica2); + // wait for the replicas to be seen as active on temp source leader + log.info("Asking temp source leader to wait for: " + tempCollectionReplica2 + " to be alive on: " + targetLeader.getNodeName()); + cmd = new CoreAdminRequest.WaitForState(); + cmd.setCoreName(tempSourceLeader.getStr("core")); + cmd.setNodeName(targetLeader.getNodeName()); + cmd.setCoreNodeName(coreNodeName); + cmd.setState(Replica.State.ACTIVE); + cmd.setCheckLive(true); + cmd.setOnlyIfLeader(true); + params = new ModifiableSolrParams(cmd.getParams()); + + ocmh.sendShardRequest(tempSourceLeader.getNodeName(), params, shardHandler, asyncId, requestMap); + + ocmh.processResponses(results, shardHandler, true, "MIGRATE failed to create temp collection" + + " replica or timed out waiting for them to come up", asyncId, requestMap); + + log.info("Successfully created replica of temp source collection on target leader node"); + + log.info("Requesting merge of temp source collection replica to target leader"); + params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.MERGEINDEXES.toString()); + params.set(CoreAdminParams.CORE, targetLeader.getStr("core")); + params.set(CoreAdminParams.SRC_CORE, tempCollectionReplica2); + + ocmh.sendShardRequest(targetLeader.getNodeName(), params, shardHandler, asyncId, requestMap); + String msg = "MIGRATE failed to merge " + tempCollectionReplica2 + " to " + + targetLeader.getStr("core") + " on node: " + targetLeader.getNodeName(); + ocmh.processResponses(results, shardHandler, true, msg, asyncId, requestMap); + + log.info("Asking target leader to apply buffered updates"); + params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.REQUESTAPPLYUPDATES.toString()); + params.set(CoreAdminParams.NAME, targetLeader.getStr("core")); + + ocmh.sendShardRequest(targetLeader.getNodeName(), params, shardHandler, asyncId, requestMap); + ocmh.processResponses(results, shardHandler, true, "MIGRATE failed to request node to apply buffered updates", + asyncId, requestMap); + + try { + log.info("Deleting temporary collection: " + tempSourceCollectionName); + props = makeMap( + Overseer.QUEUE_OPERATION, DELETE.toLower(), + NAME, tempSourceCollectionName); + ocmh.commandMap.get(DELETE). call(zkStateReader.getClusterState(), new ZkNodeProps(props), results); + } catch (Exception e) { + log.error("Unable to delete temporary collection: " + tempSourceCollectionName + + ". Please remove it manually", e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java index f8f84462df0..8c7a056299e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java @@ -16,6 +16,10 @@ */ package org.apache.solr.cloud; +import java.io.IOException; + +import org.apache.commons.io.IOUtils; +import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.handler.component.ShardHandler; import org.apache.solr.handler.component.ShardHandlerFactory; @@ -83,12 +87,20 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessor zkStateReader, myId, shardHandlerFactory, adminPath, stats, overseer, overseerNodePrioritizer); final OverseerConfigSetMessageHandler configMessageHandler = new OverseerConfigSetMessageHandler( zkStateReader); - return message -> { - String operation = message.getStr(Overseer.QUEUE_OPERATION); - if (operation != null && operation.startsWith(CONFIGSETS_ACTION_PREFIX)) { - return configMessageHandler; + return new OverseerMessageHandlerSelector() { + @Override + public void close() throws IOException { + IOUtils.closeQuietly(collMessageHandler); + } + + @Override + public OverseerMessageHandler selectOverseerMessageHandler(ZkNodeProps message) { + String operation = message.getStr(Overseer.QUEUE_OPERATION); + if (operation != null && operation.startsWith(CONFIGSETS_ACTION_PREFIX)) { + return configMessageHandler; + } + return collMessageHandler; } - return collMessageHandler; }; } } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java index 4e7e429bb75..36b51053a4b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java @@ -16,123 +16,90 @@ */ package org.apache.solr.cloud; +import java.io.Closeable; import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.net.URI; -import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.Optional; -import java.util.Properties; import java.util.Random; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; +import com.google.common.collect.ImmutableMap; import org.apache.commons.lang.StringUtils; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.cloud.Assign.ReplicaCount; -import org.apache.solr.cloud.overseer.ClusterStateMutator; import org.apache.solr.cloud.overseer.OverseerAction; import org.apache.solr.cloud.rule.ReplicaAssigner; import org.apache.solr.cloud.rule.ReplicaAssigner.Position; import org.apache.solr.cloud.rule.Rule; -import org.apache.solr.common.NonExistentCoreException; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.Aliases; import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.CompositeIdRouter; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.DocRouter; -import org.apache.solr.common.cloud.ImplicitDocRouter; -import org.apache.solr.common.cloud.PlainIdRouter; import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.RoutingRule; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkConfigManager; import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; +import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.common.util.Utils; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.backup.BackupManager; -import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.handler.component.ShardHandler; import org.apache.solr.handler.component.ShardHandlerFactory; import org.apache.solr.handler.component.ShardRequest; import org.apache.solr.handler.component.ShardResponse; -import org.apache.solr.update.SolrIndexSplitter; +import org.apache.solr.util.DefaultSolrThreadFactory; import org.apache.solr.util.RTimer; import org.apache.solr.util.TimeOut; -import org.apache.solr.util.stats.Snapshot; -import org.apache.solr.util.stats.Timer; -import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.cloud.Assign.getNodesForNewReplicas; import static org.apache.solr.common.cloud.DocCollection.SNITCH; import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.BALANCESHARDUNIQUE; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICAPROP; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MIGRATESTATEFORMAT; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.*; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.common.util.StrUtils.formatString; import static org.apache.solr.common.util.Utils.makeMap; /** * A {@link OverseerMessageHandler} that handles Collections API related * overseer messages. */ -public class OverseerCollectionMessageHandler implements OverseerMessageHandler { +public class OverseerCollectionMessageHandler implements OverseerMessageHandler , Closeable { public static final String NUM_SLICES = "numShards"; @@ -157,7 +124,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler public static final String ONLY_ACTIVE_NODES = "onlyactivenodes"; - private static final String SKIP_CREATE_REPLICA_IN_CLUSTER_STATE = "skipCreateReplicaInClusterState"; + static final String SKIP_CREATE_REPLICA_IN_CLUSTER_STATE = "skipCreateReplicaInClusterState"; public static final Map COLL_PROPS = Collections.unmodifiableMap(makeMap( ROUTER, DocRouter.DEFAULT_NAME, @@ -169,18 +136,20 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private Overseer overseer; - private ShardHandlerFactory shardHandlerFactory; - private String adminPath; - private ZkStateReader zkStateReader; - private String myId; - private Overseer.Stats stats; - private OverseerNodePrioritizer overseerPrioritizer; + Overseer overseer; + ShardHandlerFactory shardHandlerFactory; + String adminPath; + ZkStateReader zkStateReader; + String myId; + Overseer.Stats stats; // Set that tracks collections that are currently being processed by a running task. // This is used for handling mutual exclusion of the tasks. final private LockTree lockTree = new LockTree(); + ExecutorService tpe = new ExecutorUtil.MDCAwareThreadPoolExecutor(5, 10, 0L, TimeUnit.MILLISECONDS, + new SynchronousQueue<>(), + new DefaultSolrThreadFactory("OverseerCollectionMessageHandlerThreadFactory")); static final Random RANDOM; static { @@ -194,6 +163,8 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } } + final Map commandMap; + public OverseerCollectionMessageHandler(ZkStateReader zkStateReader, String myId, final ShardHandlerFactory shardHandlerFactory, String adminPath, @@ -206,97 +177,52 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler this.myId = myId; this.stats = stats; this.overseer = overseer; - this.overseerPrioritizer = overseerPrioritizer; + commandMap = new ImmutableMap.Builder() + .put(REPLACENODE, new ReplaceNodeCmd(this)) + .put(DELETENODE, new DeleteNodeCmd(this)) + .put(BACKUP, new BackupCmd(this)) + .put(RESTORE, new RestoreCmd(this)) + .put(SPLITSHARD, new SplitShardCmd(this)) + .put(ADDROLE, new OverseerRoleCmd(this, ADDROLE, overseerPrioritizer)) + .put(REMOVEROLE, new OverseerRoleCmd(this, REMOVEROLE, overseerPrioritizer)) + .put(MOCK_COLL_TASK, this::mockOperation) + .put(MOCK_SHARD_TASK, this::mockOperation) + .put(MOCK_REPLICA_TASK, this::mockOperation) + .put(MIGRATESTATEFORMAT, this::migrateStateFormat) + .put(CREATESHARD, new CreateShardCmd(this)) + .put(MIGRATE, new MigrateCmd(this)) + .put(CREATE, new CreateCollectionCmd(this)) + .put(MODIFYCOLLECTION, this::modifyCollection) + .put(ADDREPLICAPROP, this::processReplicaAddPropertyCommand) + .put(DELETEREPLICAPROP, this::processReplicaDeletePropertyCommand) + .put(BALANCESHARDUNIQUE, this::balanceProperty) + .put(REBALANCELEADERS, this::processRebalanceLeaders) + .put(RELOAD, this::reloadCollection) + .put(DELETE, new DeleteCollectionCmd(this)) + .put(CREATEALIAS, new CreateAliasCmd(this)) + .put(DELETEALIAS, new DeleteAliasCmd(this)) + .put(OVERSEERSTATUS, new OverseerStatusCmd(this)) + .put(DELETESHARD, new DeleteShardCmd(this)) + .put(DELETEREPLICA, new DeleteReplicaCmd(this)) + .put(ADDREPLICA, new AddReplicaCmd(this)) + .build() + ; } @Override - @SuppressForbidden(reason = "Needs currentTimeMillis for mock requests") @SuppressWarnings("unchecked") public SolrResponse processMessage(ZkNodeProps message, String operation) { log.info("OverseerCollectionMessageHandler.processMessage : "+ operation + " , "+ message.toString()); NamedList results = new NamedList(); try { - CollectionParams.CollectionAction action = getCollectionAction(operation); - switch (action) { - case CREATE: - createCollection(zkStateReader.getClusterState(), message, results); - break; - case DELETE: - deleteCollection(message, results); - break; - case RELOAD: - reloadCollection(message, results); - break; - case CREATEALIAS: - createAlias(zkStateReader.getAliases(), message); - break; - case DELETEALIAS: - deleteAlias(zkStateReader.getAliases(), message); - break; - case SPLITSHARD: - splitShard(zkStateReader.getClusterState(), message, results); - break; - case DELETESHARD: - deleteShard(zkStateReader.getClusterState(), message, results); - break; - case CREATESHARD: - createShard(zkStateReader.getClusterState(), message, results); - break; - case DELETEREPLICA: - deleteReplica(zkStateReader.getClusterState(), message, results); - break; - case MIGRATE: - migrate(zkStateReader.getClusterState(), message, results); - break; - case ADDROLE: - processRoleCommand(message, operation); - break; - case REMOVEROLE: - processRoleCommand(message, operation); - break; - case ADDREPLICA: - addReplica(zkStateReader.getClusterState(), message, results); - break; - case OVERSEERSTATUS: - getOverseerStatus(message, results); - break; - case ADDREPLICAPROP: - processReplicaAddPropertyCommand(message); - break; - case DELETEREPLICAPROP: - processReplicaDeletePropertyCommand(message); - break; - case BALANCESHARDUNIQUE: - balanceProperty(message); - break; - case REBALANCELEADERS: - processRebalanceLeaders(message); - break; - case MODIFYCOLLECTION: - modifyCollection(message, results); - break; - case MIGRATESTATEFORMAT: - migrateStateFormat(message, results); - break; - case BACKUP: - processBackupAction(message, results); - break; - case RESTORE: - processRestoreAction(message, results); - break; - case MOCK_COLL_TASK: - case MOCK_SHARD_TASK: - case MOCK_REPLICA_TASK: { - //only for test purposes - Thread.sleep(message.getInt("sleep", 1)); - log.info("MOCK_TASK_EXECUTED time {} data {}",System.currentTimeMillis(), Utils.toJSONString(message)); - results.add("MOCK_FINISHED", System.currentTimeMillis()); - break; - } - default: - throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:" - + operation); + CollectionAction action = getCollectionAction(operation); + Cmd command = commandMap.get(action); + if (command != null) { + command.call(zkStateReader.getClusterState(), message, results); + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:" + + operation); } } catch (Exception e) { String collName = message.getStr("collection"); @@ -318,19 +244,23 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler return new OverseerSolrResponse(results); } - private CollectionParams.CollectionAction getCollectionAction(String operation) { - CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(operation); + @SuppressForbidden(reason = "Needs currentTimeMillis for mock requests") + private void mockOperation(ClusterState state, ZkNodeProps message, NamedList results) throws InterruptedException { + //only for test purposes + Thread.sleep(message.getInt("sleep", 1)); + log.info("MOCK_TASK_EXECUTED time {} data {}", System.currentTimeMillis(), Utils.toJSONString(message)); + results.add("MOCK_FINISHED", System.currentTimeMillis()); + } + + private CollectionAction getCollectionAction(String operation) { + CollectionAction action = CollectionAction.get(operation); if (action == null) { throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:" + operation); } return action; } - // - // TODO DWS: this class has gone out of control (too big); refactor to break it up - // - - private void reloadCollection(ZkNodeProps message, NamedList results) { + private void reloadCollection(ClusterState clusterState, ZkNodeProps message, NamedList results) { ModifiableSolrParams params = new ModifiableSolrParams(); params.set(CoreAdminParams.ACTION, CoreAdminAction.RELOAD.toString()); @@ -343,7 +273,8 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } @SuppressWarnings("unchecked") - private void processRebalanceLeaders(ZkNodeProps message) throws KeeperException, InterruptedException { + private void processRebalanceLeaders(ClusterState clusterState, ZkNodeProps message, NamedList results) + throws Exception { checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, CORE_NAME_PROP, ELECTION_NODE_PROP, CORE_NODE_NAME_PROP, BASE_URL_PROP, REJOIN_AT_HEAD_PROP); @@ -371,7 +302,8 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } @SuppressWarnings("unchecked") - private void processReplicaAddPropertyCommand(ZkNodeProps message) throws KeeperException, InterruptedException { + private void processReplicaAddPropertyCommand(ClusterState clusterState, ZkNodeProps message, NamedList results) + throws Exception { checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP, PROPERTY_VALUE_PROP); SolrZkClient zkClient = zkStateReader.getZkClient(); DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkClient); @@ -382,7 +314,8 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler inQueue.offer(Utils.toJSON(m)); } - private void processReplicaDeletePropertyCommand(ZkNodeProps message) throws KeeperException, InterruptedException { + private void processReplicaDeletePropertyCommand(ClusterState clusterState, ZkNodeProps message, NamedList results) + throws KeeperException, InterruptedException { checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP); SolrZkClient zkClient = zkStateReader.getZkClient(); DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkClient); @@ -393,7 +326,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler inQueue.offer(Utils.toJSON(m)); } - private void balanceProperty(ZkNodeProps message) throws KeeperException, InterruptedException { + private void balanceProperty(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException { if (StringUtils.isBlank(message.getStr(COLLECTION_PROP)) || StringUtils.isBlank(message.getStr(PROPERTY_PROP))) { throw new SolrException(ErrorCode.BAD_REQUEST, "The '" + COLLECTION_PROP + "' and '" + PROPERTY_PROP + @@ -407,81 +340,6 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap))); } - - @SuppressWarnings("unchecked") - private void getOverseerStatus(ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException { - String leaderNode = OverseerTaskProcessor.getLeaderNode(zkStateReader.getZkClient()); - results.add("leader", leaderNode); - Stat stat = new Stat(); - zkStateReader.getZkClient().getData("/overseer/queue",null, stat, true); - results.add("overseer_queue_size", stat.getNumChildren()); - stat = new Stat(); - zkStateReader.getZkClient().getData("/overseer/queue-work",null, stat, true); - results.add("overseer_work_queue_size", stat.getNumChildren()); - stat = new Stat(); - zkStateReader.getZkClient().getData("/overseer/collection-queue-work",null, stat, true); - results.add("overseer_collection_queue_size", stat.getNumChildren()); - - NamedList overseerStats = new NamedList(); - NamedList collectionStats = new NamedList(); - NamedList stateUpdateQueueStats = new NamedList(); - NamedList workQueueStats = new NamedList(); - NamedList collectionQueueStats = new NamedList(); - for (Map.Entry entry : stats.getStats().entrySet()) { - String key = entry.getKey(); - NamedList lst = new SimpleOrderedMap<>(); - if (key.startsWith("collection_")) { - collectionStats.add(key.substring(11), lst); - int successes = stats.getSuccessCount(entry.getKey()); - int errors = stats.getErrorCount(entry.getKey()); - lst.add("requests", successes); - lst.add("errors", errors); - List failureDetails = stats.getFailureDetails(key); - if (failureDetails != null) { - List> failures = new ArrayList<>(); - for (Overseer.FailedOp failedOp : failureDetails) { - SimpleOrderedMap fail = new SimpleOrderedMap<>(); - fail.add("request", failedOp.req.getProperties()); - fail.add("response", failedOp.resp.getResponse()); - failures.add(fail); - } - lst.add("recent_failures", failures); - } - } else if (key.startsWith("/overseer/queue_")) { - stateUpdateQueueStats.add(key.substring(16), lst); - } else if (key.startsWith("/overseer/queue-work_")) { - workQueueStats.add(key.substring(21), lst); - } else if (key.startsWith("/overseer/collection-queue-work_")) { - collectionQueueStats.add(key.substring(32), lst); - } else { - // overseer stats - overseerStats.add(key, lst); - int successes = stats.getSuccessCount(entry.getKey()); - int errors = stats.getErrorCount(entry.getKey()); - lst.add("requests", successes); - lst.add("errors", errors); - } - Timer timer = entry.getValue().requestTime; - Snapshot snapshot = timer.getSnapshot(); - lst.add("totalTime", timer.getSum()); - lst.add("avgRequestsPerMinute", timer.getMeanRate()); - lst.add("5minRateRequestsPerMinute", timer.getFiveMinuteRate()); - lst.add("15minRateRequestsPerMinute", timer.getFifteenMinuteRate()); - lst.add("avgTimePerRequest", timer.getMean()); - lst.add("medianRequestTime", snapshot.getMedian()); - lst.add("75thPctlRequestTime", snapshot.get75thPercentile()); - lst.add("95thPctlRequestTime", snapshot.get95thPercentile()); - lst.add("99thPctlRequestTime", snapshot.get99thPercentile()); - lst.add("999thPctlRequestTime", snapshot.get999thPercentile()); - } - results.add("overseer_operations", overseerStats); - results.add("collection_operations", collectionStats); - results.add("overseer_queue", stateUpdateQueueStats); - results.add("overseer_internal_queue", workQueueStats); - results.add("collection_queue", collectionQueueStats); - - } - /** * Walks the tree of collection status to verify that any replicas not reporting a "down" status is * on a live node, if any replicas reporting their status as "active" but the node is not live is @@ -548,110 +406,13 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } @SuppressWarnings("unchecked") - private void processRoleCommand(ZkNodeProps message, String operation) throws KeeperException, InterruptedException { - SolrZkClient zkClient = zkStateReader.getZkClient(); - Map roles = null; - String node = message.getStr("node"); - - String roleName = message.getStr("role"); - boolean nodeExists = false; - if(nodeExists = zkClient.exists(ZkStateReader.ROLES, true)){ - roles = (Map) Utils.fromJSON(zkClient.getData(ZkStateReader.ROLES, null, new Stat(), true)); - } else { - roles = new LinkedHashMap(1); - } - - List nodeList= (List) roles.get(roleName); - if(nodeList == null) roles.put(roleName, nodeList = new ArrayList()); - if(ADDROLE.toString().toLowerCase(Locale.ROOT).equals(operation) ){ - log.info("Overseer role added to {}", node); - if(!nodeList.contains(node)) nodeList.add(node); - } else if(REMOVEROLE.toString().toLowerCase(Locale.ROOT).equals(operation)) { - log.info("Overseer role removed from {}", node); - nodeList.remove(node); - } - - if(nodeExists){ - zkClient.setData(ZkStateReader.ROLES, Utils.toJSON(roles),true); - } else { - zkClient.create(ZkStateReader.ROLES, Utils.toJSON(roles), CreateMode.PERSISTENT,true); - } - //if there are too many nodes this command may time out. And most likely dedicated - // overseers are created when there are too many nodes . So , do this operation in a separate thread - new Thread(() -> { - try { - overseerPrioritizer.prioritizeOverseerNodes(myId); - } catch (Exception e) { - log.error("Error in prioritizing Overseer", e); - } - - }).start(); + void deleteReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete) + throws Exception { + ((DeleteReplicaCmd) commandMap.get(DELETEREPLICA)).deleteReplica(clusterState, message, results, onComplete); } - @SuppressWarnings("unchecked") - private void deleteReplica(ClusterState clusterState, ZkNodeProps message, NamedList results) - throws KeeperException, InterruptedException { - checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP); - String collectionName = message.getStr(COLLECTION_PROP); - String shard = message.getStr(SHARD_ID_PROP); - String replicaName = message.getStr(REPLICA_PROP); - - DocCollection coll = clusterState.getCollection(collectionName); - Slice slice = coll.getSlice(shard); - if (slice == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Invalid shard name : " + shard + " in collection : " + collectionName); - } - Replica replica = slice.getReplica(replicaName); - if (replica == null) { - ArrayList l = new ArrayList<>(); - for (Replica r : slice.getReplicas()) - l.add(r.getName()); - throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid replica : " + replicaName + " in shard/collection : " - + shard + "/" + collectionName + " available replicas are " + StrUtils.join(l, ',')); - } - - // If users are being safe and only want to remove a shard if it is down, they can specify onlyIfDown=true - // on the command. - if (Boolean.parseBoolean(message.getStr(ONLY_IF_DOWN)) && replica.getState() != Replica.State.DOWN) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Attempted to remove replica : " + collectionName + "/" + shard + "/" + replicaName - + " with onlyIfDown='true', but state is '" + replica.getStr(ZkStateReader.STATE_PROP) + "'"); - } - - ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); - String core = replica.getStr(ZkStateReader.CORE_NAME_PROP); - String asyncId = message.getStr(ASYNC); - Map requestMap = null; - if (asyncId != null) { - requestMap = new HashMap<>(1, 1.0f); - } - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.add(CoreAdminParams.ACTION, CoreAdminAction.UNLOAD.toString()); - params.add(CoreAdminParams.CORE, core); - - params.set(CoreAdminParams.DELETE_INDEX, message.getBool(CoreAdminParams.DELETE_INDEX, true)); - params.set(CoreAdminParams.DELETE_INSTANCE_DIR, message.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, true)); - params.set(CoreAdminParams.DELETE_DATA_DIR, message.getBool(CoreAdminParams.DELETE_DATA_DIR, true)); - - sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap); - - processResponses(results, shardHandler, false, null, asyncId, requestMap); - - //check if the core unload removed the corenode zk entry - if (waitForCoreNodeGone(collectionName, shard, replicaName, 5000)) return; - - // try and ensure core info is removed from cluster state - deleteCoreNode(collectionName, replicaName, replica, core); - if (waitForCoreNodeGone(collectionName, shard, replicaName, 30000)) return; - - throw new SolrException(ErrorCode.SERVER_ERROR, - "Could not remove replica : " + collectionName + "/" + shard + "/" + replicaName); - } - - private boolean waitForCoreNodeGone(String collectionName, String shard, String replicaName, int timeoutms) throws InterruptedException { + boolean waitForCoreNodeGone(String collectionName, String shard, String replicaName, int timeoutms) throws InterruptedException { TimeOut timeout = new TimeOut(timeoutms, TimeUnit.MILLISECONDS); boolean deleted = false; while (! timeout.hasTimedOut()) { @@ -669,7 +430,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler return deleted; } - private void deleteCoreNode(String collectionName, String replicaName, Replica replica, String core) throws KeeperException, InterruptedException { + void deleteCoreNode(String collectionName, String replicaName, Replica replica, String core) throws KeeperException, InterruptedException { ZkNodeProps m = new ZkNodeProps( Overseer.QUEUE_OPERATION, OverseerAction.DELETECORE.toLower(), ZkStateReader.CORE_NAME_PROP, core, @@ -679,7 +440,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(m)); } - private void checkRequired(ZkNodeProps message, String... props) { + void checkRequired(ZkNodeProps message, String... props) { for (String prop : props) { if(message.get(prop) == null){ throw new SolrException(ErrorCode.BAD_REQUEST, StrUtils.join(Arrays.asList(props),',') +" are required params" ); @@ -688,73 +449,8 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } - private void deleteCollection(ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException { - final String collection = message.getStr(NAME); - try { - if (zkStateReader.getClusterState().getCollectionOrNull(collection) == null) { - if (zkStateReader.getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection, true)) { - // if the collection is not in the clusterstate, but is listed in zk, do nothing, it will just - // be removed in the finally - we cannot continue, because the below code will error if the collection - // is not in the clusterstate - return; - } - } - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.UNLOAD.toString()); - params.set(CoreAdminParams.DELETE_INSTANCE_DIR, true); - params.set(CoreAdminParams.DELETE_DATA_DIR, true); - - String asyncId = message.getStr(ASYNC); - Map requestMap = null; - if (asyncId != null) { - requestMap = new HashMap<>(); - } - - Set okayExceptions = new HashSet<>(1); - okayExceptions.add(NonExistentCoreException.class.getName()); - - collectionCmd(message, params, results, null, asyncId, requestMap, okayExceptions); - - ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, collection); - Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(m)); - - // wait for a while until we don't see the collection - TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); - boolean removed = false; - while (! timeout.hasTimedOut()) { - Thread.sleep(100); - removed = !zkStateReader.getClusterState().hasCollection(collection); - if (removed) { - Thread.sleep(500); // just a bit of time so it's more likely other - // readers see on return - break; - } - } - if (!removed) { - throw new SolrException(ErrorCode.SERVER_ERROR, - "Could not fully remove collection: " + collection); - } - - } finally { - - try { - if (zkStateReader.getZkClient().exists( - ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection, true)) { - zkStateReader.getZkClient().clean( - ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection); - } - } catch (InterruptedException e) { - SolrException.log(log, "Cleaning up collection in zk was interrupted:" - + collection, e); - Thread.currentThread().interrupt(); - } catch (KeeperException e) { - SolrException.log(log, "Problem cleaning up collection in zk:" - + collection, e); - } - } - } - - private void migrateStateFormat(ZkNodeProps message, NamedList results) + //TODO should we not remove in the next release ? + private void migrateStateFormat(ClusterState state, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException { final String collectionName = message.getStr(COLLECTION_PROP); @@ -783,549 +479,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler throw new SolrException(ErrorCode.SERVER_ERROR, "Could not migrate state format for collection: " + collectionName); } - private void createAlias(Aliases aliases, ZkNodeProps message) { - String aliasName = message.getStr(NAME); - String collections = message.getStr("collections"); - - Map> newAliasesMap = new HashMap<>(); - Map newCollectionAliasesMap = new HashMap<>(); - Map prevColAliases = aliases.getCollectionAliasMap(); - if (prevColAliases != null) { - newCollectionAliasesMap.putAll(prevColAliases); - } - newCollectionAliasesMap.put(aliasName, collections); - newAliasesMap.put("collection", newCollectionAliasesMap); - Aliases newAliases = new Aliases(newAliasesMap); - byte[] jsonBytes = null; - if (newAliases.collectionAliasSize() > 0) { // only sub map right now - jsonBytes = Utils.toJSON(newAliases.getAliasMap()); - } - try { - zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, jsonBytes, true); - - checkForAlias(aliasName, collections); - // some fudge for other nodes - Thread.sleep(100); - } catch (KeeperException e) { - log.error("", e); - throw new SolrException(ErrorCode.SERVER_ERROR, e); - } catch (InterruptedException e) { - log.warn("", e); - throw new SolrException(ErrorCode.SERVER_ERROR, e); - } - } - - private void checkForAlias(String name, String value) { - - TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); - boolean success = false; - Aliases aliases; - while (! timeout.hasTimedOut()) { - aliases = zkStateReader.getAliases(); - String collections = aliases.getCollectionAlias(name); - if (collections != null && collections.equals(value)) { - success = true; - break; - } - } - if (!success) { - log.warn("Timeout waiting to be notified of Alias change..."); - } - } - - private void checkForAliasAbsence(String name) { - - TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); - boolean success = false; - Aliases aliases = null; - while (! timeout.hasTimedOut()) { - aliases = zkStateReader.getAliases(); - String collections = aliases.getCollectionAlias(name); - if (collections == null) { - success = true; - break; - } - } - if (!success) { - log.warn("Timeout waiting to be notified of Alias change..."); - } - } - - private void deleteAlias(Aliases aliases, ZkNodeProps message) { - String aliasName = message.getStr(NAME); - - Map> newAliasesMap = new HashMap<>(); - Map newCollectionAliasesMap = new HashMap<>(); - newCollectionAliasesMap.putAll(aliases.getCollectionAliasMap()); - newCollectionAliasesMap.remove(aliasName); - newAliasesMap.put("collection", newCollectionAliasesMap); - Aliases newAliases = new Aliases(newAliasesMap); - byte[] jsonBytes = null; - if (newAliases.collectionAliasSize() > 0) { // only sub map right now - jsonBytes = Utils.toJSON(newAliases.getAliasMap()); - } - try { - zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, - jsonBytes, true); - checkForAliasAbsence(aliasName); - // some fudge for other nodes - Thread.sleep(100); - } catch (KeeperException e) { - log.error("", e); - throw new SolrException(ErrorCode.SERVER_ERROR, e); - } catch (InterruptedException e) { - log.warn("", e); - throw new SolrException(ErrorCode.SERVER_ERROR, e); - } - - } - - private boolean createShard(ClusterState clusterState, ZkNodeProps message, NamedList results) - throws KeeperException, InterruptedException { - String collectionName = message.getStr(COLLECTION_PROP); - String sliceName = message.getStr(SHARD_ID_PROP); - - log.info("Create shard invoked: {}", message); - if (collectionName == null || sliceName == null) - throw new SolrException(ErrorCode.BAD_REQUEST, "'collection' and 'shard' are required parameters"); - int numSlices = 1; - - ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); - DocCollection collection = clusterState.getCollection(collectionName); - int repFactor = message.getInt(REPLICATION_FACTOR, collection.getInt(REPLICATION_FACTOR, 1)); - String createNodeSetStr = message.getStr(CREATE_NODE_SET); - List sortedNodeList = getNodesForNewReplicas(clusterState, collectionName, sliceName, repFactor, - createNodeSetStr, overseer.getZkController().getCoreContainer()); - - Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message)); - // wait for a while until we see the shard - TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); - boolean created = false; - while (! timeout.hasTimedOut()) { - Thread.sleep(100); - created = zkStateReader.getClusterState().getCollection(collectionName).getSlice(sliceName) != null; - if (created) break; - } - if (!created) - throw new SolrException(ErrorCode.SERVER_ERROR, "Could not fully create shard: " + message.getStr(NAME)); - - String configName = message.getStr(COLL_CONF); - - String async = message.getStr(ASYNC); - Map requestMap = null; - if (async != null) { - requestMap = new HashMap<>(repFactor, 1.0f); - } - - for (int j = 1; j <= repFactor; j++) { - String nodeName = sortedNodeList.get(((j - 1)) % sortedNodeList.size()).nodeName; - String shardName = collectionName + "_" + sliceName + "_replica" + j; - log.info("Creating shard " + shardName + " as part of slice " + sliceName + " of collection " + collectionName - + " on " + nodeName); - - // Need to create new params for each request - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString()); - params.set(CoreAdminParams.NAME, shardName); - params.set(COLL_CONF, configName); - params.set(CoreAdminParams.COLLECTION, collectionName); - params.set(CoreAdminParams.SHARD, sliceName); - params.set(ZkStateReader.NUM_SHARDS_PROP, numSlices); - addPropertyParams(message, params); - - sendShardRequest(nodeName, params, shardHandler, async, requestMap); - } - - processResponses(results, shardHandler, true, "Failed to create shard", async, requestMap, Collections.emptySet()); - - log.info("Finished create command on all shards for collection: " + collectionName); - - return true; - } - - - private boolean splitShard(ClusterState clusterState, ZkNodeProps message, NamedList results) { - String collectionName = message.getStr("collection"); - String slice = message.getStr(ZkStateReader.SHARD_ID_PROP); - - log.info("Split shard invoked"); - String splitKey = message.getStr("split.key"); - ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); - - DocCollection collection = clusterState.getCollection(collectionName); - DocRouter router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT; - - Slice parentSlice; - - if (slice == null) { - if (router instanceof CompositeIdRouter) { - Collection searchSlices = router.getSearchSlicesSingle(splitKey, new ModifiableSolrParams(), collection); - if (searchSlices.isEmpty()) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an active shard for split.key: " + splitKey); - } - if (searchSlices.size() > 1) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Splitting a split.key: " + splitKey + " which spans multiple shards is not supported"); - } - parentSlice = searchSlices.iterator().next(); - slice = parentSlice.getName(); - log.info("Split by route.key: {}, parent shard is: {} ", splitKey, slice); - } else { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Split by route key can only be used with CompositeIdRouter or subclass. Found router: " - + router.getClass().getName()); - } - } else { - parentSlice = collection.getSlice(slice); - } - - if (parentSlice == null) { - // no chance of the collection being null because ClusterState#getCollection(String) would have thrown - // an exception already - throw new SolrException(ErrorCode.BAD_REQUEST, "No shard with the specified name exists: " + slice); - } - - // find the leader for the shard - Replica parentShardLeader = null; - try { - parentShardLeader = zkStateReader.getLeaderRetry(collectionName, slice, 10000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - DocRouter.Range range = parentSlice.getRange(); - if (range == null) { - range = new PlainIdRouter().fullRange(); - } - - List subRanges = null; - String rangesStr = message.getStr(CoreAdminParams.RANGES); - if (rangesStr != null) { - String[] ranges = rangesStr.split(","); - if (ranges.length == 0 || ranges.length == 1) { - throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split a shard"); - } else { - subRanges = new ArrayList<>(ranges.length); - for (int i = 0; i < ranges.length; i++) { - String r = ranges[i]; - try { - subRanges.add(DocRouter.DEFAULT.fromString(r)); - } catch (Exception e) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Exception in parsing hexadecimal hash range: " + r, e); - } - if (!subRanges.get(i).isSubsetOf(range)) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Specified hash range: " + r + " is not a subset of parent shard's range: " + range.toString()); - } - } - List temp = new ArrayList<>(subRanges); // copy to preserve original order - Collections.sort(temp); - if (!range.equals(new DocRouter.Range(temp.get(0).min, temp.get(temp.size() - 1).max))) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Specified hash ranges: " + rangesStr + " do not cover the entire range of parent shard: " + range); - } - for (int i = 1; i < temp.size(); i++) { - if (temp.get(i - 1).max + 1 != temp.get(i).min) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Specified hash ranges: " + rangesStr - + " either overlap with each other or " + "do not cover the entire range of parent shard: " + range); - } - } - } - } else if (splitKey != null) { - if (router instanceof CompositeIdRouter) { - CompositeIdRouter compositeIdRouter = (CompositeIdRouter) router; - subRanges = compositeIdRouter.partitionRangeByKey(splitKey, range); - if (subRanges.size() == 1) { - throw new SolrException(ErrorCode.BAD_REQUEST, "The split.key: " + splitKey - + " has a hash range that is exactly equal to hash range of shard: " + slice); - } - for (DocRouter.Range subRange : subRanges) { - if (subRange.min == subRange.max) { - throw new SolrException(ErrorCode.BAD_REQUEST, "The split.key: " + splitKey + " must be a compositeId"); - } - } - log.info("Partitioning parent shard " + slice + " range: " + parentSlice.getRange() + " yields: " + subRanges); - rangesStr = ""; - for (int i = 0; i < subRanges.size(); i++) { - DocRouter.Range subRange = subRanges.get(i); - rangesStr += subRange.toString(); - if (i < subRanges.size() - 1) rangesStr += ','; - } - } - } else { - // todo: fixed to two partitions? - subRanges = router.partitionRange(2, range); - } - - try { - List subSlices = new ArrayList<>(subRanges.size()); - List subShardNames = new ArrayList<>(subRanges.size()); - String nodeName = parentShardLeader.getNodeName(); - for (int i = 0; i < subRanges.size(); i++) { - String subSlice = slice + "_" + i; - subSlices.add(subSlice); - String subShardName = collectionName + "_" + subSlice + "_replica1"; - subShardNames.add(subShardName); - - Slice oSlice = collection.getSlice(subSlice); - if (oSlice != null) { - final Slice.State state = oSlice.getState(); - if (state == Slice.State.ACTIVE) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Sub-shard: " + subSlice + " exists in active state. Aborting split shard."); - } else if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY) { - // delete the shards - for (String sub : subSlices) { - log.info("Sub-shard: {} already exists therefore requesting its deletion", sub); - Map propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, "deleteshard"); - propMap.put(COLLECTION_PROP, collectionName); - propMap.put(SHARD_ID_PROP, sub); - ZkNodeProps m = new ZkNodeProps(propMap); - try { - deleteShard(clusterState, m, new NamedList()); - } catch (Exception e) { - throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to delete already existing sub shard: " + sub, - e); - } - } - } - } - } - - final String asyncId = message.getStr(ASYNC); - Map requestMap = new HashMap<>(); - - for (int i = 0; i < subRanges.size(); i++) { - String subSlice = subSlices.get(i); - String subShardName = subShardNames.get(i); - DocRouter.Range subRange = subRanges.get(i); - - log.info("Creating slice " + subSlice + " of collection " + collectionName + " on " + nodeName); - - Map propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD.toLower()); - propMap.put(ZkStateReader.SHARD_ID_PROP, subSlice); - propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); - propMap.put(ZkStateReader.SHARD_RANGE_PROP, subRange.toString()); - propMap.put(ZkStateReader.SHARD_STATE_PROP, Slice.State.CONSTRUCTION.toString()); - propMap.put(ZkStateReader.SHARD_PARENT_PROP, parentSlice.getName()); - DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient()); - inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap))); - - // wait until we are able to see the new shard in cluster state - waitForNewShard(collectionName, subSlice); - - // refresh cluster state - clusterState = zkStateReader.getClusterState(); - - log.info("Adding replica " + subShardName + " as part of slice " + subSlice + " of collection " + collectionName - + " on " + nodeName); - propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower()); - propMap.put(COLLECTION_PROP, collectionName); - propMap.put(SHARD_ID_PROP, subSlice); - propMap.put("node", nodeName); - propMap.put(CoreAdminParams.NAME, subShardName); - // copy over property params: - for (String key : message.keySet()) { - if (key.startsWith(COLL_PROP_PREFIX)) { - propMap.put(key, message.getStr(key)); - } - } - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } - addReplica(clusterState, new ZkNodeProps(propMap), results); - } - - processResponses(results, shardHandler, true, "SPLITSHARD failed to create subshard leaders", asyncId, requestMap); - - for (String subShardName : subShardNames) { - // wait for parent leader to acknowledge the sub-shard core - log.info("Asking parent leader to wait for: " + subShardName + " to be alive on: " + nodeName); - String coreNodeName = waitForCoreNodeName(collectionName, nodeName, subShardName); - CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState(); - cmd.setCoreName(subShardName); - cmd.setNodeName(nodeName); - cmd.setCoreNodeName(coreNodeName); - cmd.setState(Replica.State.ACTIVE); - cmd.setCheckLive(true); - cmd.setOnlyIfLeader(true); - - ModifiableSolrParams p = new ModifiableSolrParams(cmd.getParams()); - sendShardRequest(nodeName, p, shardHandler, asyncId, requestMap); - } - - processResponses(results, shardHandler, true, "SPLITSHARD timed out waiting for subshard leaders to come up", - asyncId, requestMap); - - log.info("Successfully created all sub-shards for collection " + collectionName + " parent shard: " + slice - + " on: " + parentShardLeader); - - log.info("Splitting shard " + parentShardLeader.getName() + " as part of slice " + slice + " of collection " - + collectionName + " on " + parentShardLeader); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.SPLIT.toString()); - params.set(CoreAdminParams.CORE, parentShardLeader.getStr("core")); - for (int i = 0; i < subShardNames.size(); i++) { - String subShardName = subShardNames.get(i); - params.add(CoreAdminParams.TARGET_CORE, subShardName); - } - params.set(CoreAdminParams.RANGES, rangesStr); - - sendShardRequest(parentShardLeader.getNodeName(), params, shardHandler, asyncId, requestMap); - - processResponses(results, shardHandler, true, "SPLITSHARD failed to invoke SPLIT core admin command", asyncId, - requestMap); - - log.info("Index on shard: " + nodeName + " split into two successfully"); - - // apply buffered updates on sub-shards - for (int i = 0; i < subShardNames.size(); i++) { - String subShardName = subShardNames.get(i); - - log.info("Applying buffered updates on : " + subShardName); - - params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.REQUESTAPPLYUPDATES.toString()); - params.set(CoreAdminParams.NAME, subShardName); - - sendShardRequest(nodeName, params, shardHandler, asyncId, requestMap); - } - - processResponses(results, shardHandler, true, "SPLITSHARD failed while asking sub shard leaders" + - " to apply buffered updates", asyncId, requestMap); - - log.info("Successfully applied buffered updates on : " + subShardNames); - - // Replica creation for the new Slices - - // look at the replication factor and see if it matches reality - // if it does not, find best nodes to create more cores - - // TODO: Have replication factor decided in some other way instead of numShards for the parent - - int repFactor = parentSlice.getReplicas().size(); - - // we need to look at every node and see how many cores it serves - // add our new cores to existing nodes serving the least number of cores - // but (for now) require that each core goes on a distinct node. - - // TODO: add smarter options that look at the current number of cores per - // node? - // for now we just go random - Set nodes = clusterState.getLiveNodes(); - List nodeList = new ArrayList<>(nodes.size()); - nodeList.addAll(nodes); - - // TODO: Have maxShardsPerNode param for this operation? - - // Remove the node that hosts the parent shard for replica creation. - nodeList.remove(nodeName); - - // TODO: change this to handle sharding a slice into > 2 sub-shards. - - - Map nodeMap = identifyNodes(clusterState, - new ArrayList<>(clusterState.getLiveNodes()), - new ZkNodeProps(collection.getProperties()), - subSlices, repFactor - 1); - - List> replicas = new ArrayList<>((repFactor - 1) * 2); - - for (Map.Entry entry : nodeMap.entrySet()) { - String sliceName = entry.getKey().shard; - String subShardNodeName = entry.getValue(); - String shardName = collectionName + "_" + sliceName + "_replica" + (entry.getKey().index); - - log.info("Creating replica shard " + shardName + " as part of slice " + sliceName + " of collection " - + collectionName + " on " + subShardNodeName); - - ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(), - ZkStateReader.COLLECTION_PROP, collectionName, - ZkStateReader.SHARD_ID_PROP, sliceName, - ZkStateReader.CORE_NAME_PROP, shardName, - ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), - ZkStateReader.BASE_URL_PROP, zkStateReader.getBaseUrlForNodeName(subShardNodeName), - ZkStateReader.NODE_NAME_PROP, subShardNodeName); - Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(props)); - - HashMap propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower()); - propMap.put(COLLECTION_PROP, collectionName); - propMap.put(SHARD_ID_PROP, sliceName); - propMap.put("node", subShardNodeName); - propMap.put(CoreAdminParams.NAME, shardName); - // copy over property params: - for (String key : message.keySet()) { - if (key.startsWith(COLL_PROP_PREFIX)) { - propMap.put(key, message.getStr(key)); - } - } - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } - // special flag param to instruct addReplica not to create the replica in cluster state again - propMap.put(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, "true"); - - replicas.add(propMap); - } - - // we must set the slice state into recovery before actually creating the replica cores - // this ensures that the logic inside Overseer to update sub-shard state to 'active' - // always gets a chance to execute. See SOLR-7673 - - if (repFactor == 1) { - // switch sub shard states to 'active' - log.info("Replication factor is 1 so switching shard states"); - DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient()); - Map propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); - propMap.put(slice, Slice.State.INACTIVE.toString()); - for (String subSlice : subSlices) { - propMap.put(subSlice, Slice.State.ACTIVE.toString()); - } - propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); - ZkNodeProps m = new ZkNodeProps(propMap); - inQueue.offer(Utils.toJSON(m)); - } else { - log.info("Requesting shard state be set to 'recovery'"); - DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient()); - Map propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); - for (String subSlice : subSlices) { - propMap.put(subSlice, Slice.State.RECOVERY.toString()); - } - propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); - ZkNodeProps m = new ZkNodeProps(propMap); - inQueue.offer(Utils.toJSON(m)); - } - - // now actually create replica cores on sub shard nodes - for (Map replica : replicas) { - addReplica(clusterState, new ZkNodeProps(replica), results); - } - - processResponses(results, shardHandler, true, "SPLITSHARD failed to create subshard replicas", asyncId, requestMap); - - log.info("Successfully created all replica shards for all sub-slices " + subSlices); - - commit(results, slice, parentShardLeader); - - return true; - } catch (SolrException e) { - throw e; - } catch (Exception e) { - log.error("Error executing split operation for collection: " + collectionName + " parent shard: " + slice, e); - throw new SolrException(ErrorCode.SERVER_ERROR, null, e); - } - } - - private void commit(NamedList results, String slice, Replica parentShardLeader) { + void commit(NamedList results, String slice, Replica parentShardLeader) { log.info("Calling soft commit to make sub shard updates visible"); String coreUrl = new ZkCoreNodeProps(parentShardLeader).getCoreUrl(); // HttpShardHandler is hard coded to send a QueryRequest hence we go direct @@ -1353,7 +507,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } } - private String waitForCoreNodeName(String collectionName, String msgNodeName, String msgCore) { + String waitForCoreNodeName(String collectionName, String msgNodeName, String msgCore) { int retryCount = 320; while (retryCount-- > 0) { Map slicesMap = zkStateReader.getClusterState() @@ -1382,7 +536,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find coreNodeName"); } - private void waitForNewShard(String collectionName, String sliceName) throws KeeperException, InterruptedException { + void waitForNewShard(String collectionName, String sliceName) throws KeeperException, InterruptedException { log.info("Waiting for slice {} of collection {} to be available", sliceName, collectionName); RTimer timer = new RTimer(); int retryCount = 320; @@ -1406,339 +560,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler ); } - private void deleteShard(ClusterState clusterState, ZkNodeProps message, NamedList results) { - String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP); - String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP); - - log.info("Delete shard invoked"); - Slice slice = clusterState.getSlice(collectionName, sliceId); - - if (slice == null) { - if (clusterState.hasCollection(collectionName)) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "No shard with name " + sliceId + " exists for collection " + collectionName); - } else { - throw new SolrException(ErrorCode.BAD_REQUEST, "No collection with the specified name exists: " + collectionName); - } - } - // For now, only allow for deletions of Inactive slices or custom hashes (range==null). - // TODO: Add check for range gaps on Slice deletion - final Slice.State state = slice.getState(); - if (!(slice.getRange() == null || state == Slice.State.INACTIVE || state == Slice.State.RECOVERY - || state == Slice.State.CONSTRUCTION)) { - throw new SolrException(ErrorCode.BAD_REQUEST, "The slice: " + slice.getName() + " is currently " + state - + ". Only non-active (or custom-hashed) slices can be deleted."); - } - ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); - - String asyncId = message.getStr(ASYNC); - Map requestMap = null; - if (asyncId != null) { - requestMap = new HashMap<>(slice.getReplicas().size(), 1.0f); - } - - try { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.UNLOAD.toString()); - params.set(CoreAdminParams.DELETE_INDEX, message.getBool(CoreAdminParams.DELETE_INDEX, true)); - params.set(CoreAdminParams.DELETE_INSTANCE_DIR, message.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, true)); - params.set(CoreAdminParams.DELETE_DATA_DIR, message.getBool(CoreAdminParams.DELETE_DATA_DIR, true)); - - sliceCmd(clusterState, params, null, slice, shardHandler, asyncId, requestMap); - - processResponses(results, shardHandler, true, "Failed to delete shard", asyncId, requestMap, Collections.emptySet()); - - ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, DELETESHARD.toLower(), ZkStateReader.COLLECTION_PROP, - collectionName, ZkStateReader.SHARD_ID_PROP, sliceId); - Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(m)); - - // wait for a while until we don't see the shard - TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); - boolean removed = false; - while (! timeout.hasTimedOut()) { - Thread.sleep(100); - DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName); - removed = collection.getSlice(sliceId) == null; - if (removed) { - Thread.sleep(100); // just a bit of time so it's more likely other readers see on return - break; - } - } - if (!removed) { - throw new SolrException(ErrorCode.SERVER_ERROR, - "Could not fully remove collection: " + collectionName + " shard: " + sliceId); - } - - log.info("Successfully deleted collection: " + collectionName + ", shard: " + sliceId); - - } catch (SolrException e) { - throw e; - } catch (Exception e) { - throw new SolrException(ErrorCode.SERVER_ERROR, - "Error executing delete operation for collection: " + collectionName + " shard: " + sliceId, e); - } - } - - private void migrate(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException { - String sourceCollectionName = message.getStr("collection"); - String splitKey = message.getStr("split.key"); - String targetCollectionName = message.getStr("target.collection"); - int timeout = message.getInt("forward.timeout", 10 * 60) * 1000; - - DocCollection sourceCollection = clusterState.getCollection(sourceCollectionName); - if (sourceCollection == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown source collection: " + sourceCollectionName); - } - DocCollection targetCollection = clusterState.getCollection(targetCollectionName); - if (targetCollection == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown target collection: " + sourceCollectionName); - } - if (!(sourceCollection.getRouter() instanceof CompositeIdRouter)) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Source collection must use a compositeId router"); - } - if (!(targetCollection.getRouter() instanceof CompositeIdRouter)) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Target collection must use a compositeId router"); - } - CompositeIdRouter sourceRouter = (CompositeIdRouter) sourceCollection.getRouter(); - CompositeIdRouter targetRouter = (CompositeIdRouter) targetCollection.getRouter(); - Collection sourceSlices = sourceRouter.getSearchSlicesSingle(splitKey, null, sourceCollection); - if (sourceSlices.isEmpty()) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "No active slices available in source collection: " + sourceCollection + "for given split.key: " + splitKey); - } - Collection targetSlices = targetRouter.getSearchSlicesSingle(splitKey, null, targetCollection); - if (targetSlices.isEmpty()) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "No active slices available in target collection: " + targetCollection + "for given split.key: " + splitKey); - } - - String asyncId = null; - if(message.containsKey(ASYNC) && message.get(ASYNC) != null) - asyncId = message.getStr(ASYNC); - - for (Slice sourceSlice : sourceSlices) { - for (Slice targetSlice : targetSlices) { - log.info("Migrating source shard: {} to target shard: {} for split.key = " + splitKey, sourceSlice, targetSlice); - migrateKey(clusterState, sourceCollection, sourceSlice, targetCollection, targetSlice, splitKey, - timeout, results, asyncId, message); - } - } - } - - private void migrateKey(ClusterState clusterState, DocCollection sourceCollection, Slice sourceSlice, - DocCollection targetCollection, Slice targetSlice, - String splitKey, int timeout, - NamedList results, String asyncId, ZkNodeProps message) throws KeeperException, InterruptedException { - String tempSourceCollectionName = "split_" + sourceSlice.getName() + "_temp_" + targetSlice.getName(); - if (clusterState.hasCollection(tempSourceCollectionName)) { - log.info("Deleting temporary collection: " + tempSourceCollectionName); - Map props = makeMap( - Overseer.QUEUE_OPERATION, DELETE.toLower(), - NAME, tempSourceCollectionName); - - try { - deleteCollection(new ZkNodeProps(props), results); - clusterState = zkStateReader.getClusterState(); - } catch (Exception e) { - log.warn("Unable to clean up existing temporary collection: " + tempSourceCollectionName, e); - } - } - - CompositeIdRouter sourceRouter = (CompositeIdRouter) sourceCollection.getRouter(); - DocRouter.Range keyHashRange = sourceRouter.keyHashRange(splitKey); - - ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); - - log.info("Hash range for split.key: {} is: {}", splitKey, keyHashRange); - // intersect source range, keyHashRange and target range - // this is the range that has to be split from source and transferred to target - DocRouter.Range splitRange = intersect(targetSlice.getRange(), intersect(sourceSlice.getRange(), keyHashRange)); - if (splitRange == null) { - log.info("No common hashes between source shard: {} and target shard: {}", sourceSlice.getName(), targetSlice.getName()); - return; - } - log.info("Common hash range between source shard: {} and target shard: {} = " + splitRange, sourceSlice.getName(), targetSlice.getName()); - - Replica targetLeader = zkStateReader.getLeaderRetry(targetCollection.getName(), targetSlice.getName(), 10000); - // For tracking async calls. - Map requestMap = new HashMap<>(); - - log.info("Asking target leader node: " + targetLeader.getNodeName() + " core: " - + targetLeader.getStr("core") + " to buffer updates"); - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.REQUESTBUFFERUPDATES.toString()); - params.set(CoreAdminParams.NAME, targetLeader.getStr("core")); - - sendShardRequest(targetLeader.getNodeName(), params, shardHandler, asyncId, requestMap); - - processResponses(results, shardHandler, true, "MIGRATE failed to request node to buffer updates", asyncId, requestMap); - - ZkNodeProps m = new ZkNodeProps( - Overseer.QUEUE_OPERATION, OverseerAction.ADDROUTINGRULE.toLower(), - COLLECTION_PROP, sourceCollection.getName(), - SHARD_ID_PROP, sourceSlice.getName(), - "routeKey", SolrIndexSplitter.getRouteKey(splitKey) + "!", - "range", splitRange.toString(), - "targetCollection", targetCollection.getName(), - "expireAt", RoutingRule.makeExpiryAt(timeout)); - log.info("Adding routing rule: " + m); - Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(m)); - - // wait for a while until we see the new rule - log.info("Waiting to see routing rule updated in clusterstate"); - TimeOut waitUntil = new TimeOut(60, TimeUnit.SECONDS); - boolean added = false; - while (! waitUntil.hasTimedOut()) { - Thread.sleep(100); - sourceCollection = zkStateReader.getClusterState().getCollection(sourceCollection.getName()); - sourceSlice = sourceCollection.getSlice(sourceSlice.getName()); - Map rules = sourceSlice.getRoutingRules(); - if (rules != null) { - RoutingRule rule = rules.get(SolrIndexSplitter.getRouteKey(splitKey) + "!"); - if (rule != null && rule.getRouteRanges().contains(splitRange)) { - added = true; - break; - } - } - } - if (!added) { - throw new SolrException(ErrorCode.SERVER_ERROR, "Could not add routing rule: " + m); - } - - log.info("Routing rule added successfully"); - - // Create temp core on source shard - Replica sourceLeader = zkStateReader.getLeaderRetry(sourceCollection.getName(), sourceSlice.getName(), 10000); - - // create a temporary collection with just one node on the shard leader - String configName = zkStateReader.readConfigName(sourceCollection.getName()); - Map props = makeMap( - Overseer.QUEUE_OPERATION, CREATE.toLower(), - NAME, tempSourceCollectionName, - REPLICATION_FACTOR, 1, - NUM_SLICES, 1, - COLL_CONF, configName, - CREATE_NODE_SET, sourceLeader.getNodeName()); - if (asyncId != null) { - String internalAsyncId = asyncId + Math.abs(System.nanoTime()); - props.put(ASYNC, internalAsyncId); - } - - log.info("Creating temporary collection: " + props); - createCollection(clusterState, new ZkNodeProps(props), results); - // refresh cluster state - clusterState = zkStateReader.getClusterState(); - Slice tempSourceSlice = clusterState.getCollection(tempSourceCollectionName).getSlices().iterator().next(); - Replica tempSourceLeader = zkStateReader.getLeaderRetry(tempSourceCollectionName, tempSourceSlice.getName(), 120000); - - String tempCollectionReplica1 = tempSourceCollectionName + "_" + tempSourceSlice.getName() + "_replica1"; - String coreNodeName = waitForCoreNodeName(tempSourceCollectionName, - sourceLeader.getNodeName(), tempCollectionReplica1); - // wait for the replicas to be seen as active on temp source leader - log.info("Asking source leader to wait for: " + tempCollectionReplica1 + " to be alive on: " + sourceLeader.getNodeName()); - CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState(); - cmd.setCoreName(tempCollectionReplica1); - cmd.setNodeName(sourceLeader.getNodeName()); - cmd.setCoreNodeName(coreNodeName); - cmd.setState(Replica.State.ACTIVE); - cmd.setCheckLive(true); - cmd.setOnlyIfLeader(true); - // we don't want this to happen asynchronously - sendShardRequest(tempSourceLeader.getNodeName(), new ModifiableSolrParams(cmd.getParams()), shardHandler, null, null); - - processResponses(results, shardHandler, true, "MIGRATE failed to create temp collection leader" + - " or timed out waiting for it to come up", asyncId, requestMap); - - log.info("Asking source leader to split index"); - params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.SPLIT.toString()); - params.set(CoreAdminParams.CORE, sourceLeader.getStr("core")); - params.add(CoreAdminParams.TARGET_CORE, tempSourceLeader.getStr("core")); - params.set(CoreAdminParams.RANGES, splitRange.toString()); - params.set("split.key", splitKey); - - String tempNodeName = sourceLeader.getNodeName(); - - sendShardRequest(tempNodeName, params, shardHandler, asyncId, requestMap); - processResponses(results, shardHandler, true, "MIGRATE failed to invoke SPLIT core admin command", asyncId, requestMap); - - log.info("Creating a replica of temporary collection: {} on the target leader node: {}", - tempSourceCollectionName, targetLeader.getNodeName()); - String tempCollectionReplica2 = tempSourceCollectionName + "_" + tempSourceSlice.getName() + "_replica2"; - props = new HashMap<>(); - props.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower()); - props.put(COLLECTION_PROP, tempSourceCollectionName); - props.put(SHARD_ID_PROP, tempSourceSlice.getName()); - props.put("node", targetLeader.getNodeName()); - props.put(CoreAdminParams.NAME, tempCollectionReplica2); - // copy over property params: - for (String key : message.keySet()) { - if (key.startsWith(COLL_PROP_PREFIX)) { - props.put(key, message.getStr(key)); - } - } - // add async param - if(asyncId != null) { - props.put(ASYNC, asyncId); - } - addReplica(clusterState, new ZkNodeProps(props), results); - - processResponses(results, shardHandler, true, "MIGRATE failed to create replica of " + - "temporary collection in target leader node.", asyncId, requestMap); - - coreNodeName = waitForCoreNodeName(tempSourceCollectionName, - targetLeader.getNodeName(), tempCollectionReplica2); - // wait for the replicas to be seen as active on temp source leader - log.info("Asking temp source leader to wait for: " + tempCollectionReplica2 + " to be alive on: " + targetLeader.getNodeName()); - cmd = new CoreAdminRequest.WaitForState(); - cmd.setCoreName(tempSourceLeader.getStr("core")); - cmd.setNodeName(targetLeader.getNodeName()); - cmd.setCoreNodeName(coreNodeName); - cmd.setState(Replica.State.ACTIVE); - cmd.setCheckLive(true); - cmd.setOnlyIfLeader(true); - params = new ModifiableSolrParams(cmd.getParams()); - - sendShardRequest(tempSourceLeader.getNodeName(), params, shardHandler, asyncId, requestMap); - - processResponses(results, shardHandler, true, "MIGRATE failed to create temp collection" + - " replica or timed out waiting for them to come up", asyncId, requestMap); - - log.info("Successfully created replica of temp source collection on target leader node"); - - log.info("Requesting merge of temp source collection replica to target leader"); - params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.MERGEINDEXES.toString()); - params.set(CoreAdminParams.CORE, targetLeader.getStr("core")); - params.set(CoreAdminParams.SRC_CORE, tempCollectionReplica2); - - sendShardRequest(targetLeader.getNodeName(), params, shardHandler, asyncId, requestMap); - String msg = "MIGRATE failed to merge " + tempCollectionReplica2 + " to " - + targetLeader.getStr("core") + " on node: " + targetLeader.getNodeName(); - processResponses(results, shardHandler, true, msg, asyncId, requestMap); - - log.info("Asking target leader to apply buffered updates"); - params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.REQUESTAPPLYUPDATES.toString()); - params.set(CoreAdminParams.NAME, targetLeader.getStr("core")); - - sendShardRequest(targetLeader.getNodeName(), params, shardHandler, asyncId, requestMap); - processResponses(results, shardHandler, true, "MIGRATE failed to request node to apply buffered updates", - asyncId, requestMap); - - try { - log.info("Deleting temporary collection: " + tempSourceCollectionName); - props = makeMap( - Overseer.QUEUE_OPERATION, DELETE.toLower(), - NAME, tempSourceCollectionName); - deleteCollection(new ZkNodeProps(props), results); - } catch (Exception e) { - log.error("Unable to delete temporary collection: " + tempSourceCollectionName - + ". Please remove it manually", e); - } - } - - private DocRouter.Range intersect(DocRouter.Range a, DocRouter.Range b) { + DocRouter.Range intersect(DocRouter.Range a, DocRouter.Range b) { if (a == null || b == null || !a.overlaps(b)) { return null; } else if (a.isSubsetOf(b)) @@ -1752,9 +574,9 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } } - private void sendShardRequest(String nodeName, ModifiableSolrParams params, - ShardHandler shardHandler, String asyncId, - Map requestMap) { + void sendShardRequest(String nodeName, ModifiableSolrParams params, + ShardHandler shardHandler, String asyncId, + Map requestMap) { sendShardRequest(nodeName, params, shardHandler, asyncId, requestMap, adminPath, zkStateReader); } @@ -1780,7 +602,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler shardHandler.submit(sreq, replica, sreq.params); } - private void addPropertyParams(ZkNodeProps message, ModifiableSolrParams params) { + void addPropertyParams(ZkNodeProps message, ModifiableSolrParams params) { // Now add the property.key=value pairs for (String key : message.keySet()) { if (key.startsWith(COLL_PROP_PREFIX)) { @@ -1789,7 +611,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } } - private void addPropertyParams(ZkNodeProps message, Map map) { + void addPropertyParams(ZkNodeProps message, Map map) { // Now add the property.key=value pairs for (String key : message.keySet()) { if (key.startsWith(COLL_PROP_PREFIX)) { @@ -1798,7 +620,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } } - private static List getLiveOrLiveAndCreateNodeSetList(final Set liveNodes, final ZkNodeProps message, final Random random) { + static List getLiveOrLiveAndCreateNodeSetList(final Set liveNodes, final ZkNodeProps message, final Random random) { // TODO: add smarter options that look at the current number of cores per // node? // for now we just go random (except when createNodeSet and createNodeSet.shuffle=false are passed in) @@ -1821,9 +643,10 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler return nodeList; } - - - private void modifyCollection(ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException { + + + private void modifyCollection(ClusterState clusterState, ZkNodeProps message, NamedList results) + throws KeeperException, InterruptedException { final String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP); //the rest of the processing is based on writing cluster state properties @@ -1835,223 +658,25 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler boolean isLegacyCloud = Overseer.isLegacy(zkStateReader); createConfNode(configName, collectionName, isLegacyCloud); - reloadCollection(new ZkNodeProps(NAME, collectionName), results); + reloadCollection(null, new ZkNodeProps(NAME, collectionName), results); } overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message)); } - private void createCollection(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException { - final String collectionName = message.getStr(NAME); - log.info("Create collection {}", collectionName); - if (clusterState.hasCollection(collectionName)) { - throw new SolrException(ErrorCode.BAD_REQUEST, "collection already exists: " + collectionName); - } - - String configName = getConfigName(collectionName, message); - if (configName == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, "No config set found to associate with the collection."); - } - - validateConfigOrThrowSolrException(configName); - - - try { - // look at the replication factor and see if it matches reality - // if it does not, find best nodes to create more cores - - int repFactor = message.getInt(REPLICATION_FACTOR, 1); - - ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); - final String async = message.getStr(ASYNC); - - Integer numSlices = message.getInt(NUM_SLICES, null); - String router = message.getStr("router.name", DocRouter.DEFAULT_NAME); - List shardNames = new ArrayList<>(); - if(ImplicitDocRouter.NAME.equals(router)){ - ClusterStateMutator.getShardNames(shardNames, message.getStr("shards", null)); - numSlices = shardNames.size(); - } else { - if (numSlices == null ) { - throw new SolrException(ErrorCode.BAD_REQUEST, NUM_SLICES + " is a required param (when using CompositeId router)."); - } - ClusterStateMutator.getShardNames(numSlices, shardNames); - } - - int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1); - - if (repFactor <= 0) { - throw new SolrException(ErrorCode.BAD_REQUEST, REPLICATION_FACTOR + " must be greater than 0"); - } - - if (numSlices <= 0) { - throw new SolrException(ErrorCode.BAD_REQUEST, NUM_SLICES + " must be > 0"); - } - - // we need to look at every node and see how many cores it serves - // add our new cores to existing nodes serving the least number of cores - // but (for now) require that each core goes on a distinct node. - - final List nodeList = getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, RANDOM); - Map positionVsNodes; - if (nodeList.isEmpty()) { - log.warn("It is unusual to create a collection ("+collectionName+") without cores."); - - positionVsNodes = new HashMap<>(); - } else { - if (repFactor > nodeList.size()) { - log.warn("Specified " - + REPLICATION_FACTOR - + " of " - + repFactor - + " on collection " - + collectionName - + " is higher than or equal to the number of Solr instances currently live or live and part of your " + CREATE_NODE_SET + "(" - + nodeList.size() - + "). It's unusual to run two replica of the same slice on the same Solr-instance."); - } - - int maxShardsAllowedToCreate = maxShardsPerNode * nodeList.size(); - int requestedShardsToCreate = numSlices * repFactor; - if (maxShardsAllowedToCreate < requestedShardsToCreate) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Cannot create collection " + collectionName + ". Value of " - + MAX_SHARDS_PER_NODE + " is " + maxShardsPerNode - + ", and the number of nodes currently live or live and part of your "+CREATE_NODE_SET+" is " + nodeList.size() - + ". This allows a maximum of " + maxShardsAllowedToCreate - + " to be created. Value of " + NUM_SLICES + " is " + numSlices - + " and value of " + REPLICATION_FACTOR + " is " + repFactor - + ". This requires " + requestedShardsToCreate - + " shards to be created (higher than the allowed number)"); - } - - positionVsNodes = identifyNodes(clusterState, nodeList, message, shardNames, repFactor); - } - - boolean isLegacyCloud = Overseer.isLegacy(zkStateReader); - - createConfNode(configName, collectionName, isLegacyCloud); - - Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message)); - - // wait for a while until we don't see the collection - TimeOut waitUntil = new TimeOut(30, TimeUnit.SECONDS); - boolean created = false; - while (! waitUntil.hasTimedOut()) { - Thread.sleep(100); - created = zkStateReader.getClusterState().hasCollection(collectionName); - if(created) break; - } - if (!created) - throw new SolrException(ErrorCode.SERVER_ERROR, "Could not fully create collection: " + collectionName); - - if (nodeList.isEmpty()) { - log.info("Finished create command for collection: {}", collectionName); - return; - } - - // For tracking async calls. - Map requestMap = new HashMap<>(); - - - log.info(formatString("Creating SolrCores for new collection {0}, shardNames {1} , replicationFactor : {2}", - collectionName, shardNames, repFactor)); - Map coresToCreate = new LinkedHashMap<>(); - for (Map.Entry e : positionVsNodes.entrySet()) { - Position position = e.getKey(); - String nodeName = e.getValue(); - String coreName = collectionName + "_" + position.shard + "_replica" + (position.index + 1); - log.info(formatString("Creating core {0} as part of shard {1} of collection {2} on {3}" - , coreName, position.shard, collectionName, nodeName)); - - - String baseUrl = zkStateReader.getBaseUrlForNodeName(nodeName); - //in the new mode, create the replica in clusterstate prior to creating the core. - // Otherwise the core creation fails - if (!isLegacyCloud) { - ZkNodeProps props = new ZkNodeProps( - Overseer.QUEUE_OPERATION, ADDREPLICA.toString(), - ZkStateReader.COLLECTION_PROP, collectionName, - ZkStateReader.SHARD_ID_PROP, position.shard, - ZkStateReader.CORE_NAME_PROP, coreName, - ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), - ZkStateReader.BASE_URL_PROP, baseUrl); - Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(props)); - } - - // Need to create new params for each request - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString()); - - params.set(CoreAdminParams.NAME, coreName); - params.set(COLL_CONF, configName); - params.set(CoreAdminParams.COLLECTION, collectionName); - params.set(CoreAdminParams.SHARD, position.shard); - params.set(ZkStateReader.NUM_SHARDS_PROP, numSlices); - - if (async != null) { - String coreAdminAsyncId = async + Math.abs(System.nanoTime()); - params.add(ASYNC, coreAdminAsyncId); - requestMap.put(nodeName, coreAdminAsyncId); - } - addPropertyParams(message, params); - - ShardRequest sreq = new ShardRequest(); - sreq.nodeName = nodeName; - params.set("qt", adminPath); - sreq.purpose = 1; - sreq.shards = new String[]{baseUrl}; - sreq.actualShards = sreq.shards; - sreq.params = params; - - if (isLegacyCloud) { - shardHandler.submit(sreq, sreq.shards[0], sreq.params); - } else { - coresToCreate.put(coreName, sreq); - } - } - - if(!isLegacyCloud) { - // wait for all replica entries to be created - Map replicas = waitToSeeReplicasInState(collectionName, coresToCreate.keySet()); - for (Map.Entry e : coresToCreate.entrySet()) { - ShardRequest sreq = e.getValue(); - sreq.params.set(CoreAdminParams.CORE_NODE_NAME, replicas.get(e.getKey()).getName()); - shardHandler.submit(sreq, sreq.shards[0], sreq.params); - } - } - - processResponses(results, shardHandler, false, null, async, requestMap, Collections.emptySet()); - if(results.get("failure") != null && ((SimpleOrderedMap)results.get("failure")).size() > 0) { - // Let's cleanup as we hit an exception - // We shouldn't be passing 'results' here for the cleanup as the response would then contain 'success' - // element, which may be interpreted by the user as a positive ack - cleanupCollection(collectionName, new NamedList()); - log.info("Cleaned up artifacts for failed create collection for [" + collectionName + "]"); - } else { - log.debug("Finished create command on all shards for collection: " - + collectionName); - } - } catch (SolrException ex) { - throw ex; - } catch (Exception ex) { - throw new SolrException(ErrorCode.SERVER_ERROR, null, ex); - } - } - - - private void cleanupCollection(String collectionName, NamedList results) throws KeeperException, InterruptedException { + void cleanupCollection(String collectionName, NamedList results) throws Exception { log.error("Cleaning up collection [" + collectionName + "]." ); Map props = makeMap( Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, collectionName); - deleteCollection(new ZkNodeProps(props), results); + commandMap.get(DELETE).call(zkStateReader.getClusterState(), new ZkNodeProps(props), results); } - private Map identifyNodes(ClusterState clusterState, - List nodeList, - ZkNodeProps message, - List shardNames, - int repFactor) throws IOException { + Map identifyNodes(ClusterState clusterState, + List nodeList, + ZkNodeProps message, + List shardNames, + int repFactor) throws IOException { List rulesMap = (List) message.get("rule"); if (rulesMap == null) { int i = 0; @@ -2082,7 +707,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler return replicaAssigner.getNodeMappings(); } - private Map waitToSeeReplicasInState(String collectionName, Collection coreNames) throws InterruptedException { + Map waitToSeeReplicasInState(String collectionName, Collection coreNames) throws InterruptedException { Map result = new HashMap<>(); TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); while (true) { @@ -2110,353 +735,18 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } } - private void addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results) + ZkNodeProps addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete) throws KeeperException, InterruptedException { - String collection = message.getStr(COLLECTION_PROP); - String node = message.getStr(CoreAdminParams.NODE); - String shard = message.getStr(SHARD_ID_PROP); - String coreName = message.getStr(CoreAdminParams.NAME); - if (StringUtils.isBlank(coreName)) { - coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME); - } - - final String asyncId = message.getStr(ASYNC); - - DocCollection coll = clusterState.getCollection(collection); - if (coll == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Collection: " + collection + " does not exist"); - } - if (coll.getSlice(shard) == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Collection: " + collection + " shard: " + shard + " does not exist"); - } - ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); - boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false); - // Kind of unnecessary, but it does put the logic of whether to override maxShardsPerNode in one place. - if (!skipCreateReplicaInClusterState) { - node = getNodesForNewReplicas(clusterState, collection, shard, 1, node, - overseer.getZkController().getCoreContainer()).get(0).nodeName; - } - log.info("Node not provided, Identified {} for creating new replica", node); - - if (!clusterState.liveNodesContain(node)) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Node: " + node + " is not live"); - } - if (coreName == null) { - coreName = Assign.buildCoreName(coll, shard); - } else if (!skipCreateReplicaInClusterState) { - //Validate that the core name is unique in that collection - for (Slice slice : coll.getSlices()) { - for (Replica replica : slice.getReplicas()) { - String replicaCoreName = replica.getStr(CORE_NAME_PROP); - if (coreName.equals(replicaCoreName)) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Another replica with the same core name already exists" + - " for this collection"); - } - } - } - } - ModifiableSolrParams params = new ModifiableSolrParams(); - - if (!Overseer.isLegacy(zkStateReader)) { - if (!skipCreateReplicaInClusterState) { - ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(), ZkStateReader.COLLECTION_PROP, - collection, ZkStateReader.SHARD_ID_PROP, shard, ZkStateReader.CORE_NAME_PROP, coreName, - ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), ZkStateReader.BASE_URL_PROP, - zkStateReader.getBaseUrlForNodeName(node), ZkStateReader.NODE_NAME_PROP, node); - Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(props)); - } - params.set(CoreAdminParams.CORE_NODE_NAME, - waitToSeeReplicasInState(collection, Collections.singletonList(coreName)).get(coreName).getName()); - } - - String configName = zkStateReader.readConfigName(collection); - String routeKey = message.getStr(ShardParams._ROUTE_); - String dataDir = message.getStr(CoreAdminParams.DATA_DIR); - String instanceDir = message.getStr(CoreAdminParams.INSTANCE_DIR); - - params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString()); - params.set(CoreAdminParams.NAME, coreName); - params.set(COLL_CONF, configName); - params.set(CoreAdminParams.COLLECTION, collection); - if (shard != null) { - params.set(CoreAdminParams.SHARD, shard); - } else if (routeKey != null) { - Collection slices = coll.getRouter().getSearchSlicesSingle(routeKey, null, coll); - if (slices.isEmpty()) { - throw new SolrException(ErrorCode.BAD_REQUEST, "No active shard serving _route_=" + routeKey + " found"); - } else { - params.set(CoreAdminParams.SHARD, slices.iterator().next().getName()); - } - } else { - throw new SolrException(ErrorCode.BAD_REQUEST, "Specify either 'shard' or _route_ param"); - } - if (dataDir != null) { - params.set(CoreAdminParams.DATA_DIR, dataDir); - } - if (instanceDir != null) { - params.set(CoreAdminParams.INSTANCE_DIR, instanceDir); - } - addPropertyParams(message, params); - - // For tracking async calls. - Map requestMap = new HashMap<>(); - sendShardRequest(node, params, shardHandler, asyncId, requestMap); - - processResponses(results, shardHandler, true, "ADDREPLICA failed to create replica", asyncId, requestMap); - - waitForCoreNodeName(collection, node, coreName); + return ((AddReplicaCmd) commandMap.get(ADDREPLICA)).addReplica(clusterState, message, results, onComplete); } - private void processBackupAction(ZkNodeProps message, NamedList results) throws IOException, KeeperException, InterruptedException { - String collectionName = message.getStr(COLLECTION_PROP); - String backupName = message.getStr(NAME); - ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); - String asyncId = message.getStr(ASYNC); - String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); - String location = message.getStr(CoreAdminParams.BACKUP_LOCATION); - - Map requestMap = new HashMap<>(); - Instant startTime = Instant.now(); - - CoreContainer cc = this.overseer.getZkController().getCoreContainer(); - BackupRepository repository = cc.newBackupRepository(Optional.ofNullable(repo)); - BackupManager backupMgr = new BackupManager(repository, zkStateReader, collectionName); - - // Backup location - URI backupPath = repository.createURI(location, backupName); - - //Validating if the directory already exists. - if (repository.exists(backupPath)) { - throw new SolrException(ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupPath); - } - - // Create a directory to store backup details. - repository.createDirectory(backupPath); - - log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName, - backupPath); - - for (Slice slice : zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) { - Replica replica = slice.getLeader(); - - String coreName = replica.getStr(CORE_NAME_PROP); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.BACKUPCORE.toString()); - params.set(NAME, slice.getName()); - params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); - params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.getPath()); // note: index dir will be here then the "snapshot." + slice name - params.set(CORE_NAME_PROP, coreName); - - sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap); - log.debug("Sent backup request to core={} for backupName={}", coreName, backupName); - } - log.debug("Sent backup requests to all shard leaders for backupName={}", backupName); - - processResponses(results, shardHandler, true, "Could not backup all replicas", asyncId, requestMap); - - log.info("Starting to backup ZK data for backupName={}", backupName); - - //Download the configs - String configName = zkStateReader.readConfigName(collectionName); - backupMgr.downloadConfigDir(location, backupName, configName); - - //Save the collection's state. Can be part of the monolithic clusterstate.json or a individual state.json - //Since we don't want to distinguish we extract the state and back it up as a separate json - DocCollection collectionState = zkStateReader.getClusterState().getCollection(collectionName); - backupMgr.writeCollectionState(location, backupName, collectionName, collectionState); - - Properties properties = new Properties(); - - properties.put(BackupManager.BACKUP_NAME_PROP, backupName); - properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName); - properties.put(COLL_CONF, configName); - properties.put(BackupManager.START_TIME_PROP, startTime.toString()); - //TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same. - //if they are not the same then we can throw an error or have an 'overwriteConfig' flag - //TODO save numDocs for the shardLeader. We can use it to sanity check the restore. - - backupMgr.writeBackupProperties(location, backupName, properties); - - log.info("Completed backing up ZK data for backupName={}", backupName); - } - - private void processRestoreAction(ZkNodeProps message, NamedList results) throws IOException, KeeperException, InterruptedException { - // TODO maybe we can inherit createCollection's options/code - String restoreCollectionName = message.getStr(COLLECTION_PROP); - String backupName = message.getStr(NAME); // of backup - ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); - String asyncId = message.getStr(ASYNC); - String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); - String location = message.getStr(CoreAdminParams.BACKUP_LOCATION); - Map requestMap = new HashMap<>(); - - CoreContainer cc = this.overseer.getZkController().getCoreContainer(); - BackupRepository repository = cc.newBackupRepository(Optional.ofNullable(repo)); - - URI backupPath = repository.createURI(location, backupName); - BackupManager backupMgr = new BackupManager(repository, zkStateReader, restoreCollectionName); - - Properties properties = backupMgr.readBackupProperties(location, backupName); - String backupCollection = properties.getProperty(BackupManager.COLLECTION_NAME_PROP); - DocCollection backupCollectionState = backupMgr.readCollectionState(location, backupName, backupCollection); - - //Upload the configs - String configName = (String) properties.get(COLL_CONF); - String restoreConfigName = message.getStr(COLL_CONF, configName); - if (zkStateReader.getConfigManager().configExists(restoreConfigName)) { - log.info("Using existing config {}", restoreConfigName); - //TODO add overwrite option? - } else { - log.info("Uploading config {}", restoreConfigName); - backupMgr.uploadConfigDir(location, backupName, configName, restoreConfigName); - } - - log.info("Starting restore into collection={} with backup_name={} at location={}", restoreCollectionName, backupName, - location); - - //Create core-less collection - { - Map propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString()); - propMap.put("fromApi", "true"); // mostly true. Prevents autoCreated=true in the collection state. - - // inherit settings from input API, defaulting to the backup's setting. Ex: replicationFactor - for (String collProp : COLL_PROPS.keySet()) { - Object val = message.getProperties().getOrDefault(collProp, backupCollectionState.get(collProp)); - if (val != null) { - propMap.put(collProp, val); - } - } - - propMap.put(NAME, restoreCollectionName); - propMap.put(CREATE_NODE_SET, CREATE_NODE_SET_EMPTY); //no cores - propMap.put(COLL_CONF, restoreConfigName); - - // router.* - @SuppressWarnings("unchecked") - Map routerProps = (Map) backupCollectionState.getProperties().get(DocCollection.DOC_ROUTER); - for (Map.Entry pair : routerProps.entrySet()) { - propMap.put(DocCollection.DOC_ROUTER + "." + pair.getKey(), pair.getValue()); - } - - Set sliceNames = backupCollectionState.getActiveSlicesMap().keySet(); - if (backupCollectionState.getRouter() instanceof ImplicitDocRouter) { - propMap.put(SHARDS_PROP, StrUtils.join(sliceNames, ',')); - } else { - propMap.put(NUM_SLICES, sliceNames.size()); - // ClusterStateMutator.createCollection detects that "slices" is in fact a slice structure instead of a - // list of names, and if so uses this instead of building it. We clear the replica list. - Collection backupSlices = backupCollectionState.getActiveSlices(); - Map newSlices = new LinkedHashMap<>(backupSlices.size()); - for (Slice backupSlice : backupSlices) { - newSlices.put(backupSlice.getName(), - new Slice(backupSlice.getName(), Collections.emptyMap(), backupSlice.getProperties())); - } - propMap.put(SHARDS_PROP, newSlices); - } - - createCollection(zkStateReader.getClusterState(), new ZkNodeProps(propMap), new NamedList()); - // note: when createCollection() returns, the collection exists (no race) - } - - DocCollection restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName); - - DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient()); - - //Mark all shards in CONSTRUCTION STATE while we restore the data - { - //TODO might instead createCollection accept an initial state? Is there a race? - Map propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); - for (Slice shard : restoreCollection.getSlices()) { - propMap.put(shard.getName(), Slice.State.CONSTRUCTION.toString()); - } - propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName); - inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap))); - } - - // TODO how do we leverage the CREATE_NODE_SET / RULE / SNITCH logic in createCollection? - - ClusterState clusterState = zkStateReader.getClusterState(); - //Create one replica per shard and copy backed up data to it - for (Slice slice: restoreCollection.getSlices()) { - log.debug("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection); - HashMap propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD); - propMap.put(COLLECTION_PROP, restoreCollectionName); - propMap.put(SHARD_ID_PROP, slice.getName()); - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } - addPropertyParams(message, propMap); - - addReplica(clusterState, new ZkNodeProps(propMap), new NamedList()); - } - - //refresh the location copy of collection state - restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName); - - //Copy data from backed up index to each replica - for (Slice slice: restoreCollection.getSlices()) { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminAction.RESTORECORE.toString()); - params.set(NAME, "snapshot." + slice.getName()); - params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.getPath()); - params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); - - sliceCmd(clusterState, params, null, slice, shardHandler, asyncId, requestMap); - } - processResponses(new NamedList(), shardHandler, true, "Could not restore core", asyncId, requestMap); - - //Mark all shards in ACTIVE STATE - { - HashMap propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); - propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName); - for (Slice shard : restoreCollection.getSlices()) { - propMap.put(shard.getName(), Slice.State.ACTIVE.toString()); - } - inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap))); - } - - //refresh the location copy of collection state - restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName); - - //Add the remaining replicas for each shard - Integer numReplicas = restoreCollection.getReplicationFactor(); - if (numReplicas != null && numReplicas > 1) { - log.info("Adding replicas to restored collection={}", restoreCollection); - - for (Slice slice: restoreCollection.getSlices()) { - for(int i=1; i propMap = new HashMap<>(); - propMap.put(COLLECTION_PROP, restoreCollectionName); - propMap.put(SHARD_ID_PROP, slice.getName()); - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } - addPropertyParams(message, propMap); - - addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results); - } - } - } - - log.info("Completed restoring collection={} backupName={}", restoreCollection, backupName); - } - - private void processResponses(NamedList results, ShardHandler shardHandler, boolean abortOnError, String msgOnError, - String asyncId, Map requestMap) { + void processResponses(NamedList results, ShardHandler shardHandler, boolean abortOnError, String msgOnError, + String asyncId, Map requestMap) { processResponses(results, shardHandler, abortOnError, msgOnError, asyncId, requestMap, Collections.emptySet()); } - private void processResponses(NamedList results, ShardHandler shardHandler, boolean abortOnError, String msgOnError, + void processResponses(NamedList results, ShardHandler shardHandler, boolean abortOnError, String msgOnError, String asyncId, Map requestMap, Set okayExceptions) { //Processes all shard responses ShardResponse srsp; @@ -2482,29 +772,8 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } } - private String getConfigName(String coll, ZkNodeProps message) throws KeeperException, InterruptedException { - String configName = message.getStr(COLL_CONF); - - if (configName == null) { - // if there is only one conf, use that - List configNames = null; - try { - configNames = zkStateReader.getZkClient().getChildren(ZkConfigManager.CONFIGS_ZKNODE, null, true); - if (configNames != null && configNames.size() == 1) { - configName = configNames.get(0); - // no config set named, but there is only 1 - use it - log.info("Only one config set found in zk - using it:" + configName); - } else if (configNames.contains(coll)) { - configName = coll; - } - } catch (KeeperException.NoNodeException e) { - } - } - return configName; - } - - private void validateConfigOrThrowSolrException(String configName) throws KeeperException, InterruptedException { + void validateConfigOrThrowSolrException(String configName) throws KeeperException, InterruptedException { boolean isValid = zkStateReader.getZkClient().exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true); if(!isValid) { throw new SolrException(ErrorCode.BAD_REQUEST, "Can not find the specified config set: " + configName); @@ -2515,7 +784,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler * This doesn't validate the config (path) itself and is just responsible for creating the confNode. * That check should be done before the config node is created. */ - private void createConfNode(String configName, String coll, boolean isLegacyCloud) throws KeeperException, InterruptedException { + void createConfNode(String configName, String coll, boolean isLegacyCloud) throws KeeperException, InterruptedException { if (configName != null) { String collDir = ZkStateReader.COLLECTIONS_ZKNODE + "/" + coll; @@ -2542,8 +811,8 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } - private void collectionCmd(ZkNodeProps message, ModifiableSolrParams params, - NamedList results, Replica.State stateMatcher, String asyncId, Map requestMap, Set okayExceptions) { + void collectionCmd(ZkNodeProps message, ModifiableSolrParams params, + NamedList results, Replica.State stateMatcher, String asyncId, Map requestMap, Set okayExceptions) { log.info("Executing Collection Cmd : " + params); String collectionName = message.getStr(NAME); ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); @@ -2559,8 +828,8 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler } - private void sliceCmd(ClusterState clusterState, ModifiableSolrParams params, Replica.State stateMatcher, - Slice slice, ShardHandler shardHandler, String asyncId, Map requestMap) { + void sliceCmd(ClusterState clusterState, ModifiableSolrParams params, Replica.State stateMatcher, + Slice slice, ShardHandler shardHandler, String asyncId, Map requestMap) { for (Replica replica : slice.getReplicas()) { if (clusterState.liveNodesContain(replica.getStr(ZkStateReader.NODE_NAME_PROP)) @@ -2723,4 +992,16 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler ); } + @Override + public void close() throws IOException { + if (tpe != null) { + if (!tpe.isShutdown()) { + ExecutorUtil.shutdownAndAwaitTermination(tpe); + } + } + } + + interface Cmd { + void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception; + } } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerRoleCmd.java b/solr/core/src/java/org/apache/solr/cloud/OverseerRoleCmd.java new file mode 100644 index 00000000000..0f450bdc51a --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerRoleCmd.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams.CollectionAction; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Utils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE; + +public class OverseerRoleCmd implements Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final OverseerCollectionMessageHandler ocmh; + private final CollectionAction operation; + private final OverseerNodePrioritizer overseerPrioritizer; + + + + public OverseerRoleCmd(OverseerCollectionMessageHandler ocmh, CollectionAction operation, OverseerNodePrioritizer prioritizer) { + this.ocmh = ocmh; + this.operation = operation; + this.overseerPrioritizer = prioritizer; + } + + @Override + @SuppressWarnings("unchecked") + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + ZkStateReader zkStateReader = ocmh.zkStateReader; + SolrZkClient zkClient = zkStateReader.getZkClient(); + Map roles = null; + String node = message.getStr("node"); + + String roleName = message.getStr("role"); + boolean nodeExists = false; + if (nodeExists = zkClient.exists(ZkStateReader.ROLES, true)) { + roles = (Map) Utils.fromJSON(zkClient.getData(ZkStateReader.ROLES, null, new Stat(), true)); + } else { + roles = new LinkedHashMap(1); + } + + List nodeList = (List) roles.get(roleName); + if (nodeList == null) roles.put(roleName, nodeList = new ArrayList()); + if (ADDROLE == operation) { + log.info("Overseer role added to {}", node); + if (!nodeList.contains(node)) nodeList.add(node); + } else if (REMOVEROLE == operation) { + log.info("Overseer role removed from {}", node); + nodeList.remove(node); + } + + if (nodeExists) { + zkClient.setData(ZkStateReader.ROLES, Utils.toJSON(roles), true); + } else { + zkClient.create(ZkStateReader.ROLES, Utils.toJSON(roles), CreateMode.PERSISTENT, true); + } + //if there are too many nodes this command may time out. And most likely dedicated + // overseers are created when there are too many nodes . So , do this operation in a separate thread + new Thread(() -> { + try { + overseerPrioritizer.prioritizeOverseerNodes(ocmh.myId); + } catch (Exception e) { + log.error("Error in prioritizing Overseer", e); + } + + }).start(); + + } + +} diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerStatusCmd.java b/solr/core/src/java/org/apache/solr/cloud/OverseerStatusCmd.java new file mode 100644 index 00000000000..22e2270ec36 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerStatusCmd.java @@ -0,0 +1,122 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.util.stats.Snapshot; +import org.apache.solr.util.stats.Timer; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OverseerStatusCmd implements Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final OverseerCollectionMessageHandler ocmh; + + public OverseerStatusCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + @SuppressWarnings("unchecked") + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + ZkStateReader zkStateReader = ocmh.zkStateReader; + String leaderNode = OverseerTaskProcessor.getLeaderNode(zkStateReader.getZkClient()); + results.add("leader", leaderNode); + Stat stat = new Stat(); + zkStateReader.getZkClient().getData("/overseer/queue",null, stat, true); + results.add("overseer_queue_size", stat.getNumChildren()); + stat = new Stat(); + zkStateReader.getZkClient().getData("/overseer/queue-work",null, stat, true); + results.add("overseer_work_queue_size", stat.getNumChildren()); + stat = new Stat(); + zkStateReader.getZkClient().getData("/overseer/collection-queue-work",null, stat, true); + results.add("overseer_collection_queue_size", stat.getNumChildren()); + + NamedList overseerStats = new NamedList(); + NamedList collectionStats = new NamedList(); + NamedList stateUpdateQueueStats = new NamedList(); + NamedList workQueueStats = new NamedList(); + NamedList collectionQueueStats = new NamedList(); + Overseer.Stats stats = ocmh.stats; + for (Map.Entry entry : stats.getStats().entrySet()) { + String key = entry.getKey(); + NamedList lst = new SimpleOrderedMap<>(); + if (key.startsWith("collection_")) { + collectionStats.add(key.substring(11), lst); + int successes = stats.getSuccessCount(entry.getKey()); + int errors = stats.getErrorCount(entry.getKey()); + lst.add("requests", successes); + lst.add("errors", errors); + List failureDetails = stats.getFailureDetails(key); + if (failureDetails != null) { + List> failures = new ArrayList<>(); + for (Overseer.FailedOp failedOp : failureDetails) { + SimpleOrderedMap fail = new SimpleOrderedMap<>(); + fail.add("request", failedOp.req.getProperties()); + fail.add("response", failedOp.resp.getResponse()); + failures.add(fail); + } + lst.add("recent_failures", failures); + } + } else if (key.startsWith("/overseer/queue_")) { + stateUpdateQueueStats.add(key.substring(16), lst); + } else if (key.startsWith("/overseer/queue-work_")) { + workQueueStats.add(key.substring(21), lst); + } else if (key.startsWith("/overseer/collection-queue-work_")) { + collectionQueueStats.add(key.substring(32), lst); + } else { + // overseer stats + overseerStats.add(key, lst); + int successes = stats.getSuccessCount(entry.getKey()); + int errors = stats.getErrorCount(entry.getKey()); + lst.add("requests", successes); + lst.add("errors", errors); + } + Timer timer = entry.getValue().requestTime; + Snapshot snapshot = timer.getSnapshot(); + lst.add("totalTime", timer.getSum()); + lst.add("avgRequestsPerMinute", timer.getMeanRate()); + lst.add("5minRateRequestsPerMinute", timer.getFiveMinuteRate()); + lst.add("15minRateRequestsPerMinute", timer.getFifteenMinuteRate()); + lst.add("avgTimePerRequest", timer.getMean()); + lst.add("medianRequestTime", snapshot.getMedian()); + lst.add("75thPctlRequestTime", snapshot.get75thPercentile()); + lst.add("95thPctlRequestTime", snapshot.get95thPercentile()); + lst.add("99thPctlRequestTime", snapshot.get99thPercentile()); + lst.add("999thPctlRequestTime", snapshot.get999thPercentile()); + } + results.add("overseer_operations", overseerStats); + results.add("collection_operations", collectionStats); + results.add("overseer_queue", stateUpdateQueueStats); + results.add("overseer_internal_queue", workQueueStats); + results.add("collection_queue", collectionQueueStats); + + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java index e3bc1f3af18..074cf163c4a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java @@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import com.google.common.collect.ImmutableSet; +import org.apache.commons.io.IOUtils; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; import org.apache.solr.cloud.Overseer.LeaderStatus; @@ -115,7 +116,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable { private final Object waitLock = new Object(); - private OverseerMessageHandlerSelector selector; + protected OverseerMessageHandlerSelector selector; private OverseerNodePrioritizer prioritizer; @@ -328,6 +329,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable { ExecutorUtil.shutdownAndAwaitTermination(tpe); } } + IOUtils.closeQuietly(selector); } public static List getSortedOverseerNodeNames(SolrZkClient zk) throws KeeperException, InterruptedException { @@ -588,7 +590,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable { * messages only) , or a different handler could be selected based on the * contents of the message. */ - public interface OverseerMessageHandlerSelector { + public interface OverseerMessageHandlerSelector extends Closeable { OverseerMessageHandler selectOverseerMessageHandler(ZkNodeProps message); } diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java new file mode 100644 index 00000000000..aad9cc721fb --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.util.NamedList; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; + +public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final OverseerCollectionMessageHandler ocmh; + + public ReplaceNodeCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + ZkStateReader zkStateReader = ocmh.zkStateReader; + ocmh.checkRequired(message, "source", "target"); + String source = message.getStr("source"); + String target = message.getStr("target"); + boolean parallel = message.getBool("parallel", false); + ClusterState clusterState = zkStateReader.getClusterState(); + + if (!clusterState.liveNodesContain(source)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source Node: " + source + " is not live"); + } + if (!clusterState.liveNodesContain(target)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Target Node: " + target + " is not live"); + } + List sourceReplicas = getReplicasOfNode(source, clusterState); + + List createdReplicas = new ArrayList<>(); + + AtomicBoolean anyOneFailed = new AtomicBoolean(false); + CountDownLatch countDownLatch = new CountDownLatch(sourceReplicas.size()); + + for (ZkNodeProps sourceReplica : sourceReplicas) { + NamedList nl = new NamedList(); + log.info("Going to create replica for collection={} shard={} on node={}", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target); + ZkNodeProps msg = sourceReplica.plus("parallel", String.valueOf(parallel)).plus(CoreAdminParams.NODE, target); + final ZkNodeProps addedReplica = ocmh.addReplica(clusterState, + msg, nl, () -> { + countDownLatch.countDown(); + if (nl.get("failure") != null) { + String errorString = String.format(Locale.ROOT, "Failed to create replica for collection=%s shard=%s" + + " on node=%s", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target); + log.warn(errorString); + // one replica creation failed. Make the best attempt to + // delete all the replicas created so far in the target + // and exit + synchronized (results) { + results.add("failure", errorString); + anyOneFailed.set(true); + } + } else { + log.debug("Successfully created replica for collection={} shard={} on node={}", + sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target); + } + }); + + if (addedReplica != null) { + createdReplicas.add(addedReplica); + } + } + + log.debug("Waiting for replace node action to complete"); + countDownLatch.await(5, TimeUnit.MINUTES); + log.debug("Finished waiting for replace node action to complete"); + + if (anyOneFailed.get()) { + log.info("Failed to create some replicas. Cleaning up all replicas on target node"); + CountDownLatch cleanupLatch = new CountDownLatch(createdReplicas.size()); + for (ZkNodeProps createdReplica : createdReplicas) { + NamedList deleteResult = new NamedList(); + try { + ocmh.deleteReplica(zkStateReader.getClusterState(), createdReplica.plus("parallel", "true"), deleteResult, () -> { + cleanupLatch.countDown(); + if (deleteResult.get("failure") != null) { + synchronized (results) { + results.add("failure", "Could not cleanup, because of : " + deleteResult.get("failure")); + } + } + }); + } catch (KeeperException e) { + cleanupLatch.countDown(); + log.warn("Error deleting replica ", e); + } catch (Exception e) { + log.warn("Error deleting replica ", e); + cleanupLatch.countDown(); + throw e; + } + } + cleanupLatch.await(5, TimeUnit.MINUTES); + } + + + // we have reached this far means all replicas could be recreated + //now cleanup the replicas in the source node + DeleteNodeCmd.cleanupReplicas(results, state, sourceReplicas, ocmh, source); + results.add("success", "REPLACENODE action completed successfully from : " + source + " to : " + target); + } + + static List getReplicasOfNode(String source, ClusterState state) { + List sourceReplicas = new ArrayList<>(); + for (Map.Entry e : state.getCollectionsMap().entrySet()) { + for (Slice slice : e.getValue().getSlices()) { + for (Replica replica : slice.getReplicas()) { + if (source.equals(replica.getNodeName())) { + ZkNodeProps props = new ZkNodeProps( + COLLECTION_PROP, e.getKey(), + SHARD_ID_PROP, slice.getName(), + ZkStateReader.CORE_NAME_PROP, replica.getCoreName(), + ZkStateReader.REPLICA_PROP, replica.getName(), + CoreAdminParams.NODE, source); + sourceReplicas.add(props + ); + } + } + } + } + return sourceReplicas; + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java new file mode 100644 index 00000000000..af2215c351a --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + + +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; + +import org.apache.solr.cloud.overseer.OverseerAction; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.ImplicitDocRouter; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.backup.BackupManager; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.handler.component.ShardHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_PROPS; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET_EMPTY; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARDS_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CommonParams.NAME; + + +public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final OverseerCollectionMessageHandler ocmh; + + public RestoreCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + // TODO maybe we can inherit createCollection's options/code + String restoreCollectionName = message.getStr(COLLECTION_PROP); + String backupName = message.getStr(NAME); // of backup + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + String asyncId = message.getStr(ASYNC); + String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); + String location = message.getStr(CoreAdminParams.BACKUP_LOCATION); + Map requestMap = new HashMap<>(); + + CoreContainer cc = ocmh.overseer.getZkController().getCoreContainer(); + BackupRepository repository = cc.newBackupRepository(Optional.ofNullable(repo)); + + URI backupPath = repository.createURI(location, backupName); + ZkStateReader zkStateReader = ocmh.zkStateReader; + BackupManager backupMgr = new BackupManager(repository, zkStateReader, restoreCollectionName); + + Properties properties = backupMgr.readBackupProperties(location, backupName); + String backupCollection = properties.getProperty(BackupManager.COLLECTION_NAME_PROP); + DocCollection backupCollectionState = backupMgr.readCollectionState(location, backupName, backupCollection); + + //Upload the configs + String configName = (String) properties.get(COLL_CONF); + String restoreConfigName = message.getStr(COLL_CONF, configName); + if (zkStateReader.getConfigManager().configExists(restoreConfigName)) { + log.info("Using existing config {}", restoreConfigName); + //TODO add overwrite option? + } else { + log.info("Uploading config {}", restoreConfigName); + backupMgr.uploadConfigDir(location, backupName, configName, restoreConfigName); + } + + log.info("Starting restore into collection={} with backup_name={} at location={}", restoreCollectionName, backupName, + location); + + //Create core-less collection + { + Map propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString()); + propMap.put("fromApi", "true"); // mostly true. Prevents autoCreated=true in the collection state. + + // inherit settings from input API, defaulting to the backup's setting. Ex: replicationFactor + for (String collProp : COLL_PROPS.keySet()) { + Object val = message.getProperties().getOrDefault(collProp, backupCollectionState.get(collProp)); + if (val != null) { + propMap.put(collProp, val); + } + } + + propMap.put(NAME, restoreCollectionName); + propMap.put(CREATE_NODE_SET, CREATE_NODE_SET_EMPTY); //no cores + propMap.put(COLL_CONF, restoreConfigName); + + // router.* + @SuppressWarnings("unchecked") + Map routerProps = (Map) backupCollectionState.getProperties().get(DocCollection.DOC_ROUTER); + for (Map.Entry pair : routerProps.entrySet()) { + propMap.put(DocCollection.DOC_ROUTER + "." + pair.getKey(), pair.getValue()); + } + + Set sliceNames = backupCollectionState.getActiveSlicesMap().keySet(); + if (backupCollectionState.getRouter() instanceof ImplicitDocRouter) { + propMap.put(SHARDS_PROP, StrUtils.join(sliceNames, ',')); + } else { + propMap.put(NUM_SLICES, sliceNames.size()); + // ClusterStateMutator.createCollection detects that "slices" is in fact a slice structure instead of a + // list of names, and if so uses this instead of building it. We clear the replica list. + Collection backupSlices = backupCollectionState.getActiveSlices(); + Map newSlices = new LinkedHashMap<>(backupSlices.size()); + for (Slice backupSlice : backupSlices) { + newSlices.put(backupSlice.getName(), + new Slice(backupSlice.getName(), Collections.emptyMap(), backupSlice.getProperties())); + } + propMap.put(SHARDS_PROP, newSlices); + } + + ocmh.commandMap.get(CREATE).call(zkStateReader.getClusterState(), new ZkNodeProps(propMap), new NamedList()); + // note: when createCollection() returns, the collection exists (no race) + } + + DocCollection restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName); + + DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient()); + + //Mark all shards in CONSTRUCTION STATE while we restore the data + { + //TODO might instead createCollection accept an initial state? Is there a race? + Map propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); + for (Slice shard : restoreCollection.getSlices()) { + propMap.put(shard.getName(), Slice.State.CONSTRUCTION.toString()); + } + propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName); + inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap))); + } + + // TODO how do we leverage the CREATE_NODE_SET / RULE / SNITCH logic in createCollection? + + ClusterState clusterState = zkStateReader.getClusterState(); + //Create one replica per shard and copy backed up data to it + for (Slice slice : restoreCollection.getSlices()) { + log.debug("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection); + HashMap propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD); + propMap.put(COLLECTION_PROP, restoreCollectionName); + propMap.put(SHARD_ID_PROP, slice.getName()); + // add async param + if (asyncId != null) { + propMap.put(ASYNC, asyncId); + } + ocmh.addPropertyParams(message, propMap); + + ocmh.addReplica(clusterState, new ZkNodeProps(propMap), new NamedList(), null); + } + + //refresh the location copy of collection state + restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName); + + //Copy data from backed up index to each replica + for (Slice slice : restoreCollection.getSlices()) { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString()); + params.set(NAME, "snapshot." + slice.getName()); + params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.getPath()); + params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); + + ocmh.sliceCmd(clusterState, params, null, slice, shardHandler, asyncId, requestMap); + } + ocmh.processResponses(new NamedList(), shardHandler, true, "Could not restore core", asyncId, requestMap); + + //Mark all shards in ACTIVE STATE + { + HashMap propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); + propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName); + for (Slice shard : restoreCollection.getSlices()) { + propMap.put(shard.getName(), Slice.State.ACTIVE.toString()); + } + inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap))); + } + + //refresh the location copy of collection state + restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName); + + //Add the remaining replicas for each shard + Integer numReplicas = restoreCollection.getReplicationFactor(); + if (numReplicas != null && numReplicas > 1) { + log.info("Adding replicas to restored collection={}", restoreCollection); + + for (Slice slice : restoreCollection.getSlices()) { + for (int i = 1; i < numReplicas; i++) { + log.debug("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection); + HashMap propMap = new HashMap<>(); + propMap.put(COLLECTION_PROP, restoreCollectionName); + propMap.put(SHARD_ID_PROP, slice.getName()); + // add async param + if (asyncId != null) { + propMap.put(ASYNC, asyncId); + } + ocmh.addPropertyParams(message, propMap); + + ocmh.addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results, null); + } + } + } + + log.info("Completed restoring collection={} backupName={}", restoreCollection, backupName); + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/SplitShardCmd.java new file mode 100644 index 00000000000..d7bbf66cf5f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/SplitShardCmd.java @@ -0,0 +1,458 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; +import org.apache.solr.cloud.overseer.OverseerAction; +import org.apache.solr.cloud.rule.ReplicaAssigner; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.CompositeIdRouter; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.DocRouter; +import org.apache.solr.common.cloud.PlainIdRouter; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.component.ShardHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_PROP_PREFIX; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SKIP_CREATE_REPLICA_IN_CLUSTER_STATE; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; + + +public class SplitShardCmd implements Cmd { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final OverseerCollectionMessageHandler ocmh; + + public SplitShardCmd(OverseerCollectionMessageHandler ocmh) { + this.ocmh = ocmh; + } + + @Override + public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { + split(state, message, results); + } + + public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception { + String collectionName = message.getStr("collection"); + String slice = message.getStr(ZkStateReader.SHARD_ID_PROP); + + log.info("Split shard invoked"); + ZkStateReader zkStateReader = ocmh.zkStateReader; + + String splitKey = message.getStr("split.key"); + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + + DocCollection collection = clusterState.getCollection(collectionName); + DocRouter router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT; + + Slice parentSlice; + + if (slice == null) { + if (router instanceof CompositeIdRouter) { + Collection searchSlices = router.getSearchSlicesSingle(splitKey, new ModifiableSolrParams(), collection); + if (searchSlices.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unable to find an active shard for split.key: " + splitKey); + } + if (searchSlices.size() > 1) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Splitting a split.key: " + splitKey + " which spans multiple shards is not supported"); + } + parentSlice = searchSlices.iterator().next(); + slice = parentSlice.getName(); + log.info("Split by route.key: {}, parent shard is: {} ", splitKey, slice); + } else { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Split by route key can only be used with CompositeIdRouter or subclass. Found router: " + + router.getClass().getName()); + } + } else { + parentSlice = collection.getSlice(slice); + } + + if (parentSlice == null) { + // no chance of the collection being null because ClusterState#getCollection(String) would have thrown + // an exception already + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No shard with the specified name exists: " + slice); + } + + // find the leader for the shard + Replica parentShardLeader = null; + try { + parentShardLeader = zkStateReader.getLeaderRetry(collectionName, slice, 10000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + DocRouter.Range range = parentSlice.getRange(); + if (range == null) { + range = new PlainIdRouter().fullRange(); + } + + List subRanges = null; + String rangesStr = message.getStr(CoreAdminParams.RANGES); + if (rangesStr != null) { + String[] ranges = rangesStr.split(","); + if (ranges.length == 0 || ranges.length == 1) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split a shard"); + } else { + subRanges = new ArrayList<>(ranges.length); + for (int i = 0; i < ranges.length; i++) { + String r = ranges[i]; + try { + subRanges.add(DocRouter.DEFAULT.fromString(r)); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Exception in parsing hexadecimal hash range: " + r, e); + } + if (!subRanges.get(i).isSubsetOf(range)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Specified hash range: " + r + " is not a subset of parent shard's range: " + range.toString()); + } + } + List temp = new ArrayList<>(subRanges); // copy to preserve original order + Collections.sort(temp); + if (!range.equals(new DocRouter.Range(temp.get(0).min, temp.get(temp.size() - 1).max))) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Specified hash ranges: " + rangesStr + " do not cover the entire range of parent shard: " + range); + } + for (int i = 1; i < temp.size(); i++) { + if (temp.get(i - 1).max + 1 != temp.get(i).min) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Specified hash ranges: " + rangesStr + + " either overlap with each other or " + "do not cover the entire range of parent shard: " + range); + } + } + } + } else if (splitKey != null) { + if (router instanceof CompositeIdRouter) { + CompositeIdRouter compositeIdRouter = (CompositeIdRouter) router; + subRanges = compositeIdRouter.partitionRangeByKey(splitKey, range); + if (subRanges.size() == 1) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The split.key: " + splitKey + + " has a hash range that is exactly equal to hash range of shard: " + slice); + } + for (DocRouter.Range subRange : subRanges) { + if (subRange.min == subRange.max) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The split.key: " + splitKey + " must be a compositeId"); + } + } + log.info("Partitioning parent shard " + slice + " range: " + parentSlice.getRange() + " yields: " + subRanges); + rangesStr = ""; + for (int i = 0; i < subRanges.size(); i++) { + DocRouter.Range subRange = subRanges.get(i); + rangesStr += subRange.toString(); + if (i < subRanges.size() - 1) rangesStr += ','; + } + } + } else { + // todo: fixed to two partitions? + subRanges = router.partitionRange(2, range); + } + + try { + List subSlices = new ArrayList<>(subRanges.size()); + List subShardNames = new ArrayList<>(subRanges.size()); + String nodeName = parentShardLeader.getNodeName(); + for (int i = 0; i < subRanges.size(); i++) { + String subSlice = slice + "_" + i; + subSlices.add(subSlice); + String subShardName = collectionName + "_" + subSlice + "_replica1"; + subShardNames.add(subShardName); + + Slice oSlice = collection.getSlice(subSlice); + if (oSlice != null) { + final Slice.State state = oSlice.getState(); + if (state == Slice.State.ACTIVE) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Sub-shard: " + subSlice + " exists in active state. Aborting split shard."); + } else if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY) { + // delete the shards + for (String sub : subSlices) { + log.info("Sub-shard: {} already exists therefore requesting its deletion", sub); + Map propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, "deleteshard"); + propMap.put(COLLECTION_PROP, collectionName); + propMap.put(SHARD_ID_PROP, sub); + ZkNodeProps m = new ZkNodeProps(propMap); + try { + ocmh.commandMap.get(DELETESHARD).call(clusterState, m, new NamedList()); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to delete already existing sub shard: " + sub, + e); + } + } + } + } + } + + final String asyncId = message.getStr(ASYNC); + Map requestMap = new HashMap<>(); + + for (int i = 0; i < subRanges.size(); i++) { + String subSlice = subSlices.get(i); + String subShardName = subShardNames.get(i); + DocRouter.Range subRange = subRanges.get(i); + + log.info("Creating slice " + subSlice + " of collection " + collectionName + " on " + nodeName); + + Map propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD.toLower()); + propMap.put(ZkStateReader.SHARD_ID_PROP, subSlice); + propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); + propMap.put(ZkStateReader.SHARD_RANGE_PROP, subRange.toString()); + propMap.put(ZkStateReader.SHARD_STATE_PROP, Slice.State.CONSTRUCTION.toString()); + propMap.put(ZkStateReader.SHARD_PARENT_PROP, parentSlice.getName()); + DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient()); + inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap))); + + // wait until we are able to see the new shard in cluster state + ocmh.waitForNewShard(collectionName, subSlice); + + // refresh cluster state + clusterState = zkStateReader.getClusterState(); + + log.info("Adding replica " + subShardName + " as part of slice " + subSlice + " of collection " + collectionName + + " on " + nodeName); + propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower()); + propMap.put(COLLECTION_PROP, collectionName); + propMap.put(SHARD_ID_PROP, subSlice); + propMap.put("node", nodeName); + propMap.put(CoreAdminParams.NAME, subShardName); + // copy over property params: + for (String key : message.keySet()) { + if (key.startsWith(COLL_PROP_PREFIX)) { + propMap.put(key, message.getStr(key)); + } + } + // add async param + if (asyncId != null) { + propMap.put(ASYNC, asyncId); + } + ocmh.addReplica(clusterState, new ZkNodeProps(propMap), results, null); + } + + ocmh.processResponses(results, shardHandler, true, "SPLITSHARD failed to create subshard leaders", asyncId, requestMap); + + for (String subShardName : subShardNames) { + // wait for parent leader to acknowledge the sub-shard core + log.info("Asking parent leader to wait for: " + subShardName + " to be alive on: " + nodeName); + String coreNodeName = ocmh.waitForCoreNodeName(collectionName, nodeName, subShardName); + CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState(); + cmd.setCoreName(subShardName); + cmd.setNodeName(nodeName); + cmd.setCoreNodeName(coreNodeName); + cmd.setState(Replica.State.ACTIVE); + cmd.setCheckLive(true); + cmd.setOnlyIfLeader(true); + + ModifiableSolrParams p = new ModifiableSolrParams(cmd.getParams()); + ocmh.sendShardRequest(nodeName, p, shardHandler, asyncId, requestMap); + } + + ocmh.processResponses(results, shardHandler, true, "SPLITSHARD timed out waiting for subshard leaders to come up", + asyncId, requestMap); + + log.info("Successfully created all sub-shards for collection " + collectionName + " parent shard: " + slice + + " on: " + parentShardLeader); + + log.info("Splitting shard " + parentShardLeader.getName() + " as part of slice " + slice + " of collection " + + collectionName + " on " + parentShardLeader); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.SPLIT.toString()); + params.set(CoreAdminParams.CORE, parentShardLeader.getStr("core")); + for (int i = 0; i < subShardNames.size(); i++) { + String subShardName = subShardNames.get(i); + params.add(CoreAdminParams.TARGET_CORE, subShardName); + } + params.set(CoreAdminParams.RANGES, rangesStr); + + ocmh.sendShardRequest(parentShardLeader.getNodeName(), params, shardHandler, asyncId, requestMap); + + ocmh.processResponses(results, shardHandler, true, "SPLITSHARD failed to invoke SPLIT core admin command", asyncId, + requestMap); + + log.info("Index on shard: " + nodeName + " split into two successfully"); + + // apply buffered updates on sub-shards + for (int i = 0; i < subShardNames.size(); i++) { + String subShardName = subShardNames.get(i); + + log.info("Applying buffered updates on : " + subShardName); + + params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.REQUESTAPPLYUPDATES.toString()); + params.set(CoreAdminParams.NAME, subShardName); + + ocmh.sendShardRequest(nodeName, params, shardHandler, asyncId, requestMap); + } + + ocmh.processResponses(results, shardHandler, true, "SPLITSHARD failed while asking sub shard leaders" + + " to apply buffered updates", asyncId, requestMap); + + log.info("Successfully applied buffered updates on : " + subShardNames); + + // Replica creation for the new Slices + + // look at the replication factor and see if it matches reality + // if it does not, find best nodes to create more cores + + // TODO: Have replication factor decided in some other way instead of numShards for the parent + + int repFactor = parentSlice.getReplicas().size(); + + // we need to look at every node and see how many cores it serves + // add our new cores to existing nodes serving the least number of cores + // but (for now) require that each core goes on a distinct node. + + // TODO: add smarter options that look at the current number of cores per + // node? + // for now we just go random + Set nodes = clusterState.getLiveNodes(); + List nodeList = new ArrayList<>(nodes.size()); + nodeList.addAll(nodes); + + // TODO: Have maxShardsPerNode param for this operation? + + // Remove the node that hosts the parent shard for replica creation. + nodeList.remove(nodeName); + + // TODO: change this to handle sharding a slice into > 2 sub-shards. + + + Map nodeMap = ocmh.identifyNodes(clusterState, + new ArrayList<>(clusterState.getLiveNodes()), + new ZkNodeProps(collection.getProperties()), + subSlices, repFactor - 1); + + List> replicas = new ArrayList<>((repFactor - 1) * 2); + + for (Map.Entry entry : nodeMap.entrySet()) { + String sliceName = entry.getKey().shard; + String subShardNodeName = entry.getValue(); + String shardName = collectionName + "_" + sliceName + "_replica" + (entry.getKey().index); + + log.info("Creating replica shard " + shardName + " as part of slice " + sliceName + " of collection " + + collectionName + " on " + subShardNodeName); + + ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(), + ZkStateReader.COLLECTION_PROP, collectionName, + ZkStateReader.SHARD_ID_PROP, sliceName, + ZkStateReader.CORE_NAME_PROP, shardName, + ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), + ZkStateReader.BASE_URL_PROP, zkStateReader.getBaseUrlForNodeName(subShardNodeName), + ZkStateReader.NODE_NAME_PROP, subShardNodeName); + Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(props)); + + HashMap propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower()); + propMap.put(COLLECTION_PROP, collectionName); + propMap.put(SHARD_ID_PROP, sliceName); + propMap.put("node", subShardNodeName); + propMap.put(CoreAdminParams.NAME, shardName); + // copy over property params: + for (String key : message.keySet()) { + if (key.startsWith(COLL_PROP_PREFIX)) { + propMap.put(key, message.getStr(key)); + } + } + // add async param + if (asyncId != null) { + propMap.put(ASYNC, asyncId); + } + // special flag param to instruct addReplica not to create the replica in cluster state again + propMap.put(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, "true"); + + replicas.add(propMap); + } + + // we must set the slice state into recovery before actually creating the replica cores + // this ensures that the logic inside Overseer to update sub-shard state to 'active' + // always gets a chance to execute. See SOLR-7673 + + if (repFactor == 1) { + // switch sub shard states to 'active' + log.info("Replication factor is 1 so switching shard states"); + DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient()); + Map propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); + propMap.put(slice, Slice.State.INACTIVE.toString()); + for (String subSlice : subSlices) { + propMap.put(subSlice, Slice.State.ACTIVE.toString()); + } + propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); + ZkNodeProps m = new ZkNodeProps(propMap); + inQueue.offer(Utils.toJSON(m)); + } else { + log.info("Requesting shard state be set to 'recovery'"); + DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient()); + Map propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); + for (String subSlice : subSlices) { + propMap.put(subSlice, Slice.State.RECOVERY.toString()); + } + propMap.put(ZkStateReader.COLLECTION_PROP, collectionName); + ZkNodeProps m = new ZkNodeProps(propMap); + inQueue.offer(Utils.toJSON(m)); + } + + // now actually create replica cores on sub shard nodes + for (Map replica : replicas) { + ocmh.addReplica(clusterState, new ZkNodeProps(replica), results, null); + } + + ocmh.processResponses(results, shardHandler, true, "SPLITSHARD failed to create subshard replicas", asyncId, requestMap); + + log.info("Successfully created all replica shards for all sub-slices " + subSlices); + + ocmh.commit(results, slice, parentShardLeader); + + return true; + } catch (SolrException e) { + throw e; + } catch (Exception e) { + log.error("Error executing split operation for collection: " + collectionName + " parent shard: " + slice, e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/core/CoreSorter.java b/solr/core/src/java/org/apache/solr/core/CoreSorter.java index 4c37b8f2df3..cccd84bfe14 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreSorter.java +++ b/solr/core/src/java/org/apache/solr/core/CoreSorter.java @@ -15,12 +15,6 @@ * limitations under the License. */ -/** - * This is a utility class that sorts cores in such a way as to minimize other cores - * waiting for replicas in the current node. This helps in avoiding leaderVote timeouts - * happening in other nodes of the cluster - * - */ package org.apache.solr.core; import java.util.Collection; @@ -37,6 +31,12 @@ import org.apache.solr.common.cloud.Slice; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; + +/** + * This is a utility class that sorts cores in such a way as to minimize other cores + * waiting for replicas in the current node. This helps in avoiding leaderVote timeouts + * happening in other nodes of the cluster + */ public class CoreSorter { Map shardsVsReplicaCounts = new LinkedHashMap<>(); CoreContainer cc; diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index eb3aa5fc7f1..653c612fe65 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -28,7 +28,17 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -49,6 +59,7 @@ import org.apache.solr.schema.IndexSchemaFactory; import org.apache.solr.search.CacheConfig; import org.apache.solr.search.FastLRUCache; import org.apache.solr.search.QParserPlugin; +import org.apache.solr.search.SolrCache; import org.apache.solr.search.ValueSourceParser; import org.apache.solr.search.stats.StatsCache; import org.apache.solr.servlet.SolrRequestParsers; @@ -91,7 +102,7 @@ public class SolrConfig extends Config implements MapSerializable { public static final String DEFAULT_CONF_FILE = "solrconfig.xml"; private RequestParams requestParams; - public static enum PluginOpts { + public enum PluginOpts { MULTI_OK, REQUIRE_NAME, REQUIRE_NAME_IN_OVERLAY, @@ -254,7 +265,6 @@ public class SolrConfig extends Config implements MapSerializable { dataDir = get("dataDir", null); if (dataDir != null && dataDir.length() == 0) dataDir = null; - userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache"); org.apache.solr.search.SolrIndexSearcher.initRegenerators(this); @@ -276,6 +286,16 @@ public class SolrConfig extends Config implements MapSerializable { maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE); slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1); for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin); + + Map userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache"); + List caches = getPluginInfos(SolrCache.class.getName()); + if (!caches.isEmpty()) { + for (PluginInfo c : caches) { + userCacheConfigs.put(c.name, CacheConfig.getConfig(this, "cache", c.attributes, null)); + } + } + this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs); + updateHandlerInfo = loadUpdatehandlerInfo(); multipartUploadLimitKB = getInt( @@ -317,6 +337,7 @@ public class SolrConfig extends Config implements MapSerializable { .add(new SolrPluginInfo(TransformerFactory.class, "transformer", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) .add(new SolrPluginInfo(SearchComponent.class, "searchComponent", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) .add(new SolrPluginInfo(UpdateRequestProcessorFactory.class, "updateProcessor", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) + .add(new SolrPluginInfo(SolrCache.class, "cache", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) // TODO: WTF is up with queryConverter??? // it apparently *only* works as a singleton? - SOLR-4304 // and even then -- only if there is a single SpellCheckComponent @@ -457,7 +478,7 @@ public class SolrConfig extends Config implements MapSerializable { public final CacheConfig queryResultCacheConfig; public final CacheConfig documentCacheConfig; public final CacheConfig fieldValueCacheConfig; - public final CacheConfig[] userCacheConfigs; + public final Map userCacheConfigs; // SolrIndexSearcher - more... public final boolean useFilterForSortedQuery; public final int queryResultWindowSize; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 3ba5cb71222..a9703f30a1c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -777,7 +777,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission req.getParams().getAll(params, COLL_CONF, REPLICATION_FACTOR, MAX_SHARDS_PER_NODE, STATE_FORMAT, AUTO_ADD_REPLICAS); copyPropertiesWithPrefix(req.getParams(), params, COLL_PROP_PREFIX); return params; - }); + }), + + REPLACENODE_OP(REPLACENODE, (req, rsp, h) -> req.getParams().required().getAll(req.getParams().getAll(null, "parallel"), "source", "target")), + DELETENODE_OP(DELETENODE, (req, rsp, h) -> req.getParams().required().getAll(null, "node")); public final CollectionOp fun; CollectionAction action; long timeOut; diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java index 4c2a2b6a5ab..5df1b45cc93 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java @@ -29,8 +29,7 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang.StringUtils; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; @@ -637,13 +636,13 @@ public class StatsField { return null; } - final FieldType.LegacyNumericType hashableNumType = getHashableNumericType(field); + final LegacyNumericType hashableNumType = getHashableNumericType(field); // some sane defaults int log2m = 13; // roughly equivilent to "cardinality='0.33'" int regwidth = 6; // with decent hash, this is plenty for all valid long hashes - if (LegacyNumericType.FLOAT.equals(hashableNumType) || FieldType.LegacyNumericType.INT.equals(hashableNumType)) { + if (LegacyNumericType.FLOAT.equals(hashableNumType) || LegacyNumericType.INT.equals(hashableNumType)) { // for 32bit values, we can adjust our default regwidth down a bit regwidth--; @@ -707,7 +706,7 @@ public class StatsField { if (null == hasher) { // if this is a function, or a non Long field, pre-hashed is invalid // NOTE: we ignore hashableNumType - it's LONG for non numerics like Strings - if (null == field || !FieldType.LegacyNumericType.LONG.equals(field.getType().getNumericType())) { + if (null == field || !LegacyNumericType.LONG.equals(field.getType().getNumericType())) { throw new SolrException(ErrorCode.BAD_REQUEST, "hllPreHashed is only supported with Long based fields"); } } @@ -740,16 +739,16 @@ public class StatsField { } /** - * Returns the effective {@link org.apache.lucene.document.FieldType.LegacyNumericType} for the field for the purposes of hash values. + * Returns the effective {@link LegacyNumericType} for the field for the purposes of hash values. * ie: If the field has an explict LegacyNumericType that is returned; If the field has no explicit - * LegacyNumericType then {@link org.apache.lucene.document.FieldType.LegacyNumericType#LONG} is returned; If field is null, then - * {@link org.apache.lucene.document.FieldType.LegacyNumericType#FLOAT} is assumed for ValueSource. + * LegacyNumericType then {@link LegacyNumericType#LONG} is returned; If field is null, then + * {@link LegacyNumericType#FLOAT} is assumed for ValueSource. */ private static LegacyNumericType getHashableNumericType(SchemaField field) { if (null == field) { return LegacyNumericType.FLOAT; } final LegacyNumericType result = field.getType().getNumericType(); - return null == result ? FieldType.LegacyNumericType.LONG : result; + return null == result ? LegacyNumericType.LONG : result; } } diff --git a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java index fedc7fe0ca4..db26e12218d 100644 --- a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java +++ b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java @@ -25,7 +25,7 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.NumericDocValues; diff --git a/solr/core/src/java/org/apache/solr/request/NumericFacets.java b/solr/core/src/java/org/apache/solr/request/NumericFacets.java index 1034947ca9f..fd85d3fd5d7 100644 --- a/solr/core/src/java/org/apache/solr/request/NumericFacets.java +++ b/solr/core/src/java/org/apache/solr/request/NumericFacets.java @@ -33,6 +33,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.util.Bits; @@ -132,7 +133,7 @@ final class NumericFacets { mincount = Math.max(mincount, 1); final SchemaField sf = searcher.getSchema().getField(fieldName); final FieldType ft = sf.getType(); - final org.apache.lucene.document.FieldType.LegacyNumericType numericType = ft.getNumericType(); + final LegacyNumericType numericType = ft.getNumericType(); if (numericType == null) { throw new IllegalStateException(); } diff --git a/solr/core/src/java/org/apache/solr/schema/BBoxField.java b/solr/core/src/java/org/apache/solr/schema/BBoxField.java index e41d3c6e4e0..d7fda7ce0da 100644 --- a/solr/core/src/java/org/apache/solr/schema/BBoxField.java +++ b/solr/core/src/java/org/apache/solr/schema/BBoxField.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.legacy.LegacyFieldType; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.spatial.bbox.BBoxOverlapRatioValueSource; import org.apache.lucene.spatial.bbox.BBoxStrategy; @@ -141,7 +142,11 @@ public class BBoxField extends AbstractSpatialFieldType implements //and annoyingly this Field isn't going to have a docValues format because Solr uses a separate Field for that if (solrNumField.hasDocValues()) { - luceneType = new org.apache.lucene.document.FieldType(luceneType); + if (luceneType instanceof LegacyFieldType) { + luceneType = new LegacyFieldType((LegacyFieldType)luceneType); + } else { + luceneType = new org.apache.lucene.document.FieldType(luceneType); + } luceneType.setDocValuesType(DocValuesType.NUMERIC); } return new BBoxStrategy(ctx, fieldName, luceneType); diff --git a/solr/core/src/java/org/apache/solr/schema/EnumField.java b/solr/core/src/java/org/apache/solr/schema/EnumField.java index 27f3a0a1ea8..4820e77bb11 100644 --- a/solr/core/src/java/org/apache/solr/schema/EnumField.java +++ b/solr/core/src/java/org/apache/solr/schema/EnumField.java @@ -32,24 +32,25 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; +import org.apache.lucene.legacy.LegacyNumericType; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.EnumFieldSource; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DocValuesRangeQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.solr.common.EnumFieldValue; import org.apache.solr.common.SolrException; import org.apache.solr.response.TextResponseWriter; @@ -234,8 +235,8 @@ public class EnumField extends PrimitiveFieldType { * {@inheritDoc} */ @Override - public FieldType.LegacyNumericType getNumericType() { - return FieldType.LegacyNumericType.INT; + public LegacyNumericType getNumericType() { + return LegacyNumericType.INT; } /** @@ -387,7 +388,7 @@ public class EnumField extends PrimitiveFieldType { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown value for enum field: " + value.toString()); String intAsString = intValue.toString(); - final FieldType newType = new FieldType(); + final LegacyFieldType newType = new LegacyFieldType(); newType.setTokenized(field.isTokenized()); newType.setStored(field.stored()); @@ -397,7 +398,7 @@ public class EnumField extends PrimitiveFieldType { newType.setStoreTermVectorOffsets(field.storeTermOffsets()); newType.setStoreTermVectorPositions(field.storeTermPositions()); newType.setStoreTermVectorPayloads(field.storeTermPayloads()); - newType.setNumericType(FieldType.LegacyNumericType.INT); + newType.setNumericType(LegacyNumericType.INT); newType.setNumericPrecisionStep(DEFAULT_PRECISION_STEP); final org.apache.lucene.document.Field f; diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 6556ddb77f6..32a91d98316 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -38,6 +38,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.DocValuesRewriteMethod; @@ -621,7 +622,7 @@ public abstract class FieldType extends FieldProperties { /** Return the numeric type of this field, or null if this field is not a * numeric field. */ - public org.apache.lucene.document.FieldType.LegacyNumericType getNumericType() { + public LegacyNumericType getNumericType() { return null; } diff --git a/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java b/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java index 18d80a35146..f6bb782e515 100644 --- a/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.lucene.spatial.vector.PointVectorStrategy; /** @@ -78,8 +80,8 @@ public class SpatialPointVectorFieldType extends AbstractSpatialFieldType * For each number being added to this field, multiple terms are generated as per the algorithm described in the above @@ -82,7 +82,7 @@ import org.slf4j.LoggerFactory; * generated, range search will be no faster than any other number field, but sorting will still be possible. * * - * @see org.apache.lucene.search.LegacyNumericRangeQuery + * @see org.apache.lucene.legacy.LegacyNumericRangeQuery * @since solr 1.4 */ public class TrieField extends PrimitiveFieldType { @@ -349,17 +349,17 @@ public class TrieField extends PrimitiveFieldType { } @Override - public FieldType.LegacyNumericType getNumericType() { + public LegacyNumericType getNumericType() { switch (type) { case INTEGER: - return FieldType.LegacyNumericType.INT; + return LegacyNumericType.INT; case LONG: case DATE: - return FieldType.LegacyNumericType.LONG; + return LegacyNumericType.LONG; case FLOAT: - return FieldType.LegacyNumericType.FLOAT; + return LegacyNumericType.FLOAT; case DOUBLE: - return FieldType.LegacyNumericType.DOUBLE; + return LegacyNumericType.DOUBLE; default: throw new AssertionError(); } @@ -666,7 +666,7 @@ public class TrieField extends PrimitiveFieldType { return null; } - FieldType ft = new FieldType(); + LegacyFieldType ft = new LegacyFieldType(); ft.setStored(stored); ft.setTokenized(true); ft.setOmitNorms(field.omitNorms()); @@ -677,16 +677,16 @@ public class TrieField extends PrimitiveFieldType { ft.setNumericType(LegacyNumericType.INT); break; case FLOAT: - ft.setNumericType(FieldType.LegacyNumericType.FLOAT); + ft.setNumericType(LegacyNumericType.FLOAT); break; case LONG: - ft.setNumericType(FieldType.LegacyNumericType.LONG); + ft.setNumericType(LegacyNumericType.LONG); break; case DOUBLE: - ft.setNumericType(FieldType.LegacyNumericType.DOUBLE); + ft.setNumericType(LegacyNumericType.DOUBLE); break; case DATE: - ft.setNumericType(FieldType.LegacyNumericType.LONG); + ft.setNumericType(LegacyNumericType.LONG); break; default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type); diff --git a/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java b/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java index 8d1739e6782..bcc4d4fe3b1 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java @@ -23,13 +23,13 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.FloatDocValues; import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource; import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueFloat; diff --git a/solr/core/src/java/org/apache/solr/schema/TrieIntField.java b/solr/core/src/java/org/apache/solr/schema/TrieIntField.java index 07d9c74f472..6c8622b39e6 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieIntField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieIntField.java @@ -23,13 +23,13 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.IntDocValues; import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource; import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueInt; diff --git a/solr/core/src/java/org/apache/solr/schema/TrieLongField.java b/solr/core/src/java/org/apache/solr/schema/TrieLongField.java index 28345af9622..371d5bb0a14 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieLongField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieLongField.java @@ -23,13 +23,13 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.LongDocValues; import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource; import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueLong; diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java index 40e54dccb4e..ee333f8787b 100644 --- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java +++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java @@ -17,10 +17,10 @@ package org.apache.solr.search; import javax.xml.xpath.XPathConstants; - import java.lang.invoke.MethodHandles; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -75,14 +75,15 @@ public class CacheConfig implements MapSerializable{ this.regenerator = regenerator; } - public static CacheConfig[] getMultipleConfigs(SolrConfig solrConfig, String configPath) { - NodeList nodes = (NodeList)solrConfig.evaluate(configPath, XPathConstants.NODESET); - if (nodes==null || nodes.getLength()==0) return null; - CacheConfig[] configs = new CacheConfig[nodes.getLength()]; - for (int i=0; i getMultipleConfigs(SolrConfig solrConfig, String configPath) { + NodeList nodes = (NodeList) solrConfig.evaluate(configPath, XPathConstants.NODESET); + if (nodes == null || nodes.getLength() == 0) return new LinkedHashMap<>(); + Map result = new HashMap<>(nodes.getLength()); + for (int i = 0; i < nodes.getLength(); i++) { + CacheConfig config = getConfig(solrConfig, nodes.item(i).getNodeName(), DOMUtil.toMap(nodes.item(i).getAttributes()), configPath); + result.put(config.args.get(NAME), config); } - return configs; + return result; } @@ -101,9 +102,14 @@ public class CacheConfig implements MapSerializable{ public static CacheConfig getConfig(SolrConfig solrConfig, String nodeName, Map attrs, String xpath) { CacheConfig config = new CacheConfig(); config.nodeName = nodeName; + Map attrsCopy = new LinkedHashMap<>(attrs.size()); + for (Map.Entry e : attrs.entrySet()) { + attrsCopy.put(e.getKey(), String.valueOf(e.getValue())); + } + attrs = attrsCopy; config.args = attrs; - Map map = solrConfig.getOverlay().getEditableSubProperties(xpath); + Map map = xpath == null ? null : solrConfig.getOverlay().getEditableSubProperties(xpath); if(map != null){ HashMap mapCopy = new HashMap<>(config.args); for (Map.Entry e : map.entrySet()) { diff --git a/solr/core/src/java/org/apache/solr/search/QueryParsing.java b/solr/core/src/java/org/apache/solr/search/QueryParsing.java index e7a6c3632fa..fb32c6e934d 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryParsing.java +++ b/solr/core/src/java/org/apache/solr/search/QueryParsing.java @@ -17,12 +17,12 @@ package org.apache.solr.search; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FuzzyQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; diff --git a/solr/core/src/java/org/apache/solr/search/QueryWrapperFilter.java b/solr/core/src/java/org/apache/solr/search/QueryWrapperFilter.java index 6bba1de2c64..d526cf394ab 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryWrapperFilter.java +++ b/solr/core/src/java/org/apache/solr/search/QueryWrapperFilter.java @@ -34,7 +34,7 @@ import org.apache.lucene.util.Bits; * Constrains search results to only match those which also match a provided * query. * - *

    This could be used, for example, with a {@link org.apache.lucene.search.LegacyNumericRangeQuery} on a suitably + *

    This could be used, for example, with a {@link org.apache.lucene.legacy.LegacyNumericRangeQuery} on a suitably * formatted date field to implement date filtering. One could re-use a single * CachingWrapperFilter(QueryWrapperFilter) that matches, e.g., only documents modified * within the last week. This would only need to be reconstructed once per day. diff --git a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java index 3f0bb0e7ddb..2c462a0aae5 100644 --- a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java @@ -25,8 +25,6 @@ import com.carrotsearch.hppc.IntIntHashMap; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.FilterWeight; -import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; @@ -42,7 +40,6 @@ import org.apache.lucene.search.TopScoreDocCollector; import org.apache.lucene.search.Weight; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.handler.component.MergeStrategy; import org.apache.solr.handler.component.QueryElevationComponent; @@ -91,10 +88,7 @@ public class ReRankQParserPlugin extends QParserPlugin { double reRankWeight = localParams.getDouble(RERANK_WEIGHT, RERANK_WEIGHT_DEFAULT); - int start = params.getInt(CommonParams.START,CommonParams.START_DEFAULT); - int rows = params.getInt(CommonParams.ROWS,CommonParams.ROWS_DEFAULT); - int length = start+rows; - return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight, length); + return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight); } } @@ -121,7 +115,6 @@ public class ReRankQParserPlugin extends QParserPlugin { private Query mainQuery = defaultQuery; final private Query reRankQuery; final private int reRankDocs; - final private int length; final private double reRankWeight; final private Rescorer reRankQueryRescorer; private Map boostedPriority; @@ -142,11 +135,10 @@ public class ReRankQParserPlugin extends QParserPlugin { reRankDocs == rrq.reRankDocs; } - public ReRankQuery(Query reRankQuery, int reRankDocs, double reRankWeight, int length) { + public ReRankQuery(Query reRankQuery, int reRankDocs, double reRankWeight) { this.reRankQuery = reRankQuery; this.reRankDocs = reRankDocs; this.reRankWeight = reRankWeight; - this.length = length; this.reRankQueryRescorer = new ReRankQueryRescorer(reRankQuery, reRankWeight); } @@ -171,7 +163,7 @@ public class ReRankQParserPlugin extends QParserPlugin { } } - return new ReRankCollector(reRankDocs, length, reRankQueryRescorer, cmd, searcher, boostedPriority); + return new ReRankCollector(reRankDocs, len, reRankQueryRescorer, cmd, searcher, boostedPriority); } @Override @@ -188,29 +180,14 @@ public class ReRankQParserPlugin extends QParserPlugin { public Query rewrite(IndexReader reader) throws IOException { Query q = mainQuery.rewrite(reader); if (q != mainQuery) { - return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight, length).wrap(q); + return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight).wrap(q); } return super.rewrite(reader); } public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException{ - return new ReRankWeight(mainQuery, reRankQueryRescorer, searcher, needsScores, boost); - } - } - - private class ReRankWeight extends FilterWeight { - private IndexSearcher searcher; - final private Rescorer reRankQueryRescorer; - - public ReRankWeight(Query mainQuery, Rescorer reRankQueryRescorer, IndexSearcher searcher, boolean needsScores, float boost) throws IOException { - super(mainQuery, mainQuery.createWeight(searcher, needsScores, boost)); - this.searcher = searcher; - this.reRankQueryRescorer = reRankQueryRescorer; - } - - public Explanation explain(LeafReaderContext context, int doc) throws IOException { - Explanation mainExplain = in.explain(context, doc); - return reRankQueryRescorer.explain(searcher, mainExplain, context.docBase+doc); + final Weight mainWeight = mainQuery.createWeight(searcher, needsScores, boost); + return new ReRankWeight(mainQuery, reRankQueryRescorer, searcher, mainWeight); } } diff --git a/solr/core/src/java/org/apache/solr/search/ReRankWeight.java b/solr/core/src/java/org/apache/solr/search/ReRankWeight.java new file mode 100644 index 00000000000..9c11a894200 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/ReRankWeight.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import java.io.IOException; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FilterWeight; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Rescorer; +import org.apache.lucene.search.Weight; + +/** + * A {@code Weight} used by reranking queries. + */ +public class ReRankWeight extends FilterWeight { + + final private IndexSearcher searcher; + final private Rescorer reRankQueryRescorer; + + public ReRankWeight(Query mainQuery, Rescorer reRankQueryRescorer, IndexSearcher searcher, Weight mainWeight) throws IOException { + super(mainQuery, mainWeight); + this.searcher = searcher; + this.reRankQueryRescorer = reRankQueryRescorer; + } + + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + final Explanation mainExplain = in.explain(context, doc); + return reRankQueryRescorer.explain(searcher, mainExplain, context.docBase+doc); + } + +} diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index cc719f0be37..0f480c67579 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -36,62 +36,16 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import com.google.common.base.Function; +import com.google.common.base.Objects; +import com.google.common.collect.Iterables; import org.apache.lucene.document.Document; import org.apache.lucene.document.DocumentStoredFieldVisitor; import org.apache.lucene.document.LazyDocument; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.ExitableDirectoryReader; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiPostingsEnum; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.*; import org.apache.lucene.index.StoredFieldVisitor.Status; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermContext; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CollectionStatistics; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.EarlyTerminatingSortingCollector; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.MultiCollector; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SimpleCollector; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermStatistics; -import org.apache.lucene.search.TimeLimitingCollector; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopDocsCollector; -import org.apache.lucene.search.TopFieldCollector; -import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.TopScoreDocCollector; -import org.apache.lucene.search.TotalHitCountCollector; -import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -128,10 +82,6 @@ import org.apache.solr.update.SolrIndexConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; -import com.google.common.base.Objects; -import com.google.common.collect.Iterables; - /** * SolrIndexSearcher adds schema awareness and caching functionality over {@link IndexSearcher}. * @@ -337,13 +287,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance(); if (documentCache != null) clist.add(documentCache); - if (solrConfig.userCacheConfigs == null) { + if (solrConfig.userCacheConfigs.isEmpty()) { cacheMap = NO_GENERIC_CACHES; } else { - cacheMap = new HashMap<>(solrConfig.userCacheConfigs.length); - for (CacheConfig userCacheConfig : solrConfig.userCacheConfigs) { - SolrCache cache = null; - if (userCacheConfig != null) cache = userCacheConfig.newInstance(); + cacheMap = new HashMap<>(solrConfig.userCacheConfigs.size()); + for (Map.Entry e : solrConfig.userCacheConfigs.entrySet()) { + SolrCache cache = e.getValue().newInstance(); if (cache != null) { cacheMap.put(cache.name(), cache); clist.add(cache); diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java index a5ec1dbab91..92c64e74b16 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java @@ -16,39 +16,13 @@ */ package org.apache.solr.search.facet; -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiPostingsEnum; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.PriorityQueue; -import org.apache.lucene.util.StringHelper; -import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.legacy.LegacyNumericType; import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.SchemaField; -import org.apache.solr.schema.TrieField; -import org.apache.solr.search.DocSet; -import org.apache.solr.search.HashDocSet; -import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.search.SortedIntDocSet; public class FacetField extends FacetRequest { @@ -69,7 +43,7 @@ public class FacetField extends FacetRequest { Boolean perSeg; // TODO: put this somewhere more generic? - public static enum SortDirection { + public enum SortDirection { asc(-1) , desc(1); @@ -84,7 +58,7 @@ public class FacetField extends FacetRequest { } } - public static enum FacetMethod { + public enum FacetMethod { DV, // DocValues UIF, // UnInvertedField ENUM, @@ -109,7 +83,6 @@ public class FacetField extends FacetRequest { } } - @Override public FacetProcessor createFacetProcessor(FacetContext fcontext) { SchemaField sf = fcontext.searcher.getSchema().getField(field); @@ -119,18 +92,18 @@ public class FacetField extends FacetRequest { if (method == FacetMethod.ENUM && sf.indexed()) { throw new UnsupportedOperationException(); } else if (method == FacetMethod.STREAM && sf.indexed()) { - return new FacetFieldProcessorStream(fcontext, this, sf); + return new FacetFieldProcessorByEnumTermsStream(fcontext, this, sf); } - org.apache.lucene.document.FieldType.LegacyNumericType ntype = ft.getNumericType(); + LegacyNumericType ntype = ft.getNumericType(); if (!multiToken) { if (ntype != null) { // single valued numeric (docvalues or fieldcache) - return new FacetFieldProcessorNumeric(fcontext, this, sf); + return new FacetFieldProcessorByHashNumeric(fcontext, this, sf); } else { // single valued string... - return new FacetFieldProcessorDV(fcontext, this, sf); + return new FacetFieldProcessorByArrayDV(fcontext, this, sf); } } @@ -138,11 +111,11 @@ public class FacetField extends FacetRequest { if (sf.hasDocValues() || method == FacetMethod.DV) { // single and multi-valued string docValues - return new FacetFieldProcessorDV(fcontext, this, sf); + return new FacetFieldProcessorByArrayDV(fcontext, this, sf); } // Top-level multi-valued field cache (UIF) - return new FacetFieldProcessorUIF(fcontext, this, sf); + return new FacetFieldProcessorByArrayUIF(fcontext, this, sf); } @Override @@ -152,918 +125,12 @@ public class FacetField extends FacetRequest { @Override public Map getFacetDescription() { - Map descr = new HashMap(); + Map descr = new HashMap<>(); descr.put("field", field); - descr.put("limit", new Long(limit)); + descr.put("limit", limit); return descr; } } -abstract class FacetFieldProcessor extends FacetProcessor { - SchemaField sf; - SlotAcc indexOrderAcc; - int effectiveMincount; - - Map deferredAggs; // null if none - - // TODO: push any of this down to base class? - - // - // For sort="x desc", collectAcc would point to "x", and sortAcc would also point to "x". - // collectAcc would be used to accumulate all buckets, and sortAcc would be used to sort those buckets. - // - SlotAcc collectAcc; // Accumulator to collect across entire domain (in addition to the countAcc). May be null. - SlotAcc sortAcc; // Accumulator to use for sorting *only* (i.e. not used for collection). May be an alias of countAcc, collectAcc, or indexOrderAcc - SlotAcc[] otherAccs; // Accumulators that do not need to be calculated across all buckets. - - SpecialSlotAcc allBucketsAcc; // this can internally refer to otherAccs and/or collectAcc. setNextReader should be called on otherAccs directly if they exist. - - - FacetFieldProcessor(FacetContext fcontext, FacetField freq, SchemaField sf) { - super(fcontext, freq); - this.sf = sf; - this.effectiveMincount = (int)(fcontext.isShard() ? Math.min(1 , freq.mincount) : freq.mincount); - } - - @Override - public Object getResponse() { - return response; - } - - // This is used to create accs for second phase (or to create accs for all aggs) - @Override - protected void createAccs(int docCount, int slotCount) throws IOException { - if (accMap == null) { - accMap = new LinkedHashMap<>(); - } - - // allow a custom count acc to be used - if (countAcc == null) { - countAcc = new CountSlotArrAcc(fcontext, slotCount); - countAcc.key = "count"; - } - - if (accs != null) { - // reuse these accs, but reset them first - for (SlotAcc acc : accs) { - acc.reset(); - } - return; - } else { - accs = new SlotAcc[ freq.getFacetStats().size() ]; - } - - int accIdx = 0; - for (Map.Entry entry : freq.getFacetStats().entrySet()) { - SlotAcc acc = null; - if (slotCount == 1) { - acc = accMap.get(entry.getKey()); - if (acc != null) { - acc.reset(); - } - } - if (acc == null) { - acc = entry.getValue().createSlotAcc(fcontext, docCount, slotCount); - acc.key = entry.getKey(); - accMap.put(acc.key, acc); - } - accs[accIdx++] = acc; - } - } - - void createCollectAcc(int numDocs, int numSlots) throws IOException { - accMap = new LinkedHashMap<>(); - - // we always count... - // allow a subclass to set a custom counter. - if (countAcc == null) { - countAcc = new CountSlotArrAcc(fcontext, numSlots); - } - - if ("count".equals(freq.sortVariable)) { - sortAcc = countAcc; - deferredAggs = freq.getFacetStats(); - } else if ("index".equals(freq.sortVariable)) { - // allow subclass to set indexOrderAcc first - if (indexOrderAcc == null) { - // This sorting accumulator just goes by the slot number, so does not need to be collected - // and hence does not need to find it's way into the accMap or accs array. - indexOrderAcc = new SortSlotAcc(fcontext); - } - sortAcc = indexOrderAcc; - deferredAggs = freq.getFacetStats(); - } else { - AggValueSource sortAgg = freq.getFacetStats().get(freq.sortVariable); - if (sortAgg != null) { - collectAcc = sortAgg.createSlotAcc(fcontext, numDocs, numSlots); - collectAcc.key = freq.sortVariable; // TODO: improve this - } - sortAcc = collectAcc; - deferredAggs = new HashMap<>(freq.getFacetStats()); - deferredAggs.remove(freq.sortVariable); - } - - if (deferredAggs.size() == 0) { - deferredAggs = null; - } - - boolean needOtherAccs = freq.allBuckets; // TODO: use for missing too... - - if (!needOtherAccs) { - // we may need them later, but we don't want to create them now - // otherwise we won't know if we need to call setNextReader on them. - return; - } - - // create the deferred aggs up front for use by allBuckets - createOtherAccs(numDocs, 1); - } - - - void createOtherAccs(int numDocs, int numSlots) throws IOException { - if (otherAccs != null) { - // reuse existing accumulators - for (SlotAcc acc : otherAccs) { - acc.reset(); // todo - make reset take numDocs and numSlots? - } - return; - } - - int numDeferred = deferredAggs == null ? 0 : deferredAggs.size(); - if (numDeferred <= 0) return; - - otherAccs = new SlotAcc[ numDeferred ]; - - int otherAccIdx = 0; - for (Map.Entry entry : deferredAggs.entrySet()) { - AggValueSource agg = entry.getValue(); - SlotAcc acc = agg.createSlotAcc(fcontext, numDocs, numSlots); - acc.key = entry.getKey(); - accMap.put(acc.key, acc); - otherAccs[otherAccIdx++] = acc; - } - - if (numDeferred == freq.getFacetStats().size()) { - // accs and otherAccs are the same... - accs = otherAccs; - } - } - - - int collectFirstPhase(DocSet docs, int slot) throws IOException { - int num = -1; - if (collectAcc != null) { - num = collectAcc.collect(docs, slot); - } - if (allBucketsAcc != null) { - num = allBucketsAcc.collect(docs, slot); - } - return num >= 0 ? num : docs.size(); - } - - void collectFirstPhase(int segDoc, int slot) throws IOException { - if (collectAcc != null) { - collectAcc.collect(segDoc, slot); - } - if (allBucketsAcc != null) { - allBucketsAcc.collect(segDoc, slot); - } - } - - - void fillBucket(SimpleOrderedMap target, int count, int slotNum, DocSet subDomain, Query filter) throws IOException { - target.add("count", count); - if (count <= 0 && !freq.processEmpty) return; - - if (collectAcc != null && slotNum >= 0) { - collectAcc.setValues(target, slotNum); - } - - createOtherAccs(-1, 1); - - if (otherAccs == null && freq.subFacets.isEmpty()) return; - - if (subDomain == null) { - subDomain = fcontext.searcher.getDocSet(filter, fcontext.base); - } - - // if no subFacets, we only need a DocSet - // otherwise we need more? - // TODO: save something generic like "slotNum" in the context and use that to implement things like filter exclusion if necessary? - // Hmmm, but we need to look up some stuff anyway (for the label?) - // have a method like "DocSet applyConstraint(facet context, DocSet parent)" - // that's needed for domain changing things like joins anyway??? - - if (otherAccs != null) { - // do acc at a time (traversing domain each time) or do all accs for each doc? - for (SlotAcc acc : otherAccs) { - acc.reset(); // TODO: only needed if we previously used for allBuckets or missing - acc.collect(subDomain, 0); - acc.setValues(target, 0); - } - } - - processSubs(target, filter, subDomain); - } - - - @Override - protected void processStats(SimpleOrderedMap bucket, DocSet docs, int docCount) throws IOException { - if (docCount == 0 && !freq.processEmpty || freq.getFacetStats().size() == 0) { - bucket.add("count", docCount); - return; - } - createAccs(docCount, 1); - int collected = collect(docs, 0); - - // countAcc.incrementCount(0, collected); // should we set the counton the acc instead of just passing it? - - assert collected == docCount; - addStats(bucket, collected, 0); - } - - // overrides but with different signature! - void addStats(SimpleOrderedMap target, int count, int slotNum) throws IOException { - target.add("count", count); - if (count > 0 || freq.processEmpty) { - for (SlotAcc acc : accs) { - acc.setValues(target, slotNum); - } - } - } - - @Override - void setNextReader(LeafReaderContext ctx) throws IOException { - // base class calls this (for missing bucket...) ... go over accs[] in that case - super.setNextReader(ctx); - } - - void setNextReaderFirstPhase(LeafReaderContext ctx) throws IOException { - if (collectAcc != null) { - collectAcc.setNextReader(ctx); - } - if (otherAccs != null) { - for (SlotAcc acc : otherAccs) { - acc.setNextReader(ctx); - } - } - } - - static class Slot { - int slot; - public int tiebreakCompare(int slotA, int slotB) { - return slotB - slotA; - } - } -} - -class SpecialSlotAcc extends SlotAcc { - SlotAcc collectAcc; - SlotAcc[] otherAccs; - int collectAccSlot; - int otherAccsSlot; - long count; - - public SpecialSlotAcc(FacetContext fcontext, SlotAcc collectAcc, int collectAccSlot, SlotAcc[] otherAccs, int otherAccsSlot) { - super(fcontext); - this.collectAcc = collectAcc; - this.collectAccSlot = collectAccSlot; - this.otherAccs = otherAccs; - this.otherAccsSlot = otherAccsSlot; - } - - public int getCollectAccSlot() { return collectAccSlot; } - public int getOtherAccSlot() { return otherAccsSlot; } - - public long getSpecialCount() { - return count; - } - - @Override - public void collect(int doc, int slot) throws IOException { - assert slot != collectAccSlot || slot < 0; - count++; - if (collectAcc != null) { - collectAcc.collect(doc, collectAccSlot); - } - if (otherAccs != null) { - for (SlotAcc otherAcc : otherAccs) { - otherAcc.collect(doc, otherAccsSlot); - } - } - } - - @Override - public void setNextReader(LeafReaderContext readerContext) throws IOException { - // collectAcc and otherAccs will normally have setNextReader called directly on them. - // This, however, will be used when collect(DocSet,slot) variant is used on this Acc. - if (collectAcc != null) { - collectAcc.setNextReader(readerContext); - } - if (otherAccs != null) { - for (SlotAcc otherAcc : otherAccs) { - otherAcc.setNextReader(readerContext); - } - } - } - - @Override - public int compare(int slotA, int slotB) { - throw new UnsupportedOperationException(); - } - - @Override - public Object getValue(int slotNum) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public void setValues(SimpleOrderedMap bucket, int slotNum) throws IOException { - if (collectAcc != null) { - collectAcc.setValues(bucket, collectAccSlot); - } - if (otherAccs != null) { - for (SlotAcc otherAcc : otherAccs) { - otherAcc.setValues(bucket, otherAccsSlot); - } - } - } - - @Override - public void reset() { - // reset should be called on underlying accs - // TODO: but in case something does need to be done here, should we require this method to be called but do nothing for now? - throw new UnsupportedOperationException(); - } - - @Override - public void resize(Resizer resizer) { - // someone else will call resize on collectAcc directly - if (collectAccSlot >= 0) { - collectAccSlot = resizer.getNewSlot(collectAccSlot); - } - } -} - - - - -// base class for FC style of facet counting (single and multi-valued strings) -abstract class FacetFieldProcessorFCBase extends FacetFieldProcessor { - BytesRefBuilder prefixRef; - int startTermIndex; - int endTermIndex; - int nTerms; - int nDocs; - int maxSlots; - - int allBucketsSlot = -1; // slot for the primary Accs (countAcc, collectAcc) - - public FacetFieldProcessorFCBase(FacetContext fcontext, FacetField freq, SchemaField sf) { - super(fcontext, freq, sf); - } - - @Override - public void process() throws IOException { - super.process(); - sf = fcontext.searcher.getSchema().getField(freq.field); - response = getFieldCacheCounts(); - } - - - /** this BytesRef may be shared across calls and should be deep-cloned if necessary */ - abstract protected BytesRef lookupOrd(int ord) throws IOException; - abstract protected void findStartAndEndOrds() throws IOException; - abstract protected void collectDocs() throws IOException; - - - public SimpleOrderedMap getFieldCacheCounts() throws IOException { - String prefix = freq.prefix; - if (prefix == null || prefix.length() == 0) { - prefixRef = null; - } else { - prefixRef = new BytesRefBuilder(); - prefixRef.copyChars(prefix); - } - - findStartAndEndOrds(); - - maxSlots = nTerms; - - if (freq.allBuckets) { - allBucketsSlot = maxSlots++; - } - - createCollectAcc(nDocs, maxSlots); - - if (freq.allBuckets) { - allBucketsAcc = new SpecialSlotAcc(fcontext, collectAcc, allBucketsSlot, otherAccs, 0); - } - - collectDocs(); - - return findTopSlots(); - } - - - protected SimpleOrderedMap findTopSlots() throws IOException { - SimpleOrderedMap res = new SimpleOrderedMap<>(); - - int numBuckets = 0; - List bucketVals = null; - if (freq.numBuckets && fcontext.isShard()) { - bucketVals = new ArrayList(100); - } - - int off = fcontext.isShard() ? 0 : (int) freq.offset; - // add a modest amount of over-request if this is a shard request - int lim = freq.limit >= 0 ? (fcontext.isShard() ? (int)(freq.limit*1.1+4) : (int)freq.limit) : Integer.MAX_VALUE; - - int maxsize = (int)(freq.limit >= 0 ? freq.offset + lim : Integer.MAX_VALUE - 1); - maxsize = Math.min(maxsize, nTerms); - - final int sortMul = freq.sortDirection.getMultiplier(); - final SlotAcc sortAcc = this.sortAcc; - - PriorityQueue queue = new PriorityQueue(maxsize) { - @Override - protected boolean lessThan(Slot a, Slot b) { - int cmp = sortAcc.compare(a.slot, b.slot) * sortMul; - return cmp == 0 ? b.slot < a.slot : cmp < 0; - } - }; - - Slot bottom = null; - for (int i = 0; i < nTerms; i++) { - // screen out buckets not matching mincount immediately (i.e. don't even increment numBuckets) - if (effectiveMincount > 0 && countAcc.getCount(i) < effectiveMincount) { - continue; - } - - numBuckets++; - if (bucketVals != null && bucketVals.size()<100) { - int ord = startTermIndex + i; - BytesRef br = lookupOrd(ord); - Object val = sf.getType().toObject(sf, br); - bucketVals.add(val); - } - - - if (bottom != null) { - if (sortAcc.compare(bottom.slot, i) * sortMul < 0) { - bottom.slot = i; - bottom = queue.updateTop(); - } - } else if (lim > 0) { - // queue not full - Slot s = new Slot(); - s.slot = i; - queue.add(s); - if (queue.size() >= maxsize) { - bottom = queue.top(); - } - } - } - - if (freq.numBuckets) { - if (!fcontext.isShard()) { - res.add("numBuckets", numBuckets); - } else { - SimpleOrderedMap map = new SimpleOrderedMap(2); - map.add("numBuckets", numBuckets); - map.add("vals", bucketVals); - res.add("numBuckets", map); - } - } - - FacetDebugInfo fdebug = fcontext.getDebugInfo(); - if (fdebug != null) fdebug.putInfoItem("numBuckets", new Long(numBuckets)); - - // if we are deep paging, we don't have to order the highest "offset" counts. - int collectCount = Math.max(0, queue.size() - off); - assert collectCount <= lim; - int[] sortedSlots = new int[collectCount]; - for (int i = collectCount - 1; i >= 0; i--) { - sortedSlots[i] = queue.pop().slot; - } - - if (freq.allBuckets) { - SimpleOrderedMap allBuckets = new SimpleOrderedMap<>(); - allBuckets.add("count", allBucketsAcc.getSpecialCount()); - if (allBucketsAcc != null) { - allBucketsAcc.setValues(allBuckets, allBucketsSlot); - } - res.add("allBuckets", allBuckets); - } - - ArrayList bucketList = new ArrayList(collectCount); - res.add("buckets", bucketList); - - - // TODO: do this with a callback instead? - boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0; - - for (int slotNum : sortedSlots) { - SimpleOrderedMap bucket = new SimpleOrderedMap<>(); - - // get the ord of the slot... - int ord = startTermIndex + slotNum; - - BytesRef br = lookupOrd(ord); - Object val = sf.getType().toObject(sf, br); - - bucket.add("val", val); - - TermQuery filter = needFilter ? new TermQuery(new Term(sf.getName(), br)) : null; - fillBucket(bucket, countAcc.getCount(slotNum), slotNum, null, filter); - - bucketList.add(bucket); - } - - if (freq.missing) { - SimpleOrderedMap missingBucket = new SimpleOrderedMap<>(); - fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null); - res.add("missing", missingBucket); - } - - return res; - } - - -} - - - - - -// UnInvertedField implementation of field faceting -class FacetFieldProcessorUIF extends FacetFieldProcessorFCBase { - UnInvertedField uif; - TermsEnum te; - - FacetFieldProcessorUIF(FacetContext fcontext, FacetField freq, SchemaField sf) { - super(fcontext, freq, sf); - } - - @Override - protected void findStartAndEndOrds() throws IOException { - uif = UnInvertedField.getUnInvertedField(freq.field, fcontext.searcher); - te = uif.getOrdTermsEnum( fcontext.searcher.getLeafReader() ); // "te" can be null - - startTermIndex = 0; - endTermIndex = uif.numTerms(); // one past the end - - if (prefixRef != null && te != null) { - if (te.seekCeil(prefixRef.get()) == TermsEnum.SeekStatus.END) { - startTermIndex = uif.numTerms(); - } else { - startTermIndex = (int) te.ord(); - } - prefixRef.append(UnicodeUtil.BIG_TERM); - if (te.seekCeil(prefixRef.get()) == TermsEnum.SeekStatus.END) { - endTermIndex = uif.numTerms(); - } else { - endTermIndex = (int) te.ord(); - } - } - - nTerms = endTermIndex - startTermIndex; - } - - @Override - protected BytesRef lookupOrd(int ord) throws IOException { - return uif.getTermValue(te, ord); - } - - @Override - protected void collectDocs() throws IOException { - uif.collectDocs(this); - } -} - - - -class FacetFieldProcessorStream extends FacetFieldProcessor implements Closeable { - long bucketsToSkip; - long bucketsReturned; - - boolean closed; - boolean countOnly; - boolean hasSubFacets; // true if there are subfacets - int minDfFilterCache; - DocSet docs; - DocSet fastForRandomSet; - TermsEnum termsEnum = null; - SolrIndexSearcher.DocsEnumState deState = null; - PostingsEnum postingsEnum; - BytesRef startTermBytes; - BytesRef term; - LeafReaderContext[] leaves; - - - - FacetFieldProcessorStream(FacetContext fcontext, FacetField freq, SchemaField sf) { - super(fcontext, freq, sf); - } - - @Override - public void close() throws IOException { - if (!closed) { - closed = true; - // fcontext.base.decref(); // OFF-HEAP - } - } - - - @Override - public void process() throws IOException { - super.process(); - - // We need to keep the fcontext open after processing is done (since we will be streaming in the response writer). - // But if the connection is broken, we want to clean up. - // fcontext.base.incref(); // OFF-HEAP - fcontext.qcontext.addCloseHook(this); - - setup(); - response = new SimpleOrderedMap<>(); - response.add("buckets", new Iterator() { - boolean retrieveNext = true; - Object val; - - @Override - public boolean hasNext() { - if (retrieveNext) { - val = nextBucket(); - } - retrieveNext = false; - return val != null; - } - - @Override - public Object next() { - if (retrieveNext) { - val = nextBucket(); - } - retrieveNext = true; - if (val == null) { - // Last value, so clean up. In the case that we are doing streaming facets within streaming facets, - // the number of close hooks could grow very large, so we want to remove ourselves. - boolean removed = fcontext.qcontext.removeCloseHook(FacetFieldProcessorStream.this); - assert removed; - try { - close(); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error during facet streaming close", e); - } - } - return val; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }); - } - - - - public void setup() throws IOException { - - countOnly = freq.facetStats.size() == 0 || freq.facetStats.values().iterator().next() instanceof CountAgg; - hasSubFacets = freq.subFacets.size() > 0; - bucketsToSkip = freq.offset; - - createAccs(-1, 1); - - // Minimum term docFreq in order to use the filterCache for that term. - if (freq.cacheDf == -1) { // -1 means never cache - minDfFilterCache = Integer.MAX_VALUE; - } else if (freq.cacheDf == 0) { // default; compute as fraction of maxDoc - minDfFilterCache = Math.max(fcontext.searcher.maxDoc() >> 4, 3); // (minimum of 3 is for test coverage purposes) - } else { - minDfFilterCache = freq.cacheDf; - } - - docs = fcontext.base; - fastForRandomSet = null; - - if (freq.prefix != null) { - String indexedPrefix = sf.getType().toInternal(freq.prefix); - startTermBytes = new BytesRef(indexedPrefix); - } else if (sf.getType().getNumericType() != null) { - String triePrefix = TrieField.getMainValuePrefix(sf.getType()); - if (triePrefix != null) { - startTermBytes = new BytesRef(triePrefix); - } - } - - Fields fields = fcontext.searcher.getLeafReader().fields(); - Terms terms = fields == null ? null : fields.terms(sf.getName()); - - - termsEnum = null; - deState = null; - term = null; - - - if (terms != null) { - - termsEnum = terms.iterator(); - - // TODO: OPT: if seek(ord) is supported for this termsEnum, then we could use it for - // facet.offset when sorting by index order. - - if (startTermBytes != null) { - if (termsEnum.seekCeil(startTermBytes) == TermsEnum.SeekStatus.END) { - termsEnum = null; - } else { - term = termsEnum.term(); - } - } else { - // position termsEnum on first term - term = termsEnum.next(); - } - } - - List leafList = fcontext.searcher.getTopReaderContext().leaves(); - leaves = leafList.toArray( new LeafReaderContext[ leafList.size() ]); - } - - - public SimpleOrderedMap nextBucket() { - try { - return _nextBucket(); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error during facet streaming", e); - } - } - - public SimpleOrderedMap _nextBucket() throws IOException { - DocSet termSet = null; - - try { - while (term != null) { - - if (startTermBytes != null && !StringHelper.startsWith(term, startTermBytes)) { - break; - } - - int df = termsEnum.docFreq(); - if (df < effectiveMincount) { - term = termsEnum.next(); - continue; - } - - if (termSet != null) { - // termSet.decref(); // OFF-HEAP - termSet = null; - } - - int c = 0; - - if (hasSubFacets || df >= minDfFilterCache) { - // use the filter cache - - if (deState == null) { - deState = new SolrIndexSearcher.DocsEnumState(); - deState.fieldName = sf.getName(); - deState.liveDocs = fcontext.searcher.getLeafReader().getLiveDocs(); - deState.termsEnum = termsEnum; - deState.postingsEnum = postingsEnum; - deState.minSetSizeCached = minDfFilterCache; - } - - if (hasSubFacets || !countOnly) { - DocSet termsAll = fcontext.searcher.getDocSet(deState); - termSet = docs.intersection(termsAll); - // termsAll.decref(); // OFF-HEAP - c = termSet.size(); - } else { - c = fcontext.searcher.numDocs(docs, deState); - } - postingsEnum = deState.postingsEnum; - - resetStats(); - - if (!countOnly) { - collect(termSet, 0); - } - - } else { - // We don't need the docset here (meaning no sub-facets). - // if countOnly, then we are calculating some other stats... - resetStats(); - - // lazy convert to fastForRandomSet - if (fastForRandomSet == null) { - fastForRandomSet = docs; - if (docs instanceof SortedIntDocSet) { // OFF-HEAP todo: also check for native version - SortedIntDocSet sset = (SortedIntDocSet) docs; - fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size()); - } - } - // iterate over TermDocs to calculate the intersection - postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE); - - if (postingsEnum instanceof MultiPostingsEnum) { - MultiPostingsEnum.EnumWithSlice[] subs = ((MultiPostingsEnum) postingsEnum).getSubs(); - int numSubs = ((MultiPostingsEnum) postingsEnum).getNumSubs(); - for (int subindex = 0; subindex < numSubs; subindex++) { - MultiPostingsEnum.EnumWithSlice sub = subs[subindex]; - if (sub.postingsEnum == null) continue; - int base = sub.slice.start; - int docid; - - if (countOnly) { - while ((docid = sub.postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (fastForRandomSet.exists(docid + base)) c++; - } - } else { - setNextReader(leaves[sub.slice.readerIndex]); - while ((docid = sub.postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (fastForRandomSet.exists(docid + base)) { - c++; - collect(docid, 0); - } - } - } - - } - } else { - int docid; - if (countOnly) { - while ((docid = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (fastForRandomSet.exists(docid)) c++; - } - } else { - setNextReader(leaves[0]); - while ((docid = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (fastForRandomSet.exists(docid)) { - c++; - collect(docid, 0); - } - } - } - } - - } - - - - if (c < effectiveMincount) { - term = termsEnum.next(); - continue; - } - - // handle offset and limit - if (bucketsToSkip > 0) { - bucketsToSkip--; - term = termsEnum.next(); - continue; - } - - if (freq.limit >= 0 && ++bucketsReturned > freq.limit) { - return null; - } - - // set count in case other stats depend on it - countAcc.incrementCount(0, c); - - // OK, we have a good bucket to return... first get bucket value before moving to next term - Object bucketVal = sf.getType().toObject(sf, term); - TermQuery bucketQuery = hasSubFacets ? new TermQuery(new Term(freq.field, term)) : null; - term = termsEnum.next(); - - SimpleOrderedMap bucket = new SimpleOrderedMap<>(); - bucket.add("val", bucketVal); - addStats(bucket, 0); - if (hasSubFacets) { - processSubs(bucket, bucketQuery, termSet); - } - - // TODO... termSet needs to stick around for streaming sub-facets? - - return bucket; - - } - - } finally { - if (termSet != null) { - // termSet.decref(); // OFF-HEAP - termSet = null; - } - } - - - // end of the iteration - return null; - } - - - -} - - diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java new file mode 100644 index 00000000000..a73732135e8 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java @@ -0,0 +1,369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.facet; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Query; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.DocSet; + +/** + * Facet processing based on field values. (not range nor by query) + * @see FacetField + */ +abstract class FacetFieldProcessor extends FacetProcessor { + SchemaField sf; + SlotAcc indexOrderAcc; + int effectiveMincount; + + Map deferredAggs; // null if none + + // TODO: push any of this down to base class? + + // + // For sort="x desc", collectAcc would point to "x", and sortAcc would also point to "x". + // collectAcc would be used to accumulate all buckets, and sortAcc would be used to sort those buckets. + // + SlotAcc collectAcc; // Accumulator to collect across entire domain (in addition to the countAcc). May be null. + SlotAcc sortAcc; // Accumulator to use for sorting *only* (i.e. not used for collection). May be an alias of countAcc, collectAcc, or indexOrderAcc + SlotAcc[] otherAccs; // Accumulators that do not need to be calculated across all buckets. + + SpecialSlotAcc allBucketsAcc; // this can internally refer to otherAccs and/or collectAcc. setNextReader should be called on otherAccs directly if they exist. + + FacetFieldProcessor(FacetContext fcontext, FacetField freq, SchemaField sf) { + super(fcontext, freq); + this.sf = sf; + this.effectiveMincount = (int)(fcontext.isShard() ? Math.min(1 , freq.mincount) : freq.mincount); + } + + // This is used to create accs for second phase (or to create accs for all aggs) + @Override + protected void createAccs(int docCount, int slotCount) throws IOException { + if (accMap == null) { + accMap = new LinkedHashMap<>(); + } + + // allow a custom count acc to be used + if (countAcc == null) { + countAcc = new CountSlotArrAcc(fcontext, slotCount); + countAcc.key = "count"; + } + + if (accs != null) { + // reuse these accs, but reset them first + for (SlotAcc acc : accs) { + acc.reset(); + } + return; + } else { + accs = new SlotAcc[ freq.getFacetStats().size() ]; + } + + int accIdx = 0; + for (Map.Entry entry : freq.getFacetStats().entrySet()) { + SlotAcc acc = null; + if (slotCount == 1) { + acc = accMap.get(entry.getKey()); + if (acc != null) { + acc.reset(); + } + } + if (acc == null) { + acc = entry.getValue().createSlotAcc(fcontext, docCount, slotCount); + acc.key = entry.getKey(); + accMap.put(acc.key, acc); + } + accs[accIdx++] = acc; + } + } + + void createCollectAcc(int numDocs, int numSlots) throws IOException { + accMap = new LinkedHashMap<>(); + + // we always count... + // allow a subclass to set a custom counter. + if (countAcc == null) { + countAcc = new CountSlotArrAcc(fcontext, numSlots); + } + + if ("count".equals(freq.sortVariable)) { + sortAcc = countAcc; + deferredAggs = freq.getFacetStats(); + } else if ("index".equals(freq.sortVariable)) { + // allow subclass to set indexOrderAcc first + if (indexOrderAcc == null) { + // This sorting accumulator just goes by the slot number, so does not need to be collected + // and hence does not need to find it's way into the accMap or accs array. + indexOrderAcc = new SortSlotAcc(fcontext); + } + sortAcc = indexOrderAcc; + deferredAggs = freq.getFacetStats(); + } else { + AggValueSource sortAgg = freq.getFacetStats().get(freq.sortVariable); + if (sortAgg != null) { + collectAcc = sortAgg.createSlotAcc(fcontext, numDocs, numSlots); + collectAcc.key = freq.sortVariable; // TODO: improve this + } + sortAcc = collectAcc; + deferredAggs = new HashMap<>(freq.getFacetStats()); + deferredAggs.remove(freq.sortVariable); + } + + if (deferredAggs.size() == 0) { + deferredAggs = null; + } + + boolean needOtherAccs = freq.allBuckets; // TODO: use for missing too... + + if (!needOtherAccs) { + // we may need them later, but we don't want to create them now + // otherwise we won't know if we need to call setNextReader on them. + return; + } + + // create the deferred aggs up front for use by allBuckets + createOtherAccs(numDocs, 1); + } + + private void createOtherAccs(int numDocs, int numSlots) throws IOException { + if (otherAccs != null) { + // reuse existing accumulators + for (SlotAcc acc : otherAccs) { + acc.reset(); // todo - make reset take numDocs and numSlots? + } + return; + } + + int numDeferred = deferredAggs == null ? 0 : deferredAggs.size(); + if (numDeferred <= 0) return; + + otherAccs = new SlotAcc[ numDeferred ]; + + int otherAccIdx = 0; + for (Map.Entry entry : deferredAggs.entrySet()) { + AggValueSource agg = entry.getValue(); + SlotAcc acc = agg.createSlotAcc(fcontext, numDocs, numSlots); + acc.key = entry.getKey(); + accMap.put(acc.key, acc); + otherAccs[otherAccIdx++] = acc; + } + + if (numDeferred == freq.getFacetStats().size()) { + // accs and otherAccs are the same... + accs = otherAccs; + } + } + + int collectFirstPhase(DocSet docs, int slot) throws IOException { + int num = -1; + if (collectAcc != null) { + num = collectAcc.collect(docs, slot); + } + if (allBucketsAcc != null) { + num = allBucketsAcc.collect(docs, slot); + } + return num >= 0 ? num : docs.size(); + } + + void collectFirstPhase(int segDoc, int slot) throws IOException { + if (collectAcc != null) { + collectAcc.collect(segDoc, slot); + } + if (allBucketsAcc != null) { + allBucketsAcc.collect(segDoc, slot); + } + } + + void fillBucket(SimpleOrderedMap target, int count, int slotNum, DocSet subDomain, Query filter) throws IOException { + target.add("count", count); + if (count <= 0 && !freq.processEmpty) return; + + if (collectAcc != null && slotNum >= 0) { + collectAcc.setValues(target, slotNum); + } + + createOtherAccs(-1, 1); + + if (otherAccs == null && freq.subFacets.isEmpty()) return; + + if (subDomain == null) { + subDomain = fcontext.searcher.getDocSet(filter, fcontext.base); + } + + // if no subFacets, we only need a DocSet + // otherwise we need more? + // TODO: save something generic like "slotNum" in the context and use that to implement things like filter exclusion if necessary? + // Hmmm, but we need to look up some stuff anyway (for the label?) + // have a method like "DocSet applyConstraint(facet context, DocSet parent)" + // that's needed for domain changing things like joins anyway??? + + if (otherAccs != null) { + // do acc at a time (traversing domain each time) or do all accs for each doc? + for (SlotAcc acc : otherAccs) { + acc.reset(); // TODO: only needed if we previously used for allBuckets or missing + acc.collect(subDomain, 0); + acc.setValues(target, 0); + } + } + + processSubs(target, filter, subDomain); + } + + @Override + protected void processStats(SimpleOrderedMap bucket, DocSet docs, int docCount) throws IOException { + if (docCount == 0 && !freq.processEmpty || freq.getFacetStats().size() == 0) { + bucket.add("count", docCount); + return; + } + createAccs(docCount, 1); + int collected = collect(docs, 0); + + // countAcc.incrementCount(0, collected); // should we set the counton the acc instead of just passing it? + + assert collected == docCount; + addStats(bucket, collected, 0); + } + + // overrides but with different signature! + private void addStats(SimpleOrderedMap target, int count, int slotNum) throws IOException { + target.add("count", count); + if (count > 0 || freq.processEmpty) { + for (SlotAcc acc : accs) { + acc.setValues(target, slotNum); + } + } + } + + @Override + void setNextReader(LeafReaderContext ctx) throws IOException { + // base class calls this (for missing bucket...) ... go over accs[] in that case + super.setNextReader(ctx); + } + + void setNextReaderFirstPhase(LeafReaderContext ctx) throws IOException { + if (collectAcc != null) { + collectAcc.setNextReader(ctx); + } + if (otherAccs != null) { + for (SlotAcc acc : otherAccs) { + acc.setNextReader(ctx); + } + } + } + + static class Slot { + int slot; + public int tiebreakCompare(int slotA, int slotB) { + return slotB - slotA; + } + } + + static class SpecialSlotAcc extends SlotAcc { + SlotAcc collectAcc; + SlotAcc[] otherAccs; + int collectAccSlot; + int otherAccsSlot; + long count; + + SpecialSlotAcc(FacetContext fcontext, SlotAcc collectAcc, int collectAccSlot, SlotAcc[] otherAccs, int otherAccsSlot) { + super(fcontext); + this.collectAcc = collectAcc; + this.collectAccSlot = collectAccSlot; + this.otherAccs = otherAccs; + this.otherAccsSlot = otherAccsSlot; + } + + public int getCollectAccSlot() { return collectAccSlot; } + public int getOtherAccSlot() { return otherAccsSlot; } + + long getSpecialCount() { + return count; + } + + @Override + public void collect(int doc, int slot) throws IOException { + assert slot != collectAccSlot || slot < 0; + count++; + if (collectAcc != null) { + collectAcc.collect(doc, collectAccSlot); + } + if (otherAccs != null) { + for (SlotAcc otherAcc : otherAccs) { + otherAcc.collect(doc, otherAccsSlot); + } + } + } + + @Override + public void setNextReader(LeafReaderContext readerContext) throws IOException { + // collectAcc and otherAccs will normally have setNextReader called directly on them. + // This, however, will be used when collect(DocSet,slot) variant is used on this Acc. + if (collectAcc != null) { + collectAcc.setNextReader(readerContext); + } + if (otherAccs != null) { + for (SlotAcc otherAcc : otherAccs) { + otherAcc.setNextReader(readerContext); + } + } + } + + @Override + public int compare(int slotA, int slotB) { + throw new UnsupportedOperationException(); + } + + @Override + public Object getValue(int slotNum) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void setValues(SimpleOrderedMap bucket, int slotNum) throws IOException { + if (collectAcc != null) { + collectAcc.setValues(bucket, collectAccSlot); + } + if (otherAccs != null) { + for (SlotAcc otherAcc : otherAccs) { + otherAcc.setValues(bucket, otherAccsSlot); + } + } + } + + @Override + public void reset() { + // reset should be called on underlying accs + // TODO: but in case something does need to be done here, should we require this method to be called but do nothing for now? + throw new UnsupportedOperationException(); + } + + @Override + public void resize(Resizer resizer) { + // someone else will call resize on collectAcc directly + if (collectAccSlot >= 0) { + collectAccSlot = resizer.getNewSlot(collectAccSlot); + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java new file mode 100644 index 00000000000..10aa4d9b393 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.facet; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.PriorityQueue; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.schema.SchemaField; + +/** + * Base class for DV/UIF accumulating counts into an array by ordinal. + * It can handle terms (strings), not numbers directly but those encoded as terms, and is multi-valued capable. + */ +abstract class FacetFieldProcessorByArray extends FacetFieldProcessor { + BytesRefBuilder prefixRef; + int startTermIndex; + int endTermIndex; + int nTerms; + int nDocs; + int maxSlots; + + int allBucketsSlot = -1; // slot for the primary Accs (countAcc, collectAcc) + + FacetFieldProcessorByArray(FacetContext fcontext, FacetField freq, SchemaField sf) { + super(fcontext, freq, sf); + } + + abstract protected void findStartAndEndOrds() throws IOException; + + abstract protected void collectDocs() throws IOException; + + /** this BytesRef may be shared across calls and should be deep-cloned if necessary */ + abstract protected BytesRef lookupOrd(int ord) throws IOException; + + @Override + public void process() throws IOException { + super.process(); + sf = fcontext.searcher.getSchema().getField(freq.field); + response = getFieldCacheCounts(); + } + + private SimpleOrderedMap getFieldCacheCounts() throws IOException { + String prefix = freq.prefix; + if (prefix == null || prefix.length() == 0) { + prefixRef = null; + } else { + prefixRef = new BytesRefBuilder(); + prefixRef.copyChars(prefix); + } + + findStartAndEndOrds(); + + maxSlots = nTerms; + + if (freq.allBuckets) { + allBucketsSlot = maxSlots++; + } + + createCollectAcc(nDocs, maxSlots); + + if (freq.allBuckets) { + allBucketsAcc = new SpecialSlotAcc(fcontext, collectAcc, allBucketsSlot, otherAccs, 0); + } + + collectDocs(); + + return findTopSlots(); + } + + private SimpleOrderedMap findTopSlots() throws IOException { + SimpleOrderedMap res = new SimpleOrderedMap<>(); + + int numBuckets = 0; + List bucketVals = null; + if (freq.numBuckets && fcontext.isShard()) { + bucketVals = new ArrayList<>(100); + } + + int off = fcontext.isShard() ? 0 : (int) freq.offset; + // add a modest amount of over-request if this is a shard request + int lim = freq.limit >= 0 ? (fcontext.isShard() ? (int)(freq.limit*1.1+4) : (int)freq.limit) : Integer.MAX_VALUE; + + int maxsize = (int)(freq.limit >= 0 ? freq.offset + lim : Integer.MAX_VALUE - 1); + maxsize = Math.min(maxsize, nTerms); + + final int sortMul = freq.sortDirection.getMultiplier(); + final SlotAcc sortAcc = this.sortAcc; + + PriorityQueue queue = new PriorityQueue(maxsize) { + @Override + protected boolean lessThan(Slot a, Slot b) { + int cmp = sortAcc.compare(a.slot, b.slot) * sortMul; + return cmp == 0 ? b.slot < a.slot : cmp < 0; + } + }; + + Slot bottom = null; + for (int i = 0; i < nTerms; i++) { + // screen out buckets not matching mincount immediately (i.e. don't even increment numBuckets) + if (effectiveMincount > 0 && countAcc.getCount(i) < effectiveMincount) { + continue; + } + + numBuckets++; + if (bucketVals != null && bucketVals.size()<100) { + int ord = startTermIndex + i; + BytesRef br = lookupOrd(ord); + Object val = sf.getType().toObject(sf, br); + bucketVals.add(val); + } + + if (bottom != null) { + if (sortAcc.compare(bottom.slot, i) * sortMul < 0) { + bottom.slot = i; + bottom = queue.updateTop(); + } + } else if (lim > 0) { + // queue not full + Slot s = new Slot(); + s.slot = i; + queue.add(s); + if (queue.size() >= maxsize) { + bottom = queue.top(); + } + } + } + + if (freq.numBuckets) { + if (!fcontext.isShard()) { + res.add("numBuckets", numBuckets); + } else { + SimpleOrderedMap map = new SimpleOrderedMap<>(2); + map.add("numBuckets", numBuckets); + map.add("vals", bucketVals); + res.add("numBuckets", map); + } + } + + FacetDebugInfo fdebug = fcontext.getDebugInfo(); + if (fdebug != null) fdebug.putInfoItem("numBuckets", (long) numBuckets); + + // if we are deep paging, we don't have to order the highest "offset" counts. + int collectCount = Math.max(0, queue.size() - off); + assert collectCount <= lim; + int[] sortedSlots = new int[collectCount]; + for (int i = collectCount - 1; i >= 0; i--) { + sortedSlots[i] = queue.pop().slot; + } + + if (freq.allBuckets) { + SimpleOrderedMap allBuckets = new SimpleOrderedMap<>(); + allBuckets.add("count", allBucketsAcc.getSpecialCount()); + if (allBucketsAcc != null) { + allBucketsAcc.setValues(allBuckets, allBucketsSlot); + } + res.add("allBuckets", allBuckets); + } + + ArrayList> bucketList = new ArrayList<>(collectCount); + res.add("buckets", bucketList); + + // TODO: do this with a callback instead? + boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0; + + for (int slotNum : sortedSlots) { + SimpleOrderedMap bucket = new SimpleOrderedMap<>(); + + // get the ord of the slot... + int ord = startTermIndex + slotNum; + + BytesRef br = lookupOrd(ord); + Object val = sf.getType().toObject(sf, br); + + bucket.add("val", val); + + TermQuery filter = needFilter ? new TermQuery(new Term(sf.getName(), br)) : null; + fillBucket(bucket, countAcc.getCount(slotNum), slotNum, null, filter); + + bucketList.add(bucket); + } + + if (freq.missing) { + SimpleOrderedMap missingBucket = new SimpleOrderedMap<>(); + fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null); + res.add("missing", missingBucket); + } + + return res; + } + +} diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorDV.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java similarity index 98% rename from solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorDV.java rename to solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java index 12056aa118a..1ef42848ffb 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorDV.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java @@ -34,23 +34,22 @@ import org.apache.solr.common.SolrException; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.Filter; -class FacetFieldProcessorDV extends FacetFieldProcessorFCBase { +/** + * Grabs values from {@link DocValues}. + */ +class FacetFieldProcessorByArrayDV extends FacetFieldProcessorByArray { static boolean unwrap_singleValued_multiDv = true; // only set to false for test coverage boolean multiValuedField; SortedSetDocValues si; // only used for term lookups (for both single and multi-valued) MultiDocValues.OrdinalMap ordinalMap = null; // maps per-segment ords to global ords - - public FacetFieldProcessorDV(FacetContext fcontext, FacetField freq, SchemaField sf) { + FacetFieldProcessorByArrayDV(FacetContext fcontext, FacetField freq, SchemaField sf) { super(fcontext, freq, sf); multiValuedField = sf.multiValued() || sf.getType().multiValuedFieldCache(); } - protected BytesRef lookupOrd(int ord) throws IOException { - return si.lookupOrd(ord); - } - + @Override protected void findStartAndEndOrds() throws IOException { if (multiValuedField) { si = FieldUtil.getSortedSetDocValues(fcontext.qcontext, sf, null); @@ -175,16 +174,9 @@ class FacetFieldProcessorDV extends FacetFieldProcessorFCBase { reuse = null; // better GC } - private int[] reuse; - private int[] getCountArr(int maxNeeded) { - if (reuse == null) { - // make the count array large enough for any segment - // FUTURE: (optionally) directly use the array of the CountAcc for an optimized index.. - reuse = new int[(int) si.getValueCount() + 1]; - } else { - Arrays.fill(reuse, 0, maxNeeded, 0); - } - return reuse; + @Override + protected BytesRef lookupOrd(int ord) throws IOException { + return si.lookupOrd(ord); } private void collectPerSeg(SortedDocValues singleDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException { @@ -205,7 +197,6 @@ class FacetFieldProcessorDV extends FacetFieldProcessorFCBase { } } - private void collectPerSeg(SortedSetDocValues multiDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException { int segMax = (int)multiDv.getValueCount(); final int[] counts = getCountArr( segMax ); @@ -229,6 +220,18 @@ class FacetFieldProcessorDV extends FacetFieldProcessorFCBase { } } + private int[] reuse; + private int[] getCountArr(int maxNeeded) { + if (reuse == null) { + // make the count array large enough for any segment + // FUTURE: (optionally) directly use the array of the CountAcc for an optimized index.. + reuse = new int[(int) si.getValueCount() + 1]; + } else { + Arrays.fill(reuse, 0, maxNeeded, 0); + } + return reuse; + } + private void collectDocs(SortedDocValues singleDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException { int doc; while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayUIF.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayUIF.java new file mode 100644 index 00000000000..dfee257cb9e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayUIF.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.facet; + +import java.io.IOException; + +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.solr.schema.SchemaField; + +/** {@link UnInvertedField} implementation of field faceting. + * It's a top-level term cache. */ +class FacetFieldProcessorByArrayUIF extends FacetFieldProcessorByArray { + UnInvertedField uif; + TermsEnum te; + + FacetFieldProcessorByArrayUIF(FacetContext fcontext, FacetField freq, SchemaField sf) { + super(fcontext, freq, sf); + } + + @Override + protected void findStartAndEndOrds() throws IOException { + uif = UnInvertedField.getUnInvertedField(freq.field, fcontext.searcher); + te = uif.getOrdTermsEnum( fcontext.searcher.getLeafReader() ); // "te" can be null + + startTermIndex = 0; + endTermIndex = uif.numTerms(); // one past the end + + if (prefixRef != null && te != null) { + if (te.seekCeil(prefixRef.get()) == TermsEnum.SeekStatus.END) { + startTermIndex = uif.numTerms(); + } else { + startTermIndex = (int) te.ord(); + } + prefixRef.append(UnicodeUtil.BIG_TERM); + if (te.seekCeil(prefixRef.get()) == TermsEnum.SeekStatus.END) { + endTermIndex = uif.numTerms(); + } else { + endTermIndex = (int) te.ord(); + } + } + + nTerms = endTermIndex - startTermIndex; + } + + @Override + protected void collectDocs() throws IOException { + uif.collectDocs(this); + } + + @Override + protected BytesRef lookupOrd(int ord) throws IOException { + return uif.getTermValue(te, ord); + } +} diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java new file mode 100644 index 00000000000..005f4d1f174 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.facet; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiPostingsEnum; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.StringHelper; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.TrieField; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.HashDocSet; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SortedIntDocSet; + +/** + * Enumerates indexed terms in order in a streaming fashion. + * It's able to stream since no data needs to be accumulated so long as it's index order. + */ +class FacetFieldProcessorByEnumTermsStream extends FacetFieldProcessor implements Closeable { + long bucketsToSkip; + long bucketsReturned; + + boolean closed; + boolean countOnly; + boolean hasSubFacets; // true if there are subfacets + int minDfFilterCache; + DocSet docs; + DocSet fastForRandomSet; + TermsEnum termsEnum = null; + SolrIndexSearcher.DocsEnumState deState = null; + PostingsEnum postingsEnum; + BytesRef startTermBytes; + BytesRef term; + LeafReaderContext[] leaves; + + FacetFieldProcessorByEnumTermsStream(FacetContext fcontext, FacetField freq, SchemaField sf) { + super(fcontext, freq, sf); + } + + @Override + public void close() throws IOException { + if (!closed) { + closed = true; + // fcontext.base.decref(); // OFF-HEAP + } + } + + @Override + public void process() throws IOException { + super.process(); + + // We need to keep the fcontext open after processing is done (since we will be streaming in the response writer). + // But if the connection is broken, we want to clean up. + // fcontext.base.incref(); // OFF-HEAP + fcontext.qcontext.addCloseHook(this); + + setup(); + response = new SimpleOrderedMap<>(); + response.add("buckets", new Iterator() { + boolean retrieveNext = true; + Object val; + + @Override + public boolean hasNext() { + if (retrieveNext) { + val = nextBucket(); + } + retrieveNext = false; + return val != null; + } + + @Override + public Object next() { + if (retrieveNext) { + val = nextBucket(); + } + retrieveNext = true; + if (val == null) { + // Last value, so clean up. In the case that we are doing streaming facets within streaming facets, + // the number of close hooks could grow very large, so we want to remove ourselves. + boolean removed = fcontext.qcontext.removeCloseHook(FacetFieldProcessorByEnumTermsStream.this); + assert removed; + try { + close(); + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error during facet streaming close", e); + } + } + return val; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }); + } + + private void setup() throws IOException { + + countOnly = freq.facetStats.size() == 0 || freq.facetStats.values().iterator().next() instanceof CountAgg; + hasSubFacets = freq.subFacets.size() > 0; + bucketsToSkip = freq.offset; + + createAccs(-1, 1); + + // Minimum term docFreq in order to use the filterCache for that term. + if (freq.cacheDf == -1) { // -1 means never cache + minDfFilterCache = Integer.MAX_VALUE; + } else if (freq.cacheDf == 0) { // default; compute as fraction of maxDoc + minDfFilterCache = Math.max(fcontext.searcher.maxDoc() >> 4, 3); // (minimum of 3 is for test coverage purposes) + } else { + minDfFilterCache = freq.cacheDf; + } + + docs = fcontext.base; + fastForRandomSet = null; + + if (freq.prefix != null) { + String indexedPrefix = sf.getType().toInternal(freq.prefix); + startTermBytes = new BytesRef(indexedPrefix); + } else if (sf.getType().getNumericType() != null) { + String triePrefix = TrieField.getMainValuePrefix(sf.getType()); + if (triePrefix != null) { + startTermBytes = new BytesRef(triePrefix); + } + } + + Fields fields = fcontext.searcher.getLeafReader().fields(); + Terms terms = fields == null ? null : fields.terms(sf.getName()); + + termsEnum = null; + deState = null; + term = null; + + + if (terms != null) { + + termsEnum = terms.iterator(); + + // TODO: OPT: if seek(ord) is supported for this termsEnum, then we could use it for + // facet.offset when sorting by index order. + + if (startTermBytes != null) { + if (termsEnum.seekCeil(startTermBytes) == TermsEnum.SeekStatus.END) { + termsEnum = null; + } else { + term = termsEnum.term(); + } + } else { + // position termsEnum on first term + term = termsEnum.next(); + } + } + + List leafList = fcontext.searcher.getTopReaderContext().leaves(); + leaves = leafList.toArray( new LeafReaderContext[ leafList.size() ]); + } + + private SimpleOrderedMap nextBucket() { + try { + return _nextBucket(); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error during facet streaming", e); + } + } + + private SimpleOrderedMap _nextBucket() throws IOException { + DocSet termSet = null; + + try { + while (term != null) { + + if (startTermBytes != null && !StringHelper.startsWith(term, startTermBytes)) { + break; + } + + int df = termsEnum.docFreq(); + if (df < effectiveMincount) { + term = termsEnum.next(); + continue; + } + + if (termSet != null) { + // termSet.decref(); // OFF-HEAP + termSet = null; + } + + int c = 0; + + if (hasSubFacets || df >= minDfFilterCache) { + // use the filter cache + + if (deState == null) { + deState = new SolrIndexSearcher.DocsEnumState(); + deState.fieldName = sf.getName(); + deState.liveDocs = fcontext.searcher.getLeafReader().getLiveDocs(); + deState.termsEnum = termsEnum; + deState.postingsEnum = postingsEnum; + deState.minSetSizeCached = minDfFilterCache; + } + + if (hasSubFacets || !countOnly) { + DocSet termsAll = fcontext.searcher.getDocSet(deState); + termSet = docs.intersection(termsAll); + // termsAll.decref(); // OFF-HEAP + c = termSet.size(); + } else { + c = fcontext.searcher.numDocs(docs, deState); + } + postingsEnum = deState.postingsEnum; + + resetStats(); + + if (!countOnly) { + collect(termSet, 0); + } + + } else { + // We don't need the docset here (meaning no sub-facets). + // if countOnly, then we are calculating some other stats... + resetStats(); + + // lazy convert to fastForRandomSet + if (fastForRandomSet == null) { + fastForRandomSet = docs; + if (docs instanceof SortedIntDocSet) { // OFF-HEAP todo: also check for native version + SortedIntDocSet sset = (SortedIntDocSet) docs; + fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size()); + } + } + // iterate over TermDocs to calculate the intersection + postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE); + + if (postingsEnum instanceof MultiPostingsEnum) { + MultiPostingsEnum.EnumWithSlice[] subs = ((MultiPostingsEnum) postingsEnum).getSubs(); + int numSubs = ((MultiPostingsEnum) postingsEnum).getNumSubs(); + for (int subindex = 0; subindex < numSubs; subindex++) { + MultiPostingsEnum.EnumWithSlice sub = subs[subindex]; + if (sub.postingsEnum == null) continue; + int base = sub.slice.start; + int docid; + + if (countOnly) { + while ((docid = sub.postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (fastForRandomSet.exists(docid + base)) c++; + } + } else { + setNextReader(leaves[sub.slice.readerIndex]); + while ((docid = sub.postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (fastForRandomSet.exists(docid + base)) { + c++; + collect(docid, 0); + } + } + } + + } + } else { + int docid; + if (countOnly) { + while ((docid = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (fastForRandomSet.exists(docid)) c++; + } + } else { + setNextReader(leaves[0]); + while ((docid = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (fastForRandomSet.exists(docid)) { + c++; + collect(docid, 0); + } + } + } + } + + } + + if (c < effectiveMincount) { + term = termsEnum.next(); + continue; + } + + // handle offset and limit + if (bucketsToSkip > 0) { + bucketsToSkip--; + term = termsEnum.next(); + continue; + } + + if (freq.limit >= 0 && ++bucketsReturned > freq.limit) { + return null; + } + + // set count in case other stats depend on it + countAcc.incrementCount(0, c); + + // OK, we have a good bucket to return... first get bucket value before moving to next term + Object bucketVal = sf.getType().toObject(sf, term); + TermQuery bucketQuery = hasSubFacets ? new TermQuery(new Term(freq.field, term)) : null; + term = termsEnum.next(); + + SimpleOrderedMap bucket = new SimpleOrderedMap<>(); + bucket.add("val", bucketVal); + addStats(bucket, 0); + if (hasSubFacets) { + processSubs(bucket, bucketQuery, termSet); + } + + // TODO... termSet needs to stick around for streaming sub-facets? + + return bucket; + + } + + } finally { + if (termSet != null) { + // termSet.decref(); // OFF-HEAP + termSet = null; + } + } + + // end of the iteration + return null; + } + +} diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorNumeric.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java similarity index 94% rename from solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorNumeric.java rename to solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java index 6ab4c26cb02..842df202400 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorNumeric.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java @@ -32,10 +32,15 @@ import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocIterator; -class FacetFieldProcessorNumeric extends FacetFieldProcessor { +/** + * Facets numbers into a hash table. + * It currently only works with {@link NumericDocValues} (single-valued). + */ +class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor { static int MAXIMUM_STARTING_TABLE_SIZE=1024; // must be a power of two, non-final to support setting by tests - static class LongCounts { + /** a hash table with long keys (what we're counting) and integer values (counts) */ + private static class LongCounts { static final float LOAD_FACTOR = 0.7f; @@ -55,7 +60,7 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { } /** Current number of slots in the hash table */ - public int numSlots() { + int numSlots() { return vals.length; } @@ -130,69 +135,22 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { } + int allBucketsSlot = -1; - - FacetFieldProcessorNumeric(FacetContext fcontext, FacetField freq, SchemaField sf) { + FacetFieldProcessorByHashNumeric(FacetContext fcontext, FacetField freq, SchemaField sf) { super(fcontext, freq, sf); } - int allBucketsSlot = -1; - @Override public void process() throws IOException { super.process(); response = calcFacets(); } - private void doRehash(LongCounts table) { - if (collectAcc == null && allBucketsAcc == null) return; - - // Our "count" acc is backed by the hash table and will already be rehashed - // otherAccs don't need to be rehashed - - int newTableSize = table.numSlots(); - int numSlots = newTableSize; - final int oldAllBucketsSlot = allBucketsSlot; - if (oldAllBucketsSlot >= 0) { - allBucketsSlot = numSlots++; - } - - final int finalNumSlots = numSlots; - final int[] mapping = table.oldToNewMapping; - - SlotAcc.Resizer resizer = new SlotAcc.Resizer() { - @Override - public int getNewSize() { - return finalNumSlots; - } - - @Override - public int getNewSlot(int oldSlot) { - if (oldSlot < mapping.length) { - return mapping[oldSlot]; - } - if (oldSlot == oldAllBucketsSlot) { - return allBucketsSlot; - } - return -1; - } - }; - - // NOTE: resizing isn't strictly necessary for missing/allBuckets... we could just set the new slot directly - if (collectAcc != null) { - collectAcc.resize(resizer); - } - if (allBucketsAcc != null) { - allBucketsAcc.resize(resizer); - } - } - - public SimpleOrderedMap calcFacets() throws IOException { - + private SimpleOrderedMap calcFacets() throws IOException { final FacetRangeProcessor.Calc calc = FacetRangeProcessor.getNumericCalc(sf); - // TODO: it would be really nice to know the number of unique values!!!! int possibleValues = fcontext.base.size(); @@ -212,7 +170,6 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { int numMissing = 0; - if (freq.allBuckets) { allBucketsSlot = numSlots++; } @@ -325,7 +282,6 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { } } - // // collection done, time to find the top slots // @@ -333,7 +289,7 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { int numBuckets = 0; List bucketVals = null; if (freq.numBuckets && fcontext.isShard()) { - bucketVals = new ArrayList(100); + bucketVals = new ArrayList<>(100); } int off = fcontext.isShard() ? 0 : (int) freq.offset; @@ -378,13 +334,12 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { bottom = queue.insertWithOverflow(bottom); } - - SimpleOrderedMap res = new SimpleOrderedMap(); + SimpleOrderedMap res = new SimpleOrderedMap<>(); if (freq.numBuckets) { if (!fcontext.isShard()) { res.add("numBuckets", numBuckets); } else { - SimpleOrderedMap map = new SimpleOrderedMap(2); + SimpleOrderedMap map = new SimpleOrderedMap<>(2); map.add("numBuckets", numBuckets); map.add("vals", bucketVals); res.add("numBuckets", map); @@ -392,7 +347,7 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { } FacetDebugInfo fdebug = fcontext.getDebugInfo(); - if (fdebug != null) fdebug.putInfoItem("numBuckets", new Long(numBuckets)); + if (fdebug != null) fdebug.putInfoItem("numBuckets", (long) numBuckets); if (freq.allBuckets) { SimpleOrderedMap allBuckets = new SimpleOrderedMap<>(); @@ -419,7 +374,7 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { sortedSlots[i] = queue.pop().slot; } - ArrayList bucketList = new ArrayList(collectCount); + ArrayList bucketList = new ArrayList<>(collectCount); res.add("buckets", bucketList); boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0; @@ -436,8 +391,49 @@ class FacetFieldProcessorNumeric extends FacetFieldProcessor { bucketList.add(bucket); } - - return res; } + + private void doRehash(LongCounts table) { + if (collectAcc == null && allBucketsAcc == null) return; + + // Our "count" acc is backed by the hash table and will already be rehashed + // otherAccs don't need to be rehashed + + int newTableSize = table.numSlots(); + int numSlots = newTableSize; + final int oldAllBucketsSlot = allBucketsSlot; + if (oldAllBucketsSlot >= 0) { + allBucketsSlot = numSlots++; + } + + final int finalNumSlots = numSlots; + final int[] mapping = table.oldToNewMapping; + + SlotAcc.Resizer resizer = new SlotAcc.Resizer() { + @Override + public int getNewSize() { + return finalNumSlots; + } + + @Override + public int getNewSlot(int oldSlot) { + if (oldSlot < mapping.length) { + return mapping[oldSlot]; + } + if (oldSlot == oldAllBucketsSlot) { + return allBucketsSlot; + } + return -1; + } + }; + + // NOTE: resizing isn't strictly necessary for missing/allBuckets... we could just set the new slot directly + if (collectAcc != null) { + collectAcc.resize(resizer); + } + if (allBucketsAcc != null) { + allBucketsAcc.resize(resizer); + } + } } diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java index b1281f4b23b..fa26319e812 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java @@ -45,27 +45,18 @@ import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SyntaxError; import org.apache.solr.util.RTimer; -public class FacetProcessor { - protected SimpleOrderedMap response; - protected FacetContext fcontext; - protected FacetRequestT freq; +public abstract class FacetProcessor { + SimpleOrderedMap response; + FacetContext fcontext; + FacetRequestT freq; LinkedHashMap accMap; - protected SlotAcc[] accs; - protected CountSlotAcc countAcc; + SlotAcc[] accs; + CountSlotAcc countAcc; - FacetProcessor(FacetContext fcontext, FacetRequestT freq) { - this.fcontext = fcontext; - this.freq = freq; - } - - public void process() throws IOException { - handleDomainChanges(); - } - /** factory method for invoking json facet framework as whole */ - public static FacetProcessor createProcessor(SolrQueryRequest req, - Map params, DocSet docs){ + public static FacetProcessor createProcessor(SolrQueryRequest req, + Map params, DocSet docs){ FacetParser parser = new FacetTopParser(req); FacetRequest facetRequest = null; try { @@ -83,39 +74,25 @@ public class FacetProcessor { return facetRequest.createFacetProcessor(fcontext); } - protected void handleDomainChanges() throws IOException { + FacetProcessor(FacetContext fcontext, FacetRequestT freq) { + this.fcontext = fcontext; + this.freq = freq; + } + + public Object getResponse() { + return response; + } + + public void process() throws IOException { + handleDomainChanges(); + } + + private void handleDomainChanges() throws IOException { if (freq.domain == null) return; handleFilterExclusions(); handleBlockJoin(); } - private void handleBlockJoin() throws IOException { - if (!(freq.domain.toChildren || freq.domain.toParent)) return; - - // TODO: avoid query parsing per-bucket somehow... - String parentStr = freq.domain.parents; - Query parentQuery; - try { - QParser parser = QParser.getParser(parentStr, fcontext.req); - parentQuery = parser.getQuery(); - } catch (SyntaxError err) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing block join parent specification: " + parentStr); - } - - BitDocSet parents = fcontext.searcher.getDocSetBits(parentQuery); - DocSet input = fcontext.base; - DocSet result; - - if (freq.domain.toChildren) { - DocSet filt = fcontext.searcher.getDocSetBits( new MatchAllDocsQuery() ); - result = BlockJoin.toChildren(input, parents, filt, fcontext.qcontext); - } else { - result = BlockJoin.toParents(input, parents, fcontext.qcontext); - } - - fcontext.base = result; - } - private void handleFilterExclusions() throws IOException { List excludeTags = freq.domain.excludeTags; @@ -177,11 +154,44 @@ public class FacetProcessor { fcontext.base = fcontext.searcher.getDocSet(qlist); } + private void handleBlockJoin() throws IOException { + if (!(freq.domain.toChildren || freq.domain.toParent)) return; - public Object getResponse() { - return null; + // TODO: avoid query parsing per-bucket somehow... + String parentStr = freq.domain.parents; + Query parentQuery; + try { + QParser parser = QParser.getParser(parentStr, fcontext.req); + parentQuery = parser.getQuery(); + } catch (SyntaxError err) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing block join parent specification: " + parentStr); + } + + BitDocSet parents = fcontext.searcher.getDocSetBits(parentQuery); + DocSet input = fcontext.base; + DocSet result; + + if (freq.domain.toChildren) { + DocSet filt = fcontext.searcher.getDocSetBits( new MatchAllDocsQuery() ); + result = BlockJoin.toChildren(input, parents, filt, fcontext.qcontext); + } else { + result = BlockJoin.toParents(input, parents, fcontext.qcontext); + } + + fcontext.base = result; } + protected void processStats(SimpleOrderedMap bucket, DocSet docs, int docCount) throws IOException { + if (docCount == 0 && !freq.processEmpty || freq.getFacetStats().size() == 0) { + bucket.add("count", docCount); + return; + } + createAccs(docCount, 1); + int collected = collect(docs, 0); + countAcc.incrementCount(0, collected); + assert collected == docCount; + addStats(bucket, 0); + } protected void createAccs(int docCount, int slotCount) throws IOException { accMap = new LinkedHashMap<>(); @@ -198,7 +208,6 @@ public class FacetProcessor { accMap.put(acc.key, acc); } - accs = new SlotAcc[accMap.size()]; int i=0; for (SlotAcc acc : accMap.values()) { @@ -206,63 +215,14 @@ public class FacetProcessor { } } - - protected void resetStats() { + // note: only called by enum/stream prior to collect + void resetStats() { countAcc.reset(); for (SlotAcc acc : accs) { acc.reset(); } } - protected void processStats(SimpleOrderedMap bucket, DocSet docs, int docCount) throws IOException { - if (docCount == 0 && !freq.processEmpty || freq.getFacetStats().size() == 0) { - bucket.add("count", docCount); - return; - } - createAccs(docCount, 1); - int collected = collect(docs, 0); - countAcc.incrementCount(0, collected); - assert collected == docCount; - addStats(bucket, 0); - } - - - protected void processSubs(SimpleOrderedMap response, Query filter, DocSet domain) throws IOException { - - // TODO: what if a zero bucket has a sub-facet with an exclusion that would yield results? - // should we check for domain-altering exclusions, or even ask the sub-facet for - // it's domain and then only skip it if it's 0? - - if (domain == null || domain.size() == 0 && !freq.processEmpty) { - return; - } - - for (Map.Entry sub : freq.getSubFacets().entrySet()) { - // make a new context for each sub-facet since they can change the domain - FacetContext subContext = fcontext.sub(filter, domain); - FacetProcessor subProcessor = sub.getValue().createFacetProcessor(subContext); - if (fcontext.getDebugInfo() != null) { // if fcontext.debugInfo != null, it means rb.debug() == true - FacetDebugInfo fdebug = new FacetDebugInfo(); - subContext.setDebugInfo(fdebug); - fcontext.getDebugInfo().addChild(fdebug); - - fdebug.setReqDescription(sub.getValue().getFacetDescription()); - fdebug.setProcessor(subProcessor.getClass().getSimpleName()); - if (subContext.filter != null) fdebug.setFilter(subContext.filter.toString()); - - final RTimer timer = new RTimer(); - subProcessor.process(); - long timeElapsed = (long) timer.getTime(); - fdebug.setElapse(timeElapsed); - fdebug.putInfoItem("domainSize", (long)subContext.base.size()); - } else { - subProcessor.process(); - } - - response.add( sub.getKey(), subProcessor.getResponse() ); - } - } - int collect(DocSet docs, int slot) throws IOException { int count = 0; SolrIndexSearcher searcher = fcontext.searcher; @@ -310,7 +270,6 @@ public class FacetProcessor { } } - void addStats(SimpleOrderedMap target, int slotNum) throws IOException { int count = countAcc.getCount(slotNum); target.add("count", count); @@ -321,8 +280,7 @@ public class FacetProcessor { } } - - public void fillBucket(SimpleOrderedMap bucket, Query q, DocSet result) throws IOException { + void fillBucket(SimpleOrderedMap bucket, Query q, DocSet result) throws IOException { boolean needDocSet = freq.getFacetStats().size() > 0 || freq.getSubFacets().size() > 0; // TODO: always collect counts or not??? @@ -348,7 +306,7 @@ public class FacetProcessor { } try { - processStats(bucket, result, (int) count); + processStats(bucket, result, count); processSubs(bucket, q, result); } finally { if (result != null) { @@ -358,7 +316,44 @@ public class FacetProcessor { } } - public static DocSet getFieldMissing(SolrIndexSearcher searcher, DocSet docs, String fieldName) throws IOException { + void processSubs(SimpleOrderedMap response, Query filter, DocSet domain) throws IOException { + + // TODO: what if a zero bucket has a sub-facet with an exclusion that would yield results? + // should we check for domain-altering exclusions, or even ask the sub-facet for + // it's domain and then only skip it if it's 0? + + if (domain == null || domain.size() == 0 && !freq.processEmpty) { + return; + } + + for (Map.Entry sub : freq.getSubFacets().entrySet()) { + // make a new context for each sub-facet since they can change the domain + FacetContext subContext = fcontext.sub(filter, domain); + FacetProcessor subProcessor = sub.getValue().createFacetProcessor(subContext); + if (fcontext.getDebugInfo() != null) { // if fcontext.debugInfo != null, it means rb.debug() == true + FacetDebugInfo fdebug = new FacetDebugInfo(); + subContext.setDebugInfo(fdebug); + fcontext.getDebugInfo().addChild(fdebug); + + fdebug.setReqDescription(sub.getValue().getFacetDescription()); + fdebug.setProcessor(subProcessor.getClass().getSimpleName()); + if (subContext.filter != null) fdebug.setFilter(subContext.filter.toString()); + + final RTimer timer = new RTimer(); + subProcessor.process(); + long timeElapsed = (long) timer.getTime(); + fdebug.setElapse(timeElapsed); + fdebug.putInfoItem("domainSize", (long)subContext.base.size()); + } else { + subProcessor.process(); + } + + response.add( sub.getKey(), subProcessor.getResponse() ); + } + } + + @SuppressWarnings("unused") + static DocSet getFieldMissing(SolrIndexSearcher searcher, DocSet docs, String fieldName) throws IOException { SchemaField sf = searcher.getSchema().getField(fieldName); DocSet hasVal = searcher.getDocSet(sf.getType().getRangeQuery(null, sf, null, null, false, false)); DocSet answer = docs.andNot(hasVal); @@ -366,7 +361,7 @@ public class FacetProcessor { return answer; } - public static Query getFieldMissingQuery(SolrIndexSearcher searcher, String fieldName) throws IOException { + static Query getFieldMissingQuery(SolrIndexSearcher searcher, String fieldName) throws IOException { SchemaField sf = searcher.getSchema().getField(fieldName); Query hasVal = sf.getType().getRangeQuery(null, sf, null, null, false, false); BooleanQuery.Builder noVal = new BooleanQuery.Builder(); diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java b/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java index ac6d7f1c680..174b832f006 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java @@ -53,11 +53,6 @@ class FacetQueryProcessor extends FacetProcessor { super(fcontext, freq); } - @Override - public Object getResponse() { - return response; - } - @Override public void process() throws IOException { super.process(); diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java index 8d3d0f5b9a8..1b98de0ed4b 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java @@ -93,11 +93,6 @@ class FacetRangeProcessor extends FacetProcessor { response = getRangeCounts(); } - @Override - public Object getResponse() { - return response; - } - private static class Range { Object label; Comparable low; diff --git a/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java b/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java index aa8f395915e..ad3baf01bc9 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java +++ b/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java @@ -305,7 +305,7 @@ public class UnInvertedField extends DocTermOrds { - private void getCounts(FacetFieldProcessorUIF processor, CountSlotAcc counts) throws IOException { + private void getCounts(FacetFieldProcessorByArrayUIF processor, CountSlotAcc counts) throws IOException { DocSet docs = processor.fcontext.base; int baseSize = docs.size(); int maxDoc = searcher.maxDoc(); @@ -397,7 +397,7 @@ public class UnInvertedField extends DocTermOrds { - public void collectDocs(FacetFieldProcessorUIF processor) throws IOException { + public void collectDocs(FacetFieldProcessorByArrayUIF processor) throws IOException { if (processor.collectAcc==null && processor.allBucketsAcc == null && processor.startTermIndex == 0 && processor.endTermIndex >= numTermsInField) { getCounts(processor, processor.countAcc); return; @@ -408,7 +408,7 @@ public class UnInvertedField extends DocTermOrds { // called from FieldFacetProcessor // TODO: do a callback version that can be specialized! - public void collectDocsGeneric(FacetFieldProcessorUIF processor) throws IOException { + public void collectDocsGeneric(FacetFieldProcessorByArrayUIF processor) throws IOException { use.incrementAndGet(); int startTermIndex = processor.startTermIndex; diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinDocSetFacetComponent.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinDocSetFacetComponent.java index ae334850f69..b8f303491f2 100644 --- a/solr/core/src/java/org/apache/solr/search/join/BlockJoinDocSetFacetComponent.java +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinDocSetFacetComponent.java @@ -22,16 +22,13 @@ import java.util.List; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Collector; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.join.ToParentBlockJoinQuery; import org.apache.solr.common.SolrException; import org.apache.solr.handler.component.ResponseBuilder; -import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.BitDocSet; import org.apache.solr.search.DocSet; import org.apache.solr.search.Filter; @@ -40,11 +37,11 @@ import org.apache.solr.search.facet.BlockJoin; import org.apache.solr.search.join.BlockJoinFieldFacetAccumulator.AggregatableDocIter; /** - * It does the same as BlockJoinFacetComponent, but operates on docsets, - * it should be faster for static mostly indexes. This component doesn't impact - * query result caching, but hits filter cache to retrieve docsets. + * Calculates facets on children documents and aggregates hits by parent documents. + * Enables when child.facet.field parameter specifies a field name for faceting. + * So far it supports string fields only. It requires to search by {@link ToParentBlockJoinQuery}. * */ -public class BlockJoinDocSetFacetComponent extends BlockJoinFacetComponent { +public class BlockJoinDocSetFacetComponent extends BlockJoinFacetComponentSupport { private final String bjqKey = this.getClass().getSimpleName()+".bjq"; @@ -115,27 +112,6 @@ public class BlockJoinDocSetFacetComponent extends BlockJoinFacetComponent { } } - private static final class NoDelegateFacetCollector extends BlockJoinFacetCollector { - { - setDelegate(new Collector() { - - @Override - public boolean needsScores() { - return false; - } - - @Override - public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - return null; - } - }); - } - - private NoDelegateFacetCollector(SolrQueryRequest req) throws IOException { - super(req); - } - } - public BlockJoinDocSetFacetComponent() {} @Override @@ -196,7 +172,7 @@ public class BlockJoinDocSetFacetComponent extends BlockJoinFacetComponent { Filter filter = selectedChildren.getTopFilter(); - final BlockJoinFacetCollector facetCounter = new NoDelegateFacetCollector(rb.req); + final BlockJoinFacetAccsHolder facetCounter = new BlockJoinFacetAccsHolder(rb.req); for (int subIdx = 0; subIdx < leaves.size(); subIdx++) { LeafReaderContext subCtx = leaves.get(subIdx); diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetCollector.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetAccsHolder.java similarity index 75% rename from solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetCollector.java rename to solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetAccsHolder.java index da84d9f7892..bf7f7b2de65 100644 --- a/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetCollector.java +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetAccsHolder.java @@ -17,15 +17,11 @@ package org.apache.solr.search.join; import java.io.IOException; -import java.util.LinkedList; -import java.util.Queue; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.Scorer; import org.apache.lucene.search.join.ToParentBlockJoinQuery.ChildrenMatchesScorer; import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.search.DelegatingCollector; import org.apache.solr.search.join.BlockJoinFieldFacetAccumulator.AggregatableDocIter; import org.apache.solr.search.join.BlockJoinFieldFacetAccumulator.SortedIntsAggDocIterator; @@ -33,14 +29,14 @@ import org.apache.solr.search.join.BlockJoinFieldFacetAccumulator.SortedIntsAggD * For each collected parent document creates matched block, which is a docSet with matched children and parent doc * itself. Then updates each BlockJoinFieldFacetAccumulator with the created matched block. */ -class BlockJoinFacetCollector extends DelegatingCollector { +class BlockJoinFacetAccsHolder { private BlockJoinFieldFacetAccumulator[] blockJoinFieldFacetAccumulators; private boolean firstSegment = true; private ChildrenMatchesScorer blockJoinScorer; private int[] childDocs = new int[0]; - BlockJoinFacetCollector(SolrQueryRequest req) throws IOException { - String[] facetFieldNames = BlockJoinFacetComponent.getChildFacetFields(req); + BlockJoinFacetAccsHolder(SolrQueryRequest req) throws IOException { + String[] facetFieldNames = BlockJoinFacetComponentSupport.getChildFacetFields(req); assert facetFieldNames != null; blockJoinFieldFacetAccumulators = new BlockJoinFieldFacetAccumulator[facetFieldNames.length]; for (int i = 0; i < facetFieldNames.length; i++) { @@ -48,28 +44,7 @@ class BlockJoinFacetCollector extends DelegatingCollector { } } - @Override - public void setScorer(Scorer scorer) throws IOException { - super.setScorer(scorer); - blockJoinScorer = getToParentScorer(scorer, new LinkedList()); - if (blockJoinScorer != null) { - // instruct scorer to keep track of the child docIds for retrieval purposes. - blockJoinScorer.trackPendingChildHits(); - } - } - - private ChildrenMatchesScorer getToParentScorer(Scorer scorer, Queue queue) { - if (scorer == null || scorer instanceof ChildrenMatchesScorer) { - return (ChildrenMatchesScorer) scorer; - } else { - for (Scorer.ChildScorer child : scorer.getChildren()) { - queue.add(child.child); - } - return getToParentScorer(queue.poll(), queue); - } - } - - @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { for (BlockJoinFieldFacetAccumulator blockJoinFieldFacetAccumulator : blockJoinFieldFacetAccumulators) { if(!firstSegment){ @@ -78,21 +53,12 @@ class BlockJoinFacetCollector extends DelegatingCollector { blockJoinFieldFacetAccumulator.setNextReader(context); } firstSegment = false; - super.doSetNextReader(context); } - @Override - public void collect(int doc) throws IOException { - incrementFacets(doc); - super.collect(doc); - } - - @Override public void finish() throws IOException { for (BlockJoinFieldFacetAccumulator blockJoinFieldFacetAccumulator : blockJoinFieldFacetAccumulators) { blockJoinFieldFacetAccumulator.migrateGlobal(); } - super.finish(); } protected void incrementFacets(int parent) throws IOException { diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetComponent.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetComponent.java index 03a33d13e40..16f84cc71e4 100644 --- a/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetComponent.java +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetComponent.java @@ -16,167 +16,8 @@ */ package org.apache.solr.search.join; -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +/** this is just a stub refers to {@link BlockJoinDocSetFacetComponent} to avoid + * changes in configs */ +public class BlockJoinFacetComponent extends BlockJoinDocSetFacetComponent { -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.join.ToParentBlockJoinQuery; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.ShardParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.handler.component.ResponseBuilder; -import org.apache.solr.handler.component.SearchComponent; -import org.apache.solr.handler.component.ShardRequest; -import org.apache.solr.handler.component.ShardResponse; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.search.DelegatingCollector; -import org.apache.solr.search.SolrIndexSearcher; - - -/** - * Calculates facets on children documents and aggregates hits by parent documents. - * Enables when child.facet.field parameter specifies a field name for faceting. - * So far it supports string fields only. It requires to search by {@link ToParentBlockJoinQuery}. - * It disables query result cache but only when it's ebaled for request by child.facet.field parameter - * */ -public class BlockJoinFacetComponent extends SearchComponent { - public static final String CHILD_FACET_FIELD_PARAMETER = "child.facet.field"; - public static final String NO_TO_PARENT_BJQ_MESSAGE = "Block join faceting is allowed with ToParentBlockJoinQuery only"; - public static final String COLLECTOR_CONTEXT_PARAM = "blockJoinFacetCollector"; - - @Override - public void prepare(ResponseBuilder rb) throws IOException { - - if (getChildFacetFields(rb.req) != null) { - validateQuery(rb.getQuery()); - // we count facets only when searching - rb.setFieldFlags(rb.getFieldFlags() | SolrIndexSearcher.NO_CHECK_QCACHE); - if (rb.getFilters() == null) { - rb.setFilters(new LinkedList()); - } - DelegatingCollector blockJoinFacetCollector = new BlockJoinFacetCollector(rb.req); - rb.req.getContext().put(COLLECTOR_CONTEXT_PARAM, blockJoinFacetCollector); - rb.getFilters().add(new BlockJoinFacetFilter(blockJoinFacetCollector)); - } - } - - protected void validateQuery(Query query) { - if (!(query instanceof ToParentBlockJoinQuery)) { - if (query instanceof BooleanQuery) { - List clauses = ((BooleanQuery) query).clauses(); - for (BooleanClause clause : clauses) { - if (clause.getQuery() instanceof ToParentBlockJoinQuery) { - return; - } - } - } - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NO_TO_PARENT_BJQ_MESSAGE); - } - } - - static String[] getChildFacetFields(SolrQueryRequest req) { - return req.getParams().getParams(CHILD_FACET_FIELD_PARAMETER); - } - - @Override - public void process(ResponseBuilder rb) throws IOException { - if (getChildFacetFields(rb.req) != null) { - BlockJoinFacetCollector blockJoinFacetCollector = (BlockJoinFacetCollector) rb.req.getContext().get(COLLECTOR_CONTEXT_PARAM); - assert blockJoinFacetCollector != null; - NamedList output; - if (isShard(rb)) { - // distributed search, put results into own cell in order not to clash with facet component - output = getChildFacetFields(rb.rsp.getValues(), true); - } else { - // normal process, put results into standard response - output = getFacetFieldsList(rb); - } - mergeFacets(output, blockJoinFacetCollector.getFacets()); - } - } - - private boolean isShard(ResponseBuilder rb) { - return "true".equals(rb.req.getParams().get(ShardParams.IS_SHARD)); - } - - private NamedList getChildFacetFields(NamedList responseValues, boolean createIfAbsent) { - return getNamedListFromList(responseValues, "child_facet_fields", createIfAbsent); - } - - private void mergeFacets(NamedList childFacetFields, NamedList shardFacets) { - if (shardFacets != null) { - for (Map.Entry> nextShardFacet : (Iterable>>) shardFacets) { - String fieldName = nextShardFacet.getKey(); - NamedList collectedFacet = (NamedList) childFacetFields.get(fieldName); - NamedList shardFacet = nextShardFacet.getValue(); - if (collectedFacet == null) { - childFacetFields.add(fieldName, shardFacet); - } else { - mergeFacetValues(collectedFacet, shardFacet); - } - } - } - } - - private void mergeFacetValues(NamedList collectedFacetValue, NamedList shardFacetValue) { - for (Map.Entry nextShardValue : shardFacetValue) { - String facetValue = nextShardValue.getKey(); - Integer shardCount = nextShardValue.getValue(); - int indexOfCollectedValue = collectedFacetValue.indexOf(facetValue, 0); - if (indexOfCollectedValue == -1) { - collectedFacetValue.add(facetValue, shardCount); - } else { - int newCount = collectedFacetValue.getVal(indexOfCollectedValue) + shardCount; - collectedFacetValue.setVal(indexOfCollectedValue, newCount); - } - } - } - - private NamedList getNamedListFromList(NamedList parentList, String name, boolean createIfAbsent) { - NamedList result = null; - if (parentList != null) { - result = (NamedList) parentList.get(name); - if (result == null && createIfAbsent) { - result = new NamedList(); - parentList.add(name, result); - } - } - return result; - } - - @Override - public void handleResponses(ResponseBuilder rb, ShardRequest sreq) { - NamedList collectedChildFacetFields = getChildFacetFields(rb.rsp.getValues(), true); - List responses = sreq.responses; - for (ShardResponse shardResponse : responses) { - NamedList shardChildFacetFields = getChildFacetFields(shardResponse.getSolrResponse().getResponse(), false); - mergeFacets(collectedChildFacetFields, shardChildFacetFields); - } - } - - @Override - public void finishStage(ResponseBuilder rb) { - if (rb.stage != ResponseBuilder.STAGE_GET_FIELDS) return; - NamedList childFacetFields = getChildFacetFields(rb.rsp.getValues(), true); - NamedList facetFields = getFacetFieldsList(rb); - for (Map.Entry childFacetField : (Iterable>) childFacetFields) { - facetFields.add(childFacetField.getKey(), childFacetField.getValue()); - } - rb.rsp.getValues().remove("child_facet_fields"); - } - - private NamedList getFacetFieldsList(ResponseBuilder rb) { - NamedList facetCounts = getNamedListFromList(rb.rsp.getValues(), "facet_counts", true); - return getNamedListFromList(facetCounts, "facet_fields", true); - } - - - @Override - public String getDescription() { - return "BlockJoin facet component"; - } } diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetComponentSupport.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetComponentSupport.java new file mode 100644 index 00000000000..85aa7996ed4 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinFacetComponentSupport.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search.join; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.ToParentBlockJoinQuery; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.handler.component.SearchComponent; +import org.apache.solr.handler.component.ShardRequest; +import org.apache.solr.handler.component.ShardResponse; +import org.apache.solr.request.SolrQueryRequest; + +abstract class BlockJoinFacetComponentSupport extends SearchComponent { + public static final String CHILD_FACET_FIELD_PARAMETER = "child.facet.field"; + public static final String NO_TO_PARENT_BJQ_MESSAGE = "Block join faceting is allowed with ToParentBlockJoinQuery only"; + public static final String COLLECTOR_CONTEXT_PARAM = "blockJoinFacetCollector"; + + protected void validateQuery(Query query) { + if (!(query instanceof ToParentBlockJoinQuery)) { + if (query instanceof BooleanQuery) { + List clauses = ((BooleanQuery) query).clauses(); + for (BooleanClause clause : clauses) { + if (clause.getQuery() instanceof ToParentBlockJoinQuery) { + return; + } + } + } + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NO_TO_PARENT_BJQ_MESSAGE); + } + } + + static String[] getChildFacetFields(SolrQueryRequest req) { + return req.getParams().getParams(CHILD_FACET_FIELD_PARAMETER); + } + + @Override + public void process(ResponseBuilder rb) throws IOException { + if (getChildFacetFields(rb.req) != null) { + BlockJoinFacetAccsHolder blockJoinFacetCollector = (BlockJoinFacetAccsHolder) rb.req.getContext().get(COLLECTOR_CONTEXT_PARAM); + assert blockJoinFacetCollector != null; + NamedList output; + if (isShard(rb)) { + // distributed search, put results into own cell in order not to clash with facet component + output = getChildFacetFields(rb.rsp.getValues(), true); + } else { + // normal process, put results into standard response + output = getFacetFieldsList(rb); + } + mergeFacets(output, blockJoinFacetCollector.getFacets()); + } + } + + private boolean isShard(ResponseBuilder rb) { + return "true".equals(rb.req.getParams().get(ShardParams.IS_SHARD)); + } + + private NamedList getChildFacetFields(NamedList responseValues, boolean createIfAbsent) { + return getNamedListFromList(responseValues, "child_facet_fields", createIfAbsent); + } + + private void mergeFacets(NamedList childFacetFields, NamedList shardFacets) { + if (shardFacets != null) { + for (Map.Entry> nextShardFacet : (Iterable>>) shardFacets) { + String fieldName = nextShardFacet.getKey(); + NamedList collectedFacet = (NamedList) childFacetFields.get(fieldName); + NamedList shardFacet = nextShardFacet.getValue(); + if (collectedFacet == null) { + childFacetFields.add(fieldName, shardFacet); + } else { + mergeFacetValues(collectedFacet, shardFacet); + } + } + } + } + + private void mergeFacetValues(NamedList collectedFacetValue, NamedList shardFacetValue) { + for (Map.Entry nextShardValue : shardFacetValue) { + String facetValue = nextShardValue.getKey(); + Integer shardCount = nextShardValue.getValue(); + int indexOfCollectedValue = collectedFacetValue.indexOf(facetValue, 0); + if (indexOfCollectedValue == -1) { + collectedFacetValue.add(facetValue, shardCount); + } else { + int newCount = collectedFacetValue.getVal(indexOfCollectedValue) + shardCount; + collectedFacetValue.setVal(indexOfCollectedValue, newCount); + } + } + } + + private NamedList getNamedListFromList(NamedList parentList, String name, boolean createIfAbsent) { + NamedList result = null; + if (parentList != null) { + result = (NamedList) parentList.get(name); + if (result == null && createIfAbsent) { + result = new NamedList(); + parentList.add(name, result); + } + } + return result; + } + + @Override + public void handleResponses(ResponseBuilder rb, ShardRequest sreq) { + NamedList collectedChildFacetFields = getChildFacetFields(rb.rsp.getValues(), true); + List responses = sreq.responses; + for (ShardResponse shardResponse : responses) { + NamedList shardChildFacetFields = getChildFacetFields(shardResponse.getSolrResponse().getResponse(), false); + mergeFacets(collectedChildFacetFields, shardChildFacetFields); + } + } + + @Override + public void finishStage(ResponseBuilder rb) { + if (rb.stage != ResponseBuilder.STAGE_GET_FIELDS) return; + NamedList childFacetFields = getChildFacetFields(rb.rsp.getValues(), true); + NamedList facetFields = getFacetFieldsList(rb); + for (Map.Entry childFacetField : (Iterable>) childFacetFields) { + facetFields.add(childFacetField.getKey(), childFacetField.getValue()); + } + rb.rsp.getValues().remove("child_facet_fields"); + } + + private NamedList getFacetFieldsList(ResponseBuilder rb) { + NamedList facetCounts = getNamedListFromList(rb.rsp.getValues(), "facet_counts", true); + return getNamedListFromList(facetCounts, "facet_fields", true); + } + + + @Override + public String getDescription() { + return "BlockJoin facet component"; + } +} diff --git a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java index 46b927efd97..0f85feb13c9 100644 --- a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java +++ b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.regex.Pattern; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.mlt.MoreLikeThis; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -30,7 +31,6 @@ import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrException; import org.apache.solr.common.StringUtils; diff --git a/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java index 962f452e918..da3a4874681 100644 --- a/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java +++ b/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java @@ -16,6 +16,7 @@ */ package org.apache.solr.search.mlt; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.mlt.MoreLikeThis; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -25,7 +26,6 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.solr.common.SolrException; import org.apache.solr.common.StringUtils; import org.apache.solr.common.params.SolrParams; diff --git a/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java b/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java index 7ef495618e2..be08a603b40 100644 --- a/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java +++ b/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java @@ -28,10 +28,10 @@ import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.RamUsageEstimator; @@ -161,8 +161,8 @@ interface FieldCache { }; /** - * A parser instance for int values encoded by {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed - * via {@link org.apache.lucene.document.LegacyIntField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * A parser instance for int values encoded by {@link org.apache.lucene.legacy.LegacyNumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.legacy.LegacyIntField}/{@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * @deprecated Index with points and use {@link #INT_POINT_PARSER} instead. */ @Deprecated @@ -184,8 +184,8 @@ interface FieldCache { }; /** - * A parser instance for float values encoded with {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed - * via {@link org.apache.lucene.document.LegacyFloatField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * A parser instance for float values encoded with {@link org.apache.lucene.legacy.LegacyNumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.legacy.LegacyFloatField}/{@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * @deprecated Index with points and use {@link #FLOAT_POINT_PARSER} instead. */ @Deprecated @@ -209,8 +209,8 @@ interface FieldCache { }; /** - * A parser instance for long values encoded by {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed - * via {@link org.apache.lucene.document.LegacyLongField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * A parser instance for long values encoded by {@link org.apache.lucene.legacy.LegacyNumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.legacy.LegacyLongField}/{@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * @deprecated Index with points and use {@link #LONG_POINT_PARSER} instead. */ @Deprecated @@ -231,8 +231,8 @@ interface FieldCache { }; /** - * A parser instance for double values encoded with {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed - * via {@link org.apache.lucene.document.LegacyDoubleField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * A parser instance for double values encoded with {@link org.apache.lucene.legacy.LegacyNumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.legacy.LegacyDoubleField}/{@link org.apache.lucene.legacy.LegacyNumericTokenStream}. * @deprecated Index with points and use {@link #DOUBLE_POINT_PARSER} instead. */ @Deprecated @@ -279,7 +279,7 @@ interface FieldCache { * @param parser * Computes long for string values. May be {@code null} if the * requested field was indexed as {@link NumericDocValuesField} or - * {@link org.apache.lucene.document.LegacyLongField}. + * {@link org.apache.lucene.legacy.LegacyLongField}. * @param setDocsWithField * If true then {@link #getDocsWithField} will also be computed and * stored in the FieldCache. diff --git a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java index 4450cbb7d86..42b2f7628ba 100644 --- a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java +++ b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java @@ -86,7 +86,7 @@ public class UninvertingReader extends FilterLeafReader { */ DOUBLE_POINT, /** - * Single-valued Integer, (e.g. indexed with {@link org.apache.lucene.document.LegacyIntField}) + * Single-valued Integer, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyIntField}) *

    * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. @@ -95,7 +95,7 @@ public class UninvertingReader extends FilterLeafReader { @Deprecated LEGACY_INTEGER, /** - * Single-valued Long, (e.g. indexed with {@link org.apache.lucene.document.LegacyLongField}) + * Single-valued Long, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyLongField}) *

    * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. @@ -104,7 +104,7 @@ public class UninvertingReader extends FilterLeafReader { @Deprecated LEGACY_LONG, /** - * Single-valued Float, (e.g. indexed with {@link org.apache.lucene.document.LegacyFloatField}) + * Single-valued Float, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyFloatField}) *

    * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. @@ -113,7 +113,7 @@ public class UninvertingReader extends FilterLeafReader { @Deprecated LEGACY_FLOAT, /** - * Single-valued Double, (e.g. indexed with {@link org.apache.lucene.document.LegacyDoubleField}) + * Single-valued Double, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyDoubleField}) *

    * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. @@ -143,28 +143,28 @@ public class UninvertingReader extends FilterLeafReader { */ SORTED_SET_BINARY, /** - * Multi-valued Integer, (e.g. indexed with {@link org.apache.lucene.document.LegacyIntField}) + * Multi-valued Integer, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyIntField}) *

    * Fields with this type act as if they were indexed with * {@link SortedSetDocValuesField}. */ SORTED_SET_INTEGER, /** - * Multi-valued Float, (e.g. indexed with {@link org.apache.lucene.document.LegacyFloatField}) + * Multi-valued Float, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyFloatField}) *

    * Fields with this type act as if they were indexed with * {@link SortedSetDocValuesField}. */ SORTED_SET_FLOAT, /** - * Multi-valued Long, (e.g. indexed with {@link org.apache.lucene.document.LegacyLongField}) + * Multi-valued Long, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyLongField}) *

    * Fields with this type act as if they were indexed with * {@link SortedSetDocValuesField}. */ SORTED_SET_LONG, /** - * Multi-valued Double, (e.g. indexed with {@link org.apache.lucene.document.LegacyDoubleField}) + * Multi-valued Double, (e.g. indexed with {@link org.apache.lucene.legacy.LegacyDoubleField}) *

    * Fields with this type act as if they were indexed with * {@link SortedSetDocValuesField}. diff --git a/solr/core/src/java/org/apache/solr/update/TransactionLog.java b/solr/core/src/java/org/apache/solr/update/TransactionLog.java index 673d6832511..f7213edbb78 100644 --- a/solr/core/src/java/org/apache/solr/update/TransactionLog.java +++ b/solr/core/src/java/org/apache/solr/update/TransactionLog.java @@ -87,7 +87,7 @@ public class TransactionLog implements Closeable { int snapshot_numRecords; // write a BytesRef as a byte array - JavaBinCodec.ObjectResolver resolver = new JavaBinCodec.ObjectResolver() { + static final JavaBinCodec.ObjectResolver resolver = new JavaBinCodec.ObjectResolver() { @Override public Object resolve(Object o, JavaBinCodec codec) throws IOException { if (o instanceof BytesRef) { diff --git a/solr/core/src/java/org/apache/solr/update/VersionInfo.java b/solr/core/src/java/org/apache/solr/update/VersionInfo.java index bee30f500d8..0003c24358b 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionInfo.java +++ b/solr/core/src/java/org/apache/solr/update/VersionInfo.java @@ -25,12 +25,12 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Terms; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.BitUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.index.SlowCompositeReaderWrapper; diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java new file mode 100644 index 00000000000..8d2f6f25a78 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; + +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.common.util.StrUtils; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DeleteNodeTest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(6) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) + .configure(); + } + + protected String getSolrXml() { + return "solr.xml"; + } + + @Test + public void test() throws Exception { + cluster.waitForAllNodes(5000); + CloudSolrClient cloudClient = cluster.getSolrClient(); + String coll = "deletenodetest_coll"; + Set liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + Collections.shuffle(l, random()); + CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 5, 2); + create.setCreateNodeSet(StrUtils.join(l, ',')).setMaxShardsPerNode(3); + cloudClient.request(create); + String node2bdecommissioned = l.get(0); + new CollectionAdminRequest.DeleteNode(node2bdecommissioned).processAsync("003", cloudClient); + CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("003"); + boolean success = false; + for (int i = 0; i < 200; i++) { + CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); + if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) { + success = true; + break; + } + assertFalse(rsp.getRequestStatus() == RequestStatusState.FAILED); + Thread.sleep(50); + } + assertTrue(success); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java new file mode 100644 index 00000000000..1c7575d5225 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; + +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.common.util.StrUtils; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReplaceNodeTest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(6) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) + .configure(); + } + + protected String getSolrXml() { + return "solr.xml"; + } + + @Test + public void test() throws Exception { + cluster.waitForAllNodes(5000); + String coll = "replacenodetest_coll"; + log.info("total_jettys: " + cluster.getJettySolrRunners().size()); + + CloudSolrClient cloudClient = cluster.getSolrClient(); + Set liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + Collections.shuffle(l, random()); + String emptyNode = l.remove(0); + String node2bdecommissioned = l.get(0); + CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 5, 2); + create.setCreateNodeSet(StrUtils.join(l, ',')).setMaxShardsPerNode(3); + cloudClient.request(create); + log.info("excluded_node : {} ", emptyNode); + new CollectionAdminRequest.ReplaceNode(node2bdecommissioned, emptyNode).processAsync("000", cloudClient); + CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("000"); + boolean success = false; + for (int i = 0; i < 200; i++) { + CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); + if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) { + success = true; + break; + } + assertFalse(rsp.getRequestStatus() == RequestStatusState.FAILED); + Thread.sleep(50); + } + assertTrue(success); + try (HttpSolrClient coreclient = getHttpSolrClient(cloudClient.getZkStateReader().getBaseUrlForNodeName(node2bdecommissioned))) { + CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient); + assertTrue(status.getCoreStatus().size() == 0); + } + + //let's do it back + new CollectionAdminRequest.ReplaceNode(emptyNode, node2bdecommissioned).setParallel(Boolean.TRUE).processAsync("001", cloudClient); + requestStatus = CollectionAdminRequest.requestStatus("001"); + + for (int i = 0; i < 200; i++) { + CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); + if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) { + success = true; + break; + } + assertFalse(rsp.getRequestStatus() == RequestStatusState.FAILED); + Thread.sleep(50); + } + assertTrue(success); + try (HttpSolrClient coreclient = getHttpSolrClient(cloudClient.getZkStateReader().getBaseUrlForNodeName(emptyNode))) { + CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient); + assertTrue(status.getCoreStatus().size() == 0); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java index 41e32dd697f..c182495e91a 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java @@ -22,6 +22,7 @@ import java.io.StringReader; import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -35,8 +36,12 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.DumpRequestHandler; import org.apache.solr.handler.TestBlobHandler; import org.apache.solr.handler.TestSolrConfigHandlerConcurrent; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrCache; import org.apache.solr.util.RestTestBase; import org.apache.solr.util.RestTestHarness; import org.eclipse.jetty.servlet.ServletHolder; @@ -449,7 +454,56 @@ public class TestSolrConfigHandler extends RestTestBase { assertEquals(2, initArgs.size()); assertTrue(((Map)initArgs.get(0)).containsKey("suggester")); assertTrue(((Map)initArgs.get(1)).containsKey("suggester")); - System.out.println(map); + + payload = "{\n" + + "'add-requesthandler' : { 'name' : '/dump101', 'class': " + + "'" + CacheTest.class.getName() + "' " + + ", 'startup' : 'lazy'}\n" + + "}"; + runConfigCommand(writeHarness, "/config?wt=json", payload); + + testForResponseElement(writeHarness, + testServerBaseUrl, + "/config/overlay?wt=json", + cloudSolrClient, + Arrays.asList("overlay", "requestHandler", "/dump101", "startup"), + "lazy", + 10); + + payload = "{\n" + + "'add-cache' : {name:'lfuCacheDecayFalse', class:'solr.search.LFUCache', size:10 ,initialSize:9 , timeDecay:false }," + + "'add-cache' : {name: 'perSegFilter', class: 'solr.search.LRUCache', size:10, initialSize:0 , autowarmCount:10}}"; + runConfigCommand(writeHarness, "/config?wt=json", payload); + + map = testForResponseElement(writeHarness, + testServerBaseUrl, + "/config/overlay?wt=json", + cloudSolrClient, + Arrays.asList("overlay", "cache", "lfuCacheDecayFalse", "class"), + "solr.search.LFUCache", + 10); + assertEquals("solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of("overlay", "cache", "perSegFilter", "class"))); + + map = getRespMap("/dump101?cacheNames=lfuCacheDecayFalse&cacheNames=perSegFilter&wt=json", writeHarness); + assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of( "caches", "perSegFilter"))); + assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LFUCache",getObjectByPath(map, true, ImmutableList.of( "caches", "lfuCacheDecayFalse"))); + + } + + public static class CacheTest extends DumpRequestHandler { + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { + super.handleRequestBody(req, rsp); + String[] caches = req.getParams().getParams("cacheNames"); + if(caches != null && caches.length>0){ + HashMap m = new HashMap(); + rsp.add("caches", m); + for (String c : caches) { + SolrCache cache = req.getSearcher().getCache(c); + if(cache != null) m.put(c, cache.getClass().getName()); + } + } + } } public static Map testForResponseElement(RestTestHarness harness, diff --git a/solr/core/src/test/org/apache/solr/search/TestMaxScoreQueryParser.java b/solr/core/src/test/org/apache/solr/search/TestMaxScoreQueryParser.java index 32e3eb207c7..6059528d21f 100644 --- a/solr/core/src/test/org/apache/solr/search/TestMaxScoreQueryParser.java +++ b/solr/core/src/test/org/apache/solr/search/TestMaxScoreQueryParser.java @@ -17,6 +17,7 @@ package org.apache.solr.search; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; import org.apache.lucene.search.*; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.util.AbstractSolrTestCase; diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java index 93369bec056..7b5a561168b 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java @@ -47,8 +47,8 @@ public class TestJsonFacets extends SolrTestCaseHS { @BeforeClass public static void beforeTests() throws Exception { JSONTestUtil.failRepeatedKeys = true; - origTableSize = FacetFieldProcessorNumeric.MAXIMUM_STARTING_TABLE_SIZE; - FacetFieldProcessorNumeric.MAXIMUM_STARTING_TABLE_SIZE=2; // stress test resizing + origTableSize = FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE; + FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE=2; // stress test resizing initCore("solrconfig-tlog.xml","schema_latest.xml"); } @@ -61,7 +61,7 @@ public class TestJsonFacets extends SolrTestCaseHS { @AfterClass public static void afterTests() throws Exception { JSONTestUtil.failRepeatedKeys = false; - FacetFieldProcessorNumeric.MAXIMUM_STARTING_TABLE_SIZE=origTableSize; + FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE=origTableSize; if (servers != null) { servers.stop(); servers = null; @@ -349,11 +349,11 @@ public class TestJsonFacets extends SolrTestCaseHS { doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sd", "cat_s","cat_sd", "where_s","where_sd", "num_d","num_dd", "num_i","num_id", "num_is","num_lds", "num_fs","num_dds", "super_s","super_sd", "val_b","val_b", "date","date_dtd", "sparse_s","sparse_sd" ,"multi_ss","multi_sds") ); // multi-valued docvalues - FacetFieldProcessorDV.unwrap_singleValued_multiDv = false; // better multi-valued coverage + FacetFieldProcessorByArrayDV.unwrap_singleValued_multiDv = false; // better multi-valued coverage doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "num_is","num_ids", "num_fs","num_fds", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") ); // multi-valued docvalues - FacetFieldProcessorDV.unwrap_singleValued_multiDv = true; + FacetFieldProcessorByArrayDV.unwrap_singleValued_multiDv = true; doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "num_is","num_ids", "num_fs","num_fds", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") ); } diff --git a/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java b/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java index 785ee55883a..b3a70ae6d39 100644 --- a/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java +++ b/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java @@ -21,8 +21,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.TextField; @@ -31,6 +29,8 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.FloatFieldSource; diff --git a/solr/core/src/test/org/apache/solr/search/join/BlockJoinFacetSimpleTest.java b/solr/core/src/test/org/apache/solr/search/join/BlockJoinFacetSimpleTest.java index a34e1d9c295..5e610cfe22e 100644 --- a/solr/core/src/test/org/apache/solr/search/join/BlockJoinFacetSimpleTest.java +++ b/solr/core/src/test/org/apache/solr/search/join/BlockJoinFacetSimpleTest.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.List; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.request.SolrQueryRequest; import org.junit.BeforeClass; import org.junit.Test; @@ -94,4 +95,27 @@ public class BlockJoinFacetSimpleTest extends SolrTestCaseJ4 { } } + @Test + public void testParentLevelFQExclusion() { + SolrQueryRequest req = req( + "qt", handler, + "q", "{!parent which=type_s:parent}+SIZE_s:XL", + "fq", "{!term f=BRAND_s tag=rbrand}Nike", + "facet", "true", + "facet.field", "BRAND_s", + "child.facet.field", "COLOR_s"); + assertQ("no exclusion, brand facet got only one Nike",req, "//*[@numFound='" + 1 + "']", + "count(//lst[@name='BRAND_s']/int[.='1'])=1"); + + assertQ("nike filter is excluded, expecting both brand in facet",req( + "qt", handler, + "q", "{!parent which=type_s:parent}+SIZE_s:XL", + "fq", "{!term f=BRAND_s tag=rbrand}Nike", + "facet", "true", + "facet.field", "{!ex=rbrand}BRAND_s", + "child.facet.field", "COLOR_s"), + "//*[@numFound='" + 1 + "']", + "count(//lst[@name='BRAND_s']/int[.='1'])=2"); + + } } diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java index f1627a6ee35..46339a75ba0 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java @@ -28,8 +28,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValues; @@ -45,10 +43,12 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.TestUtil; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java index d54d5792447..a5958f9af60 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java @@ -21,14 +21,14 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyDoubleField; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.MultiReader; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.index.SlowCompositeReaderWrapper; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSort.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSort.java index 34d92379b39..d53f610f653 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSort.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSort.java @@ -24,13 +24,9 @@ import java.util.Map; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; @@ -41,6 +37,10 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java b/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java index 1192f4b77c8..5220460d811 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java @@ -28,10 +28,6 @@ import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyDoubleField; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; @@ -45,11 +41,15 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.legacy.LegacyDoubleField; +import org.apache.lucene.legacy.LegacyFloatField; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.apache.solr.index.SlowCompositeReaderWrapper; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms32.java b/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms32.java index 2b861adb43d..6fed73b829f 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms32.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms32.java @@ -21,12 +21,12 @@ import java.util.Map; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; @@ -62,17 +62,17 @@ public class TestNumericTerms32 extends LuceneTestCase { .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) .setMergePolicy(newLogMergePolicy())); - final FieldType storedInt = new FieldType(LegacyIntField.TYPE_NOT_STORED); + final LegacyFieldType storedInt = new LegacyFieldType(LegacyIntField.TYPE_NOT_STORED); storedInt.setStored(true); storedInt.freeze(); - final FieldType storedInt8 = new FieldType(storedInt); + final LegacyFieldType storedInt8 = new LegacyFieldType(storedInt); storedInt8.setNumericPrecisionStep(8); - final FieldType storedInt4 = new FieldType(storedInt); + final LegacyFieldType storedInt4 = new LegacyFieldType(storedInt); storedInt4.setNumericPrecisionStep(4); - final FieldType storedInt2 = new FieldType(storedInt); + final LegacyFieldType storedInt2 = new LegacyFieldType(storedInt); storedInt2.setNumericPrecisionStep(2); LegacyIntField diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms64.java b/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms64.java index 4da8be98c0f..2f341b708c2 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms64.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms64.java @@ -21,12 +21,12 @@ import java.util.Map; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericRangeQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; @@ -62,20 +62,20 @@ public class TestNumericTerms64 extends LuceneTestCase { .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) .setMergePolicy(newLogMergePolicy())); - final FieldType storedLong = new FieldType(LegacyLongField.TYPE_NOT_STORED); + final LegacyFieldType storedLong = new LegacyFieldType(LegacyLongField.TYPE_NOT_STORED); storedLong.setStored(true); storedLong.freeze(); - final FieldType storedLong8 = new FieldType(storedLong); + final LegacyFieldType storedLong8 = new LegacyFieldType(storedLong); storedLong8.setNumericPrecisionStep(8); - final FieldType storedLong4 = new FieldType(storedLong); + final LegacyFieldType storedLong4 = new LegacyFieldType(storedLong); storedLong4.setNumericPrecisionStep(4); - final FieldType storedLong6 = new FieldType(storedLong); + final LegacyFieldType storedLong6 = new LegacyFieldType(storedLong); storedLong6.setNumericPrecisionStep(6); - final FieldType storedLong2 = new FieldType(storedLong); + final LegacyFieldType storedLong2 = new LegacyFieldType(storedLong); storedLong2.setNumericPrecisionStep(2); LegacyLongField diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java index 2ecc63e6d11..c0188e4358c 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java @@ -28,10 +28,7 @@ import java.util.Set; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; @@ -43,9 +40,12 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.legacy.LegacyFieldType; +import org.apache.lucene.legacy.LegacyIntField; +import org.apache.lucene.legacy.LegacyLongField; +import org.apache.lucene.legacy.LegacyNumericUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.apache.solr.index.SlowCompositeReaderWrapper; @@ -224,7 +224,7 @@ public class TestUninvertingReader extends LuceneTestCase { final Directory dir = newDirectory(); final IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null)); - final FieldType NO_TRIE_TYPE = new FieldType(LegacyIntField.TYPE_NOT_STORED); + final LegacyFieldType NO_TRIE_TYPE = new LegacyFieldType(LegacyIntField.TYPE_NOT_STORED); NO_TRIE_TYPE.setNumericPrecisionStep(Integer.MAX_VALUE); final Map UNINVERT_MAP = new LinkedHashMap(); diff --git a/solr/site/SYSTEM_REQUIREMENTS.mdtext b/solr/site/SYSTEM_REQUIREMENTS.mdtext index 2c0fa87981e..bd41bb217b0 100644 --- a/solr/site/SYSTEM_REQUIREMENTS.mdtext +++ b/solr/site/SYSTEM_REQUIREMENTS.mdtext @@ -1,6 +1,6 @@ # System Requirements -Apache Solr runs of Java 8 or greater. +Apache Solr runs on Java 8 or greater. It is also recommended to always use the latest update version of your Java VM, because bugs may affect Solr. An overview of known JVM bugs diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java index eaf593e4591..c65a3283897 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java @@ -358,7 +358,6 @@ public class LBHttpSolrClient extends SolrClient { } continue; } - rsp.server = serverStr; try { MDC.put("LBHttpSolrClient.url", serverStr); HttpSolrClient client = makeSolrClient(serverStr); @@ -410,6 +409,7 @@ public class LBHttpSolrClient extends SolrClient { boolean isZombie, String zombieKey) throws SolrServerException, IOException { Exception ex = null; try { + rsp.server = client.getBaseURL(); rsp.rsp = client.request(req.getRequest(), (String) null); if (isZombie) { zombieServers.remove(zombieKey); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index 7bc9e4fd931..0a0a191ab1d 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -112,7 +112,7 @@ public abstract class CollectionAdminRequest * @deprecated Use {@link #processAsync(String, SolrClient)} or {@link #processAsync(SolrClient)} */ @Deprecated - public abstract AsyncCollectionAdminRequest setAsyncId(String id); + public AsyncCollectionAdminRequest setAsyncId(String id){return this;}; /** * Process this request asynchronously, generating and returning a request id @@ -491,6 +491,56 @@ public abstract class CollectionAdminRequest } } + public static class DeleteNode extends AsyncCollectionAdminRequest { + String node; + + /** + * @param node The node to be deleted + */ + public DeleteNode(String node) { + super(CollectionAction.DELETENODE); + this.node = node; + } + @Override + public SolrParams getParams() { + ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); + params.set("node", node); + return params; + } + + + } + + public static class ReplaceNode extends AsyncCollectionAdminRequest { + String source, target; + Boolean parallel; + + /** + * @param source node to be cleaned up + * @param target node where the new replicas are to be created + */ + public ReplaceNode(String source, String target) { + super(CollectionAction.REPLACENODE); + this.source = source; + this.target = target; + } + + public ReplaceNode setParallel(Boolean flag) { + this.parallel = flag; + return this; + } + + @Override + public SolrParams getParams() { + ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); + params.set("source", source); + params.set("target", target); + if (parallel != null) params.set("parallel", parallel.toString()); + return params; + } + + } + /* * Returns a RebalanceLeaders object to rebalance leaders for a collection */ diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java index 320ad132583..94a673e9941 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java @@ -20,6 +20,7 @@ import org.apache.solr.common.util.Utils; import org.noggit.JSONUtil; import org.noggit.JSONWriter; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -40,6 +41,17 @@ public class ZkNodeProps implements JSONWriter.Writable { // Always wrapping introduces a memory leak. } + public ZkNodeProps plus(String key , Object val) { + return plus(Collections.singletonMap(key,val)); + } + + public ZkNodeProps plus(Map newVals) { + LinkedHashMap copy = new LinkedHashMap<>(propMap); + if (newVals == null || newVals.isEmpty()) return new ZkNodeProps(copy); + copy.putAll(newVals); + return new ZkNodeProps(copy); + } + /** * Constructor that populates the from array of Strings in form key1, value1, diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index 9df4a76aa8e..a3de324e0a7 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -147,7 +147,7 @@ public class ZkStateReader implements Closeable { private class CollectionWatch { int coreRefCount = 0; - Set stateWatchers = new HashSet<>(); + Set stateWatchers = ConcurrentHashMap.newKeySet(); public boolean canBeRemoved() { return coreRefCount + stateWatchers.size() == 0; @@ -1273,10 +1273,14 @@ public class ZkStateReader implements Closeable { /* package-private for testing */ Set getStateWatchers(String collection) { - CollectionWatch watch = collectionWatches.get(collection); - if (watch == null) - return null; - return new HashSet<>(watch.stateWatchers); + final Set watchers = new HashSet<>(); + collectionWatches.compute(collection, (k, v) -> { + if (v != null) { + watchers.addAll(v.stateWatchers); + } + return v; + }); + return watchers; } // returns true if the state has changed diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java index e38ab4fbb11..f10f0895b2e 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java @@ -96,6 +96,9 @@ public interface CollectionParams { // but the overseer is aware of these tasks MOCK_COLL_TASK(false, LockLevel.COLLECTION), MOCK_SHARD_TASK(false, LockLevel.SHARD), + //TODO when we have a node level lock use it here + REPLACENODE(true, LockLevel.NONE), + DELETENODE(true, LockLevel.NONE), MOCK_REPLICA_TASK(false, LockLevel.REPLICA) ; public final boolean isWrite; diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java b/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java index d959aa83c9b..fca0e35728c 100644 --- a/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java +++ b/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java @@ -19,8 +19,9 @@ package org.apache.solr.common.cloud; import java.lang.invoke.MethodHandles; import java.util.HashMap; -import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -81,6 +82,31 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase { }); } + private static void waitFor(String message, long timeout, TimeUnit unit, Callable predicate) + throws InterruptedException, ExecutionException { + Future future = executor.submit(() -> { + try { + while (true) { + if (predicate.call()) + return true; + TimeUnit.MILLISECONDS.sleep(10); + } + } + catch (InterruptedException e) { + return false; + } + }); + try { + if (future.get(timeout, unit) == true) { + return; + } + } + catch (TimeoutException e) { + // pass failure message on + } + future.cancel(true); + fail(message); + } @Test public void testSimpleCollectionWatch() throws Exception { @@ -113,9 +139,8 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase { cluster.stopJettySolrRunner(random().nextInt(cluster.getJettySolrRunners().size())); assertTrue("CollectionStateWatcher was never notified of cluster change", latch.await(MAX_WAIT_TIMEOUT, TimeUnit.SECONDS)); - Set watchers = client.getZkStateReader().getStateWatchers("testcollection"); - assertTrue("CollectionStateWatcher wasn't cleared after completion", - watchers == null || watchers.size() == 0); + waitFor("CollectionStateWatcher wasn't cleared after completion", 1, TimeUnit.SECONDS, + () -> client.getZkStateReader().getStateWatchers("testcollection").isEmpty()); } @@ -144,8 +169,8 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase { assertTrue("CollectionStateWatcher isn't called when registering for already-watched collection", latch.await(MAX_WAIT_TIMEOUT, TimeUnit.SECONDS)); - assertEquals("CollectionStateWatcher should be removed", - 1, client.getZkStateReader().getStateWatchers("currentstate").size()); + waitFor("CollectionStateWatcher should be removed", 1, TimeUnit.SECONDS, + () -> client.getZkStateReader().getStateWatchers("currentstate").size() == 1); } @Test @@ -189,9 +214,8 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase { expectThrows(TimeoutException.class, () -> { client.waitForState("nosuchcollection", 1, TimeUnit.SECONDS, ((liveNodes, collectionState) -> false)); }); - Set watchers = client.getZkStateReader().getStateWatchers("nosuchcollection"); - assertTrue("Watchers for collection should be removed after timeout", - watchers == null || watchers.size() == 0); + waitFor("Watchers for collection should be removed after timeout", 1, TimeUnit.SECONDS, + () -> client.getZkStateReader().getStateWatchers("nosuchcollection").isEmpty()); } @@ -229,18 +253,17 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase { } @Test - public void testWatcherIsRemovedAfterTimeout() { + public void testWatcherIsRemovedAfterTimeout() throws Exception { CloudSolrClient client = cluster.getSolrClient(); assertTrue("There should be no watchers for a non-existent collection!", - client.getZkStateReader().getStateWatchers("no-such-collection") == null); + client.getZkStateReader().getStateWatchers("no-such-collection").isEmpty()); expectThrows(TimeoutException.class, () -> { client.waitForState("no-such-collection", 10, TimeUnit.MILLISECONDS, (n, c) -> DocCollection.isFullyActive(n, c, 1, 1)); }); - Set watchers = client.getZkStateReader().getStateWatchers("no-such-collection"); - assertTrue("Watchers for collection should be removed after timeout", - watchers == null || watchers.size() == 0); + waitFor("Watchers for collection should be removed after timeout", 1, TimeUnit.SECONDS, + () -> client.getZkStateReader().getStateWatchers("no-such-collection").isEmpty()); } diff --git a/solr/webapp/web/css/angular/index.css b/solr/webapp/web/css/angular/index.css index 5b77a15a866..e07b8d62686 100644 --- a/solr/webapp/web/css/angular/index.css +++ b/solr/webapp/web/css/angular/index.css @@ -110,6 +110,17 @@ limitations under the License. width: 40%; } +#content #index .data +{ + padding-bottom: 12px; + overflow: hidden; +} + +#content #index .data:hover +{ + overflow-x: auto; +} + #content #index .data li { padding-top: 3px; @@ -127,7 +138,6 @@ limitations under the License. { float: right; text-overflow: ellipsis; - overflow: hidden; white-space: nowrap; width: 80% } diff --git a/solr/webapp/web/js/angular/controllers/collections.js b/solr/webapp/web/js/angular/controllers/collections.js index e6229057a1f..2bd6ab601ba 100644 --- a/solr/webapp/web/js/angular/controllers/collections.js +++ b/solr/webapp/web/js/angular/controllers/collections.js @@ -111,8 +111,11 @@ solrAdminApp.controller('CollectionsController', } $scope.createAlias = function() { - var collections = $scope.aliasCollections.join(","); - Collections.createAlias({name: $scope.aliasToCreate, collections: collections}, function(data) { + var collections = []; + for (var i in $scope.aliasCollections) { + collections.push($scope.aliasCollections[i].name); + } + Collections.createAlias({name: $scope.aliasToCreate, collections: collections.join(",")}, function(data) { $scope.hideAll(); }); } diff --git a/solr/webapp/web/js/angular/controllers/cores.js b/solr/webapp/web/js/angular/controllers/cores.js index 347dbf4c1f8..d135395d8f2 100644 --- a/solr/webapp/web/js/angular/controllers/cores.js +++ b/solr/webapp/web/js/angular/controllers/cores.js @@ -17,7 +17,7 @@ // @todo test optimize (delete stuff, watch button appear, test button/form) solrAdminApp.controller('CoreAdminController', - function($scope, $routeParams, $location, $timeout, Cores, Update, Constants){ + function($scope, $routeParams, $location, $timeout, $route, Cores, Update, Constants){ $scope.resetMenu("cores", Constants.IS_ROOT_PAGE); $scope.selectedCore = $routeParams.corename; // use 'corename' not 'core' to distinguish from /solr/:core/ $scope.refresh = function() { @@ -129,15 +129,15 @@ solrAdminApp.controller('CoreAdminController', }; $scope.swapCores = function() { - if ($scope.swapOther) { - $swapMessage = "Please select a core to swap with"; + if (!$scope.swapOther) { + $scope.swapMessage = "Please select a core to swap with"; } else if ($scope.swapOther == $scope.selectedCore) { - $swapMessage = "Cannot swap with the same core"; + $scope.swapMessage = "Cannot swap with the same core"; } else { Cores.swap({core: $scope.selectedCore, other: $scope.swapOther}, function(data) { $location.path("/~cores/" + $scope.swapOther); delete $scope.swapOther; - $scope.cancelSwap(); + $scope.cancelSwapCores(); }); } }; diff --git a/solr/webapp/web/js/angular/controllers/dataimport.js b/solr/webapp/web/js/angular/controllers/dataimport.js index 9ca723985da..d8fbc4fea1c 100644 --- a/solr/webapp/web/js/angular/controllers/dataimport.js +++ b/solr/webapp/web/js/angular/controllers/dataimport.js @@ -39,28 +39,35 @@ solrAdminApp.controller('DataImportController', } }); - DataImport.config({core: $routeParams.core}, function (data) { - try { - var xml = $.parseXML(data.config); - } catch (err) { - $scope.hasHandlers = false; - return; - } - $scope.config = data.config; - $scope.entities = []; - $('document > entity', xml).each(function (i, element) { - $scope.entities.push($(element).attr('name')); + $scope.handler = $routeParams.handler; + if ($scope.handler && $scope.handler[0]=="/") { + $scope.handler = $scope.handler.substr(1); + } + if ($scope.handler) { + DataImport.config({core: $routeParams.core, name: $scope.handler}, function (data) { + try { + $scope.config = data.config; + var xml = $.parseXML(data.config); + $scope.entities = []; + $('document > entity', xml).each(function (i, element) { + $scope.entities.push($(element).attr('name')); + }); + $scope.refreshStatus(); + } catch (err) { + console.log(err); + } }); - }); - + } $scope.lastUpdate = "unknown"; $scope.lastUpdateUTC = ""; - - $scope.refreshStatus(); }; $scope.toggleDebug = function () { $scope.isDebugMode = !$scope.isDebugMode; + if ($scope.isDebugMode) { + // also enable Debug checkbox + $scope.form.showDebug = true; + } $scope.showConfiguration = true; } @@ -77,7 +84,7 @@ solrAdminApp.controller('DataImportController', } $scope.reload = function () { - DataImport.reload({core: $routeParams.core}, function () { + DataImport.reload({core: $routeParams.core, name: $scope.handler}, function () { $scope.reloaded = true; $timeout(function () { $scope.reloaded = false; @@ -100,7 +107,13 @@ solrAdminApp.controller('DataImportController', $scope.submit = function () { var params = {}; for (var key in $scope.form) { - params[key] = $scope.form[key]; + if (key == "showDebug") { + if ($scope.form.showDebug) { + params["debug"] = true; + } + } else { + params[key] = $scope.form[key]; + } } if (params.custom.length) { var customParams = $scope.form.custom.split("&"); @@ -111,11 +124,12 @@ solrAdminApp.controller('DataImportController', } delete params.custom; - if (params.isDebugMode) { - params.dataConfig = $scope.rawConfig; + if ($scope.isDebugMode) { + params.dataConfig = $scope.config; } - delete params.showDebug; + params.core = $routeParams.core; + params.name = $scope.handler; DataImport.post(params, function (data) { $scope.rawResponse = JSON.stringify(data, null, 2); @@ -125,7 +139,7 @@ solrAdminApp.controller('DataImportController', $scope.abort = function () { $scope.isAborting = true; - DataImport.abort({core: $routeParams.core}, function () { + DataImport.abort({core: $routeParams.core, name: $scope.handler}, function () { $timeout(function () { $scope.isAborting = false; $scope.refreshStatus(); @@ -138,7 +152,7 @@ solrAdminApp.controller('DataImportController', console.log("Refresh Status"); $scope.isStatusLoading = true; - DataImport.status({core: $routeParams.core}, function (data) { + DataImport.status({core: $routeParams.core, name: $scope.handler}, function (data) { if (data[0] == "<") { $scope.hasHandlers = false; return; diff --git a/solr/webapp/web/js/angular/controllers/files.js b/solr/webapp/web/js/angular/controllers/files.js index 1cb9e5cd55e..00ea4b19ff7 100644 --- a/solr/webapp/web/js/angular/controllers/files.js +++ b/solr/webapp/web/js/angular/controllers/files.js @@ -16,7 +16,7 @@ */ var contentTypeMap = { xml : 'text/xml', html : 'text/html', js : 'text/javascript', json : 'application/json', 'css' : 'text/css' }; -var languages = {js: "javascript", xml:"xml", xsl:"xml", vm: "xml", html: "xml", json: "text", css: "css"}; +var languages = {js: "javascript", xml:"xml", xsl:"xml", vm: "xml", html: "xml", json: "json", css: "css"}; solrAdminApp.controller('FilesController', function($scope, $rootScope, $routeParams, $location, Files, Constants) { @@ -82,10 +82,10 @@ solrAdminApp.controller('FilesController', Files.get({core: $routeParams.core, file: $scope.file, contentType: contentType}, function(data) { $scope.content = data.data; $scope.url = $scope.baseurl + data.config.url + "?" + $.param(data.config.params); - if (contentType.indexOf("text/plain") && data.data.indexOf("=0) || data.data.indexOf("

    Attribute name