From 622e527cf2a05d914a91fd5e6fa0d0255cb68454 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Tue, 6 Jun 2017 14:30:19 +0100 Subject: [PATCH 001/131] SOLR-10671: remove two-and-a-half unused private static logger warnings --- .../apache/solr/metrics/reporters/SolrGangliaReporter.java | 4 ---- .../apache/solr/metrics/reporters/SolrGraphiteReporter.java | 4 ---- .../org/apache/solr/metrics/reporters/SolrSlf4jReporter.java | 3 ++- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/SolrGangliaReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/SolrGangliaReporter.java index 4dfdbeb11c0..8d879401bfd 100644 --- a/solr/core/src/java/org/apache/solr/metrics/reporters/SolrGangliaReporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/SolrGangliaReporter.java @@ -17,7 +17,6 @@ package org.apache.solr.metrics.reporters; import java.io.IOException; -import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -27,14 +26,11 @@ import com.codahale.metrics.ganglia.GangliaReporter; import info.ganglia.gmetric4j.gmetric.GMetric; import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricReporter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * */ public class SolrGangliaReporter extends SolrMetricReporter { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private String host = null; private int port = -1; diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/SolrGraphiteReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/SolrGraphiteReporter.java index b8eddf5690f..e74bb354375 100644 --- a/solr/core/src/java/org/apache/solr/metrics/reporters/SolrGraphiteReporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/SolrGraphiteReporter.java @@ -17,7 +17,6 @@ package org.apache.solr.metrics.reporters; import java.io.IOException; -import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -29,14 +28,11 @@ import com.codahale.metrics.graphite.GraphiteSender; import com.codahale.metrics.graphite.PickledGraphite; import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricReporter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Metrics reporter that wraps {@link com.codahale.metrics.graphite.GraphiteReporter}. */ public class SolrGraphiteReporter extends SolrMetricReporter { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private String host = null; private int port = -1; diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java index 7dfe5bba3b5..e3018852dde 100644 --- a/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java @@ -43,7 +43,8 @@ import org.slf4j.LoggerFactory; * */ public class SolrSlf4jReporter extends SolrMetricReporter { - // we need this to pass validate-source-patterns + + @SuppressWarnings("unused") // we need this to pass validate-source-patterns private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private int period = 60; From 29de0f75e8d8c5e89a00c3b7a7fcfffad14065c9 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Tue, 6 Jun 2017 11:05:17 -0400 Subject: [PATCH 002/131] SOLR-10722: UH optimize for Solr's getFieldInfos --- solr/CHANGES.txt | 2 ++ .../org/apache/solr/highlight/UnifiedSolrHighlighter.java | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 5343bc547f5..d7eb46f41ea 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -352,6 +352,8 @@ Optimizations so that the second phase which would normally involve calculating the domain for the bucket can be skipped entirely, leading to large performance improvements. (yonik) +* SOLR-10722: Speed up Solr's use of the UnifiedHighlighter be re-using FieldInfos. (David Smiley) + Other Changes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java index bd32fc1c16a..48cb874f499 100644 --- a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.function.Predicate; import org.apache.lucene.document.Document; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.uhighlight.CustomSeparatorBreakIterator; @@ -263,6 +264,12 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf } } + // optimization for Solr which keeps a FieldInfos on-hand + @Override + protected FieldInfo getFieldInfo(String field) { + return ((SolrIndexSearcher)searcher).getFieldInfos().fieldInfo(field); + } + @Override public int getMaxNoHighlightPassages(String field) { boolean defaultSummary = params.getFieldBool(field, HighlightParams.DEFAULT_SUMMARY, false); From 09a9fdab6d964a2be8e23537272c8a7093ee2d88 Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Tue, 6 Jun 2017 13:07:42 -0400 Subject: [PATCH 003/131] SOLR-10516: Add documention --- .../solr-ref-guide/src/stream-decorators.adoc | 23 ++++++++++++++++--- solr/solr-ref-guide/src/stream-sources.adoc | 5 +++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/solr/solr-ref-guide/src/stream-decorators.adoc b/solr/solr-ref-guide/src/stream-decorators.adoc index cbbc9ef791c..15948330f36 100644 --- a/solr/solr-ref-guide/src/stream-decorators.adoc +++ b/solr/solr-ref-guide/src/stream-decorators.adoc @@ -23,8 +23,6 @@ == cartesianProduct //TODO -== cell -//TODO == classify @@ -237,7 +235,26 @@ daemonStream.close(); ---- == eval -//todo + +The `eval` function allows for use cases where new streaming expressions are generated on the fly and then evaluated. +The `eval` function wraps a streaming expression and reads a single tuple from the underlying stream. +The `eval` function then retrieves a string Streaming Expressions from the `expr_s` field of the tuple. +The `eval` function then compiles the string Streaming Expression and emits the tuples. + +=== eval Parameters + +* `StreamExpression`: (Mandatory) The stream which provides the streaming expression to be evaluated. + +=== eval Syntax + +[source,text] +---- +eval(expr) +---- + +In the example above the `eval` expression reads the first tuple from the underlying expression. It then compiles and +executes the string Streaming Expression in the epxr_s field. + [[StreamingExpressions-executor]] == executor diff --git a/solr/solr-ref-guide/src/stream-sources.adoc b/solr/solr-ref-guide/src/stream-sources.adoc index 97a1ca46a09..973d135983f 100644 --- a/solr/solr-ref-guide/src/stream-sources.adoc +++ b/solr/solr-ref-guide/src/stream-sources.adoc @@ -284,6 +284,7 @@ The `significantTerms` function queries a SolrCloud collection, but instead of r * `collection`: (Mandatory) The collection that the function is run on. * `q`: (Mandatory) The query that describes the foreground document set. +* `field`: (Mandatory) The field to extract the terms from. * `limit`: (Optional, Default 20) The max number of terms to return. * `minDocFreq`: (Optional, Defaults to 5 documents) The minimum number of documents the term must appear in on a shard. This is a float value. If greater then 1.0 then it's considered the absolute number of documents. If less then 1.0 it's treated as a percentage of documents. * `maxDocFreq`: (Optional, Defaults to 30% of documents) The maximum number of documents the term can appear in on a shard. This is a float value. If greater then 1.0 then it's considered the absolute number of documents. If less then 1.0 it's treated as a percentage of documents. @@ -295,12 +296,14 @@ The `significantTerms` function queries a SolrCloud collection, but instead of r ---- significantTerms(collection1, q="body:Solr", + field="author", + limit="50", minDocFreq="10", maxDocFreq=".20", minTermLength="5") ---- -In the example above the `significantTerms` function is querying `collection1` and returning at most 50 significant terms that appear in 10 or more documents but not more then 20% of the corpus. +In the example above the `significantTerms` function is querying `collection1` and returning at most 50 significant terms from the `authors` field that appear in 10 or more documents but not more then 20% of the corpus. == shortestPath From d276acfbbcb8ea6124fe486e35cea9eb928e4702 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 6 Jun 2017 13:37:31 -0400 Subject: [PATCH 004/131] LUCENE-7854: enable indexing custom term frequencies --- lucene/CHANGES.txt | 4 + .../PackedTokenAttributeImpl.java | 29 +- .../TermFrequencyAttribute.java | 33 ++ .../TermFrequencyAttributeImpl.java | 82 +++ .../lucene/index/DefaultIndexingChain.java | 6 +- .../apache/lucene/index/FieldInvertState.java | 3 + .../index/FreqProxTermsWriterPerField.java | 27 +- .../index/TermVectorsConsumerPerField.java | 19 +- .../lucene/index/TermsHashPerField.java | 5 +- .../org/apache/lucene/analysis/TestToken.java | 2 + .../TestPackedTokenAttributeImpl.java | 2 + .../lucene/index/TestCustomTermFreq.java | 468 ++++++++++++++++++ .../lucene/index/TestFieldInvertState.java | 139 ++++++ .../apache/lucene/util/LuceneTestCase.java | 4 +- 14 files changed, 803 insertions(+), 20 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttribute.java create mode 100644 lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttributeImpl.java create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 4e2e4f27a5a..e4287223b15 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -14,6 +14,10 @@ New Features well as the oldest Lucene version that contributed to the segment. (Adrien Grand) +* LUCENE-7854: The new TermFrequencyAttribute used during analysis + with a custom token stream allows indexing custom term frequencies + (Mike McCandless) + API Changes * LUCENE-2605: Classic QueryParser no longer splits on whitespace by default. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java index ad1e23220f1..449ab3e20b6 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java @@ -26,15 +26,18 @@ import org.apache.lucene.util.AttributeReflector; *
  • {@link PositionIncrementAttribute} *
  • {@link PositionLengthAttribute} *
  • {@link OffsetAttribute} + *
  • {@link TermFrequencyAttribute} * */ public class PackedTokenAttributeImpl extends CharTermAttributeImpl implements TypeAttribute, PositionIncrementAttribute, - PositionLengthAttribute, OffsetAttribute { + PositionLengthAttribute, OffsetAttribute, + TermFrequencyAttribute { private int startOffset,endOffset; private String type = DEFAULT_TYPE; private int positionIncrement = 1; private int positionLength = 1; + private int termFrequency = 1; /** Constructs the attribute implementation. */ public PackedTokenAttributeImpl() { @@ -132,12 +135,26 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl this.type = type; } + @Override + public final void setTermFrequency(int termFrequency) { + if (termFrequency < 1) { + throw new IllegalArgumentException("Term frequency must be 1 or greater; got " + termFrequency); + } + this.termFrequency = termFrequency; + } + + @Override + public final int getTermFrequency() { + return termFrequency; + } + /** Resets the attributes */ @Override public void clear() { super.clear(); positionIncrement = positionLength = 1; + termFrequency = 1; startOffset = endOffset = 0; type = DEFAULT_TYPE; } @@ -147,10 +164,8 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl @Override public void end() { super.end(); + // super.end already calls this.clear, so we only set values that are different from clear: positionIncrement = 0; - positionLength = 1; - startOffset = endOffset = 0; - type = DEFAULT_TYPE; } @Override @@ -170,6 +185,7 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl positionIncrement == other.positionIncrement && positionLength == other.positionLength && (type == null ? other.type == null : type.equals(other.type)) && + termFrequency == other.termFrequency && super.equals(obj) ); } else @@ -185,6 +201,7 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl code = code * 31 + positionLength; if (type != null) code = code * 31 + type.hashCode(); + code = code * 31 + termFrequency;; return code; } @@ -198,12 +215,14 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl to.startOffset = startOffset; to.endOffset = endOffset; to.type = type; + to.termFrequency = termFrequency; } else { super.copyTo(target); ((OffsetAttribute) target).setOffset(startOffset, endOffset); ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement); ((PositionLengthAttribute) target).setPositionLength(positionLength); ((TypeAttribute) target).setType(type); + ((TermFrequencyAttribute) target).setTermFrequency(termFrequency); } } @@ -215,6 +234,6 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", positionIncrement); reflector.reflect(PositionLengthAttribute.class, "positionLength", positionLength); reflector.reflect(TypeAttribute.class, "type", type); + reflector.reflect(TermFrequencyAttribute.class, "termFrequency", termFrequency); } - } diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttribute.java new file mode 100644 index 00000000000..15aed412a13 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttribute.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.analysis.tokenattributes; + +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.util.Attribute; + +/** Sets the custom term frequency of a term within one document. If this attribute + * is present in your analysis chain for a given field, that field must be indexed with + * {@link IndexOptions#DOCS_AND_FREQS}. */ +public interface TermFrequencyAttribute extends Attribute { + + /** Set the custom term frequency of the current term within one document. */ + public void setTermFrequency(int termFrequency); + + /** Returns the custom term frequencey. */ + public int getTermFrequency(); +} diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttributeImpl.java new file mode 100644 index 00000000000..12409a7158e --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttributeImpl.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.analysis.tokenattributes; + + +import org.apache.lucene.util.AttributeImpl; +import org.apache.lucene.util.AttributeReflector; + +/** Default implementation of {@link TermFrequencyAttribute}. */ +public class TermFrequencyAttributeImpl extends AttributeImpl implements TermFrequencyAttribute, Cloneable { + private int termFrequency = 1; + + /** Initialize this attribute with term frequencey of 1 */ + public TermFrequencyAttributeImpl() {} + + @Override + public void setTermFrequency(int termFrequency) { + if (termFrequency < 1) { + throw new IllegalArgumentException("Term frequency must be 1 or greater; got " + termFrequency); + } + this.termFrequency = termFrequency; + } + + @Override + public int getTermFrequency() { + return termFrequency; + } + + @Override + public void clear() { + this.termFrequency = 1; + } + + @Override + public void end() { + this.termFrequency = 1; + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (other instanceof TermFrequencyAttributeImpl) { + TermFrequencyAttributeImpl _other = (TermFrequencyAttributeImpl) other; + return termFrequency == _other.termFrequency; + } + + return false; + } + + @Override + public int hashCode() { + return Integer.hashCode(termFrequency); + } + + @Override + public void copyTo(AttributeImpl target) { + TermFrequencyAttribute t = (TermFrequencyAttribute) target; + t.setTermFrequency(termFrequency); + } + + @Override + public void reflectWith(AttributeReflector reflector) { + reflector.reflect(TermFrequencyAttribute.class, "termFrequency", termFrequency); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java index ca384aebc71..df739d85061 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java +++ b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java @@ -770,10 +770,8 @@ final class DefaultIndexingChain extends DocConsumer { } invertState.lastStartOffset = startOffset; - invertState.length++; - if (invertState.length < 0) { - throw new IllegalArgumentException("too many tokens in field '" + field.name() + "'"); - } + invertState.length = Math.addExact(invertState.length, invertState.termFreqAttribute.getTermFrequency()); + //System.out.println(" term=" + invertState.termAttribute); // If we hit an exception in here, we abort diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java b/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java index f93edde3991..a40a06e2b3e 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java +++ b/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java @@ -20,6 +20,7 @@ import org.apache.lucene.analysis.TokenStream; // javadocs import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.util.AttributeSource; @@ -48,6 +49,7 @@ public final class FieldInvertState { PositionIncrementAttribute posIncrAttribute; PayloadAttribute payloadAttribute; TermToBytesRefAttribute termAttribute; + TermFrequencyAttribute termFreqAttribute; /** Creates {code FieldInvertState} for the specified * field name. */ @@ -88,6 +90,7 @@ public final class FieldInvertState { if (this.attributeSource != attributeSource) { this.attributeSource = attributeSource; termAttribute = attributeSource.getAttribute(TermToBytesRefAttribute.class); + termFreqAttribute = attributeSource.addAttribute(TermFrequencyAttribute.class); posIncrAttribute = attributeSource.addAttribute(PositionIncrementAttribute.class); offsetAttribute = attributeSource.addAttribute(OffsetAttribute.class); payloadAttribute = attributeSource.getAttribute(PayloadAttribute.class); diff --git a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java index 28fe8721867..7d77d0b62dd 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java +++ b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java @@ -113,9 +113,10 @@ final class FreqProxTermsWriterPerField extends TermsHashPerField { if (!hasFreq) { assert postings.termFreqs == null; postings.lastDocCodes[termID] = docState.docID; + fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency); } else { postings.lastDocCodes[termID] = docState.docID << 1; - postings.termFreqs[termID] = 1; + postings.termFreqs[termID] = getTermFreq(); if (hasProx) { writeProx(termID, fieldState.position); if (hasOffsets) { @@ -124,19 +125,21 @@ final class FreqProxTermsWriterPerField extends TermsHashPerField { } else { assert !hasOffsets; } + fieldState.maxTermFrequency = Math.max(postings.termFreqs[termID], fieldState.maxTermFrequency); } - fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency); fieldState.uniqueTermCount++; } @Override void addTerm(final int termID) { final FreqProxPostingsArray postings = freqProxPostingsArray; - assert !hasFreq || postings.termFreqs[termID] > 0; if (!hasFreq) { assert postings.termFreqs == null; + if (termFreqAtt.getTermFrequency() != 1) { + throw new IllegalStateException("field \"" + fieldInfo.name + "\": must index term freq while using custom TermFrequencyAttribute"); + } if (docState.docID != postings.lastDocIDs[termID]) { // New document; now encode docCode for previous doc: assert docState.docID > postings.lastDocIDs[termID]; @@ -160,8 +163,8 @@ final class FreqProxTermsWriterPerField extends TermsHashPerField { } // Init freq for the current document - postings.termFreqs[termID] = 1; - fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency); + postings.termFreqs[termID] = getTermFreq(); + fieldState.maxTermFrequency = Math.max(postings.termFreqs[termID], fieldState.maxTermFrequency); postings.lastDocCodes[termID] = (docState.docID - postings.lastDocIDs[termID]) << 1; postings.lastDocIDs[termID] = docState.docID; if (hasProx) { @@ -175,7 +178,8 @@ final class FreqProxTermsWriterPerField extends TermsHashPerField { } fieldState.uniqueTermCount++; } else { - fieldState.maxTermFrequency = Math.max(fieldState.maxTermFrequency, ++postings.termFreqs[termID]); + postings.termFreqs[termID] = Math.addExact(postings.termFreqs[termID], getTermFreq()); + fieldState.maxTermFrequency = Math.max(fieldState.maxTermFrequency, postings.termFreqs[termID]); if (hasProx) { writeProx(termID, fieldState.position-postings.lastPositions[termID]); if (hasOffsets) { @@ -185,6 +189,17 @@ final class FreqProxTermsWriterPerField extends TermsHashPerField { } } + private int getTermFreq() { + int freq = termFreqAtt.getTermFrequency(); + if (freq != 1) { + if (hasProx) { + throw new IllegalStateException("field \"" + fieldInfo.name + "\": cannot index positions while using custom TermFrequencyAttribute"); + } + } + + return freq; + } + @Override public void newPostingsArray() { freqProxPostingsArray = (FreqProxPostingsArray) postingsArray; diff --git a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java index 232fe70b815..4e0aa3cdaad 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java @@ -109,6 +109,7 @@ final class TermVectorsConsumerPerField extends TermsHashPerField { @Override boolean start(IndexableField field, boolean first) { + super.start(field, first); assert field.fieldType().indexOptions() != IndexOptions.NONE; if (first) { @@ -224,7 +225,7 @@ final class TermVectorsConsumerPerField extends TermsHashPerField { void newTerm(final int termID) { TermVectorsPostingsArray postings = termVectorsPostingsArray; - postings.freqs[termID] = 1; + postings.freqs[termID] = getTermFreq(); postings.lastOffsets[termID] = 0; postings.lastPositions[termID] = 0; @@ -235,11 +236,25 @@ final class TermVectorsConsumerPerField extends TermsHashPerField { void addTerm(final int termID) { TermVectorsPostingsArray postings = termVectorsPostingsArray; - postings.freqs[termID]++; + postings.freqs[termID] += getTermFreq(); writeProx(postings, termID); } + private int getTermFreq() { + int freq = termFreqAtt.getTermFrequency(); + if (freq != 1) { + if (doVectorPositions) { + throw new IllegalArgumentException("field \"" + fieldInfo.name + "\": cannot index term vector positions while using custom TermFrequencyAttribute"); + } + if (doVectorOffsets) { + throw new IllegalArgumentException("field \"" + fieldInfo.name + "\": cannot index term vector offsets while using custom TermFrequencyAttribute"); + } + } + + return freq; + } + @Override public void newPostingsArray() { termVectorsPostingsArray = (TermVectorsPostingsArray) postingsArray; diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java b/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java index d360adbb1f3..2586378267d 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java @@ -19,12 +19,13 @@ package org.apache.lucene.index; import java.io.IOException; +import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.util.ByteBlockPool; +import org.apache.lucene.util.BytesRefHash.BytesStartArray; import org.apache.lucene.util.BytesRefHash; import org.apache.lucene.util.Counter; import org.apache.lucene.util.IntBlockPool; -import org.apache.lucene.util.BytesRefHash.BytesStartArray; abstract class TermsHashPerField implements Comparable { private static final int HASH_INIT_SIZE = 4; @@ -35,6 +36,7 @@ abstract class TermsHashPerField implements Comparable { protected final DocumentsWriterPerThread.DocState docState; protected final FieldInvertState fieldState; TermToBytesRefAttribute termAtt; + protected TermFrequencyAttribute termFreqAtt; // Copied from our perThread final IntBlockPool intPool; @@ -287,6 +289,7 @@ abstract class TermsHashPerField implements Comparable { * document. */ boolean start(IndexableField field, boolean first) { termAtt = fieldState.termAttribute; + termFreqAtt = fieldState.termFreqAttribute; if (nextPerField != null) { doNextCall = nextPerField.start(field, first); } diff --git a/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java b/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java index fcf7779677f..b444aa8bb71 100644 --- a/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java +++ b/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java @@ -125,6 +125,7 @@ public class TestToken extends LuceneTestCase { t.setFlags(8); t.setPositionIncrement(3); t.setPositionLength(11); + t.setTermFrequency(42); TestUtil.assertAttributeReflection(t, new HashMap() {{ put(CharTermAttribute.class.getName() + "#term", "foobar"); @@ -136,6 +137,7 @@ public class TestToken extends LuceneTestCase { put(PayloadAttribute.class.getName() + "#payload", null); put(TypeAttribute.class.getName() + "#type", TypeAttribute.DEFAULT_TYPE); put(FlagsAttribute.class.getName() + "#flags", 8); + put(TermFrequencyAttribute.class.getName() + "#termFrequency", 42); }}); } } diff --git a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestPackedTokenAttributeImpl.java b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestPackedTokenAttributeImpl.java index cf4409e3403..c673fc65be9 100644 --- a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestPackedTokenAttributeImpl.java +++ b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestPackedTokenAttributeImpl.java @@ -82,6 +82,7 @@ public class TestPackedTokenAttributeImpl extends LuceneTestCase { t.setPositionIncrement(3); t.setPositionLength(11); t.setType("foobar"); + t.setTermFrequency(42); TestUtil.assertAttributeReflection(t, new HashMap() {{ put(CharTermAttribute.class.getName() + "#term", "foobar"); @@ -91,6 +92,7 @@ public class TestPackedTokenAttributeImpl extends LuceneTestCase { put(PositionIncrementAttribute.class.getName() + "#positionIncrement", 3); put(PositionLengthAttribute.class.getName() + "#positionLength", 11); put(TypeAttribute.class.getName() + "#type", "foobar"); + put(TermFrequencyAttribute.class.getName() + "#termFrequency", 42); }}); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java new file mode 100644 index 00000000000..5b38f576644 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java @@ -0,0 +1,468 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.TextField; +import org.apache.lucene.search.CollectionStatistics; +import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LuceneTestCase; + +import static org.apache.lucene.index.PostingsEnum.NO_MORE_DOCS; + +public class TestCustomTermFreq extends LuceneTestCase { + + private static final class CannedTermFreqs extends TokenStream { + private final String[] terms; + private final int[] termFreqs; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final TermFrequencyAttribute termFreqAtt = addAttribute(TermFrequencyAttribute.class); + private int upto; + + public CannedTermFreqs(String[] terms, int[] termFreqs) { + this.terms = terms; + this.termFreqs = termFreqs; + assert terms.length == termFreqs.length; + } + + @Override + public boolean incrementToken() { + if (upto == terms.length) { + return false; + } + + clearAttributes(); + + termAtt.append(terms[upto]); + termFreqAtt.setTermFrequency(termFreqs[upto]); + + upto++; + return true; + } + + @Override + public void reset() { + upto = 0; + } + } + + public void testSingletonTermsOneDoc() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar"}, + new int[] {42, 128}), + fieldType); + doc.add(field); + w.addDocument(doc); + IndexReader r = DirectoryReader.open(w); + PostingsEnum postings = MultiFields.getTermDocsEnum(r, "field", new BytesRef("bar")); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(128, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + postings = MultiFields.getTermDocsEnum(r, "field", new BytesRef("foo")); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(42, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + IOUtils.close(r, w, dir); + } + + public void testSingletonTermsTwoDocs() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar"}, + new int[] {42, 128}), + fieldType); + doc.add(field); + w.addDocument(doc); + + doc = new Document(); + field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar"}, + new int[] {50, 50}), + fieldType); + doc.add(field); + w.addDocument(doc); + + IndexReader r = DirectoryReader.open(w); + PostingsEnum postings = MultiFields.getTermDocsEnum(r, "field", new BytesRef("bar")); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(128, postings.freq()); + assertEquals(1, postings.nextDoc()); + assertEquals(50, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + postings = MultiFields.getTermDocsEnum(r, "field", new BytesRef("foo")); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(42, postings.freq()); + assertEquals(1, postings.nextDoc()); + assertEquals(50, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + IOUtils.close(r, w, dir); + } + + public void testRepeatTermsOneDoc() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {42, 128, 17, 100}), + fieldType); + doc.add(field); + w.addDocument(doc); + IndexReader r = DirectoryReader.open(w); + PostingsEnum postings = MultiFields.getTermDocsEnum(r, "field", new BytesRef("bar")); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(228, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + postings = MultiFields.getTermDocsEnum(r, "field", new BytesRef("foo")); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(59, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + IOUtils.close(r, w, dir); + } + + public void testRepeatTermsTwoDocs() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {42, 128, 17, 100}), + fieldType); + doc.add(field); + w.addDocument(doc); + + doc = new Document(); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {50, 60, 70, 80}), + fieldType); + doc.add(field); + w.addDocument(doc); + + IndexReader r = DirectoryReader.open(w); + PostingsEnum postings = MultiFields.getTermDocsEnum(r, "field", new BytesRef("bar")); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(228, postings.freq()); + assertEquals(1, postings.nextDoc()); + assertEquals(140, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + postings = MultiFields.getTermDocsEnum(r, "field", new BytesRef("foo")); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(59, postings.freq()); + assertEquals(1, postings.nextDoc()); + assertEquals(120, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + IOUtils.close(r, w, dir); + } + + public void testTotalTermFreq() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {42, 128, 17, 100}), + fieldType); + doc.add(field); + w.addDocument(doc); + + doc = new Document(); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {50, 60, 70, 80}), + fieldType); + doc.add(field); + w.addDocument(doc); + + IndexReader r = DirectoryReader.open(w); + + TermsEnum termsEnum = MultiFields.getTerms(r, "field").iterator(); + assertTrue(termsEnum.seekExact(new BytesRef("foo"))); + assertEquals(179, termsEnum.totalTermFreq()); + assertTrue(termsEnum.seekExact(new BytesRef("bar"))); + assertEquals(368, termsEnum.totalTermFreq()); + + IOUtils.close(r, w, dir); + } + + // you can't index proximity with custom term freqs: + public void testInvalidProx() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {42, 128, 17, 100}), + fieldType); + doc.add(field); + Exception e = expectThrows(IllegalStateException.class, () -> {w.addDocument(doc);}); + assertEquals("field \"field\": cannot index positions while using custom TermFrequencyAttribute", e.getMessage()); + IOUtils.close(w, dir); + } + + // you can't index DOCS_ONLY with custom term freq + public void testInvalidDocsOnly() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {42, 128, 17, 100}), + fieldType); + doc.add(field); + Exception e = expectThrows(IllegalStateException.class, () -> {w.addDocument(doc);}); + assertEquals("field \"field\": must index term freq while using custom TermFrequencyAttribute", e.getMessage()); + IOUtils.close(w, dir); + } + + // sum of term freqs must fit in an int + public void testOverflowInt() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS); + + Document doc = new Document(); + doc.add(new Field("field", "this field should be indexed", fieldType)); + w.addDocument(doc); + + Document doc2 = new Document(); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar"}, + new int[] {3, Integer.MAX_VALUE}), + fieldType); + doc2.add(field); + expectThrows(ArithmeticException.class, () -> {w.addDocument(doc2);}); + + IndexReader r = DirectoryReader.open(w); + assertEquals(1, r.numDocs()); + + IOUtils.close(r, w, dir); + } + + public void testInvalidTermVectorPositions() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + fieldType.setStoreTermVectors(true); + fieldType.setStoreTermVectorPositions(true); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {42, 128, 17, 100}), + fieldType); + doc.add(field); + Exception e = expectThrows(IllegalArgumentException.class, () -> {w.addDocument(doc);}); + assertEquals("field \"field\": cannot index term vector positions while using custom TermFrequencyAttribute", e.getMessage()); + IOUtils.close(w, dir); + } + + public void testInvalidTermVectorOffsets() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + fieldType.setStoreTermVectors(true); + fieldType.setStoreTermVectorOffsets(true); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {42, 128, 17, 100}), + fieldType); + doc.add(field); + Exception e = expectThrows(IllegalArgumentException.class, () -> {w.addDocument(doc);}); + assertEquals("field \"field\": cannot index term vector offsets while using custom TermFrequencyAttribute", e.getMessage()); + IOUtils.close(w, dir); + } + + public void testTermVectors() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + fieldType.setStoreTermVectors(true); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {42, 128, 17, 100}), + fieldType); + doc.add(field); + w.addDocument(doc); + + doc = new Document(); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {50, 60, 70, 80}), + fieldType); + doc.add(field); + w.addDocument(doc); + + IndexReader r = DirectoryReader.open(w); + + Fields fields = r.getTermVectors(0); + TermsEnum termsEnum = fields.terms("field").iterator(); + assertTrue(termsEnum.seekExact(new BytesRef("bar"))); + assertEquals(228, termsEnum.totalTermFreq()); + PostingsEnum postings = termsEnum.postings(null); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(228, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + assertTrue(termsEnum.seekExact(new BytesRef("foo"))); + assertEquals(59, termsEnum.totalTermFreq()); + postings = termsEnum.postings(null); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(59, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + fields = r.getTermVectors(1); + termsEnum = fields.terms("field").iterator(); + assertTrue(termsEnum.seekExact(new BytesRef("bar"))); + assertEquals(140, termsEnum.totalTermFreq()); + postings = termsEnum.postings(null); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(140, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + assertTrue(termsEnum.seekExact(new BytesRef("foo"))); + assertEquals(120, termsEnum.totalTermFreq()); + postings = termsEnum.postings(null); + assertNotNull(postings); + assertEquals(0, postings.nextDoc()); + assertEquals(120, postings.freq()); + assertEquals(NO_MORE_DOCS, postings.nextDoc()); + + IOUtils.close(r, w, dir); + } + + /** + * Similarity holds onto the FieldInvertState for subsequent verification. + */ + private static class NeverForgetsSimilarity extends Similarity { + public FieldInvertState lastState; + private final static NeverForgetsSimilarity INSTANCE = new NeverForgetsSimilarity(); + + private NeverForgetsSimilarity() { + // no + } + + @Override + public long computeNorm(FieldInvertState state) { + this.lastState = state; + return 1; + } + + @Override + public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { + throw new UnsupportedOperationException(); + } + + @Override + public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException { + throw new UnsupportedOperationException(); + } + } + + public void testFieldInvertState() throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); + iwc.setSimilarity(NeverForgetsSimilarity.INSTANCE); + IndexWriter w = new IndexWriter(dir, iwc); + + Document doc = new Document(); + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + Field field = new Field("field", + new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"}, + new int[] {42, 128, 17, 100}), + fieldType); + doc.add(field); + w.addDocument(doc); + FieldInvertState fis = NeverForgetsSimilarity.INSTANCE.lastState; + assertEquals(228, fis.getMaxTermFrequency()); + assertEquals(2, fis.getUniqueTermCount()); + assertEquals(0, fis.getNumOverlap()); + assertEquals(287, fis.getLength()); + + IOUtils.close(w, dir); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java new file mode 100644 index 00000000000..f78b7fa92c5 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.lucene.analysis.CannedTokenStream; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.search.CollectionStatistics; +import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +public class TestFieldInvertState extends LuceneTestCase { + /** + * Similarity holds onto the FieldInvertState for subsequent verification. + */ + private static class NeverForgetsSimilarity extends Similarity { + public FieldInvertState lastState; + private final static NeverForgetsSimilarity INSTANCE = new NeverForgetsSimilarity(); + + private NeverForgetsSimilarity() { + // no + } + + @Override + public long computeNorm(FieldInvertState state) { + this.lastState = state; + return 1; + } + + @Override + public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { + throw new UnsupportedOperationException(); + } + + @Override + public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException { + throw new UnsupportedOperationException(); + } + } + + public void testBasic() throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); + iwc.setSimilarity(NeverForgetsSimilarity.INSTANCE); + IndexWriter w = new IndexWriter(dir, iwc); + Document doc = new Document(); + Field field = new Field("field", + new CannedTokenStream(new Token("a", 0, 1), + new Token("b", 2, 3), + new Token("c", 4, 5)), + TextField.TYPE_NOT_STORED); + doc.add(field); + w.addDocument(doc); + FieldInvertState fis = NeverForgetsSimilarity.INSTANCE.lastState; + assertEquals(1, fis.getMaxTermFrequency()); + assertEquals(3, fis.getUniqueTermCount()); + assertEquals(0, fis.getNumOverlap()); + assertEquals(3, fis.getLength()); + IOUtils.close(w, dir); + } + + public void testRandom() throws Exception { + int numUniqueTokens = TestUtil.nextInt(random(), 1, 25); + Directory dir = newDirectory(); + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); + iwc.setSimilarity(NeverForgetsSimilarity.INSTANCE); + IndexWriter w = new IndexWriter(dir, iwc); + Document doc = new Document(); + + int numTokens = atLeast(10000); + Token[] tokens = new Token[numTokens]; + Map counts = new HashMap<>(); + int numStacked = 0; + int maxTermFreq = 0; + int pos = -1; + for (int i=0;i 0 && random().nextInt(7) == 3) { + token.setPositionIncrement(0); + numStacked++; + } else { + pos++; + } + tokens[i] = token; + } + + Field field = new Field("field", + new CannedTokenStream(tokens), + TextField.TYPE_NOT_STORED); + doc.add(field); + w.addDocument(doc); + FieldInvertState fis = NeverForgetsSimilarity.INSTANCE.lastState; + assertEquals(maxTermFreq, fis.getMaxTermFrequency()); + assertEquals(counts.size(), fis.getUniqueTermCount()); + assertEquals(numStacked, fis.getNumOverlap()); + assertEquals(numTokens, fis.getLength()); + assertEquals(pos, fis.getPosition()); + + IOUtils.close(w, dir); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index 591db20341a..0243a56ccb6 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -2676,11 +2676,11 @@ public abstract class LuceneTestCase extends Assert { if (expectedType.isInstance(e)) { return expectedType.cast(e); } - AssertionFailedError assertion = new AssertionFailedError("Unexpected exception type, expected " + expectedType.getSimpleName()); + AssertionFailedError assertion = new AssertionFailedError("Unexpected exception type, expected " + expectedType.getSimpleName() + " but got " + e); assertion.initCause(e); throw assertion; } - throw new AssertionFailedError("Expected exception " + expectedType.getSimpleName()); + throw new AssertionFailedError("Expected exception " + expectedType.getSimpleName() + " but no exception was thrown"); } /** From 98e103731f15782e9d0c67b3998216801d2164a1 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Tue, 6 Jun 2017 14:23:51 -0400 Subject: [PATCH 005/131] SOLR-10815: avoid long->float precision loss in 'product(-1,ms(date_field)' by subtracting a base date: 'product(-1,ms(date_field,base_date_field))' --- solr/core/src/test/org/apache/solr/schema/TestPointFields.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java index 02d2ac24ae7..b93b260ed91 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java +++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java @@ -2761,7 +2761,7 @@ public class TestPointFields extends SolrTestCaseJ4 { } assertU(commit()); assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof DatePointField); - assertQ(req("q", "*:*", "fl", "id, " + field, "sort", "product(-1,ms(" + field + ")) asc"), + assertQ(req("q", "*:*", "fl", "id, " + field, "sort", "product(-1,ms(" + field + "," + baseDate +")) asc"), "//*[@numFound='10']", "//result/doc[1]/date[@name='" + field + "'][.='1995-01-10T10:59:20Z']", "//result/doc[2]/date[@name='" + field + "'][.='1995-01-10T10:59:19Z']", From 46a5ae23a76fcf0cbb98ac3874ae69cdb90173a4 Mon Sep 17 00:00:00 2001 From: Tomas Fernandez Lobbe Date: Tue, 6 Jun 2017 16:37:55 -0700 Subject: [PATCH 006/131] SOLR-10233: Some more logging to chaos monkey with replica types tests --- .../ChaosMonkeyNothingIsSafeWithPullReplicasTest.java | 2 ++ .../cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java | 2 ++ .../apache/solr/cloud/AbstractFullDistribZkTestBase.java | 8 ++++---- .../java/org/apache/solr/cloud/StoppableCommitThread.java | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java index 11c25d3097d..672cd6beb72 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java @@ -269,6 +269,8 @@ public class ChaosMonkeyNothingIsSafeWithPullReplicasTest extends AbstractFullDi assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0); + log.info("collection state: " + printClusterStateInfo(DEFAULT_COLLECTION)); + if (VERBOSE) System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults() .getNumFound() + "\n\n"); diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java index e4859c005e5..ce779963b6d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java @@ -204,6 +204,8 @@ public class ChaosMonkeySafeLeaderWithPullReplicasTest extends AbstractFullDistr log.info("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n"); + log.info("collection state: " + printClusterStateInfo(DEFAULT_COLLECTION)); + waitForReplicationFromReplicas(DEFAULT_COLLECTION, cloudClient.getZkStateReader(), new TimeOut(30, TimeUnit.SECONDS)); // waitForAllWarmingSearchers(); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index 2970a8b0e13..5fa4af5812a 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -2094,7 +2094,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes while (true) { long replicaIndexVersion = getIndexVersion(pullReplica); if (leaderIndexVersion == replicaIndexVersion) { - log.debug("Leader replica's version ({}) in sync with replica({}): {} == {}", leader.getName(), pullReplica.getName(), leaderIndexVersion, replicaIndexVersion); + log.info("Leader replica's version ({}) in sync with replica({}): {} == {}", leader.getName(), pullReplica.getName(), leaderIndexVersion, replicaIndexVersion); // Make sure the host is serving the correct version try (SolrCore core = containers.get(pullReplica.getNodeName()).getCore(pullReplica.getCoreName())) { @@ -2105,7 +2105,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes if (Long.parseLong(servingVersion) == replicaIndexVersion) { break; } else { - log.debug("Replica {} has the correct version replicated, but the searcher is not ready yet. Replicated version: {}, Serving version: {}", pullReplica.getName(), replicaIndexVersion, servingVersion); + log.info("Replica {} has the correct version replicated, but the searcher is not ready yet. Replicated version: {}, Serving version: {}", pullReplica.getName(), replicaIndexVersion, servingVersion); } } finally { if (ref != null) ref.decref(); @@ -2117,9 +2117,9 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes fail(String.format(Locale.ROOT, "Timed out waiting for replica %s (%d) to replicate from leader %s (%d)", pullReplica.getName(), replicaIndexVersion, leader.getName(), leaderIndexVersion)); } if (leaderIndexVersion > replicaIndexVersion) { - log.debug("{} version is {} and leader's is {}, will wait for replication", pullReplica.getName(), replicaIndexVersion, leaderIndexVersion); + log.info("{} version is {} and leader's is {}, will wait for replication", pullReplica.getName(), replicaIndexVersion, leaderIndexVersion); } else { - log.debug("Leader replica's version ({}) is lower than pull replica({}): {} < {}", leader.getName(), pullReplica.getName(), leaderIndexVersion, replicaIndexVersion); + log.info("Leader replica's version ({}) is lower than pull replica({}): {} < {}", leader.getName(), pullReplica.getName(), leaderIndexVersion, replicaIndexVersion); } } Thread.sleep(1000); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/StoppableCommitThread.java b/solr/test-framework/src/java/org/apache/solr/cloud/StoppableCommitThread.java index f87ebb5a2a9..4d60b4e06e1 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/StoppableCommitThread.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/StoppableCommitThread.java @@ -58,7 +58,7 @@ public class StoppableCommitThread extends StoppableThread { break; } } - log.debug("StoppableCommitThread finished. Committed {} times. Failed {} times.", numCommits.get(), numFails.get()); + log.info("StoppableCommitThread finished. Committed {} times. Failed {} times.", numCommits.get(), numFails.get()); } @Override From 97655b880c0230c0d42baba314c28831ee729323 Mon Sep 17 00:00:00 2001 From: Tomas Fernandez Lobbe Date: Tue, 6 Jun 2017 17:22:01 -0700 Subject: [PATCH 007/131] SOLR-10233: Cleanup warnings from ReplicateFromLeader --- .../src/java/org/apache/solr/cloud/ReplicateFromLeader.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java index 0800e0f38e8..d0de6aec744 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java +++ b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java @@ -72,10 +72,10 @@ public class ReplicateFromLeader { } LOG.info("Will start replication from leader with poll interval: {}", pollIntervalStr ); - NamedList slaveConfig = new NamedList(); - slaveConfig.add("fetchFromLeader", true); + NamedList slaveConfig = new NamedList<>(); + slaveConfig.add("fetchFromLeader", Boolean.TRUE); slaveConfig.add("pollInterval", pollIntervalStr); - NamedList replicationConfig = new NamedList(); + NamedList replicationConfig = new NamedList<>(); replicationConfig.add("slave", slaveConfig); String lastCommitVersion = getCommitVersion(core); From a03c3369e28a1c350842649726801e79285625e7 Mon Sep 17 00:00:00 2001 From: Tomas Fernandez Lobbe Date: Tue, 6 Jun 2017 17:23:01 -0700 Subject: [PATCH 008/131] SOLR-10233: Stop warning users about misconfigured ReplicationHandler when using replica types --- solr/core/src/java/org/apache/solr/handler/IndexFetcher.java | 2 +- .../src/java/org/apache/solr/handler/ReplicationHandler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index 7d15701e396..9c9918ebdf5 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -155,7 +155,7 @@ public class IndexFetcher { private boolean useExternalCompression = false; - private boolean fetchFromLeader = false; + boolean fetchFromLeader = false; private final HttpClient myHttpClient; diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index f3dcdeb9237..2d545bac956 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -1217,7 +1217,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw NamedList master = (NamedList) initArgs.get("master"); boolean enableMaster = isEnabled( master ); - if (enableMaster || enableSlave) { + if (enableMaster || (enableSlave && !currentIndexFetcher.fetchFromLeader)) { if (core.getCoreContainer().getZkController() != null) { LOG.warn("SolrCloud is enabled for core " + core.getName() + " but so is old-style replication. Make sure you" + " intend this behavior, it usually indicates a mis-configuration. Master setting is " + From a1692c160ac24746c17eae8d6cc5d40771c38e61 Mon Sep 17 00:00:00 2001 From: yonik Date: Tue, 6 Jun 2017 20:54:29 -0400 Subject: [PATCH 009/131] SOLR-7452: fix facet refinement for range queries --- .../apache/solr/search/facet/FacetRange.java | 127 +++++++++++++++++- .../apache/solr/search/facet/DebugAgg.java | 3 +- .../search/facet/TestJsonFacetRefinement.java | 29 +++- 3 files changed, 155 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java index 682dc19036f..398fa63fc82 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java @@ -37,6 +37,8 @@ import org.apache.solr.schema.TrieField; import org.apache.solr.search.DocSet; import org.apache.solr.util.DateMathParser; +import static org.apache.solr.search.facet.FacetContext.SKIP_FACET; + public class FacetRange extends FacetRequestSorted { String field; Object start; @@ -203,6 +205,10 @@ class FacetRangeProcessor extends FacetProcessor { "Unable to range facet on field:" + sf); } + if (fcontext.facetInfo != null) { + return refineFacets(); + } + createRangeList(); return getRangeCountsIndexed(); } @@ -322,7 +328,7 @@ class FacetRangeProcessor extends FacetProcessor { } for (int idx = 0; idx { } } + + // this refineFacets method is patterned after FacetFieldProcessor.refineFacets and should + // probably be merged when range facet becomes more like field facet in it's ability to sort and limit + protected SimpleOrderedMap refineFacets() throws IOException { + boolean skipThisFacet = (fcontext.flags & SKIP_FACET) != 0; + + List leaves = FacetFieldProcessor.asList(fcontext.facetInfo.get("_l")); // We have not seen this bucket: do full faceting for this bucket, including all sub-facets + List skip = FacetFieldProcessor.asList(fcontext.facetInfo.get("_s")); // We have seen this bucket, so skip stats on it, and skip sub-facets except for the specified sub-facets that should calculate specified buckets. + List partial = FacetFieldProcessor.asList(fcontext.facetInfo.get("_p")); // We have not seen this bucket, do full faceting for this bucket, and most sub-facets... but some sub-facets are partial and should only visit specified buckets. + + // currently, only _s should be present for range facets. In the future, range facets will + // be more like field facets and will have the same refinement cases. When that happens, we should try to unify the refinement code more + assert leaves.size() == 0; + assert partial.size() == 0; + + // For leaf refinements, we do full faceting for each leaf bucket. Any sub-facets of these buckets will be fully evaluated. Because of this, we should never + // encounter leaf refinements that have sub-facets that return partial results. + + SimpleOrderedMap res = new SimpleOrderedMap<>(); + List bucketList = new ArrayList<>( leaves.size() + skip.size() + partial.size() ); + res.add("buckets", bucketList); + + // TODO: an alternate implementations can fill all accs at once + createAccs(-1, 1); + + for (Object bucketVal : leaves) { + bucketList.add( refineBucket(bucketVal, false, null) ); + } + + for (List bucketAndFacetInfo : skip) { + assert bucketAndFacetInfo.size() == 2; + Object bucketVal = bucketAndFacetInfo.get(0); + Map facetInfo = (Map) bucketAndFacetInfo.get(1); + + bucketList.add( refineBucket(bucketVal, true, facetInfo ) ); + } + + // The only difference between skip and missing is the value of "skip" passed to refineBucket + for (List bucketAndFacetInfo : partial) { + assert bucketAndFacetInfo.size() == 2; + Object bucketVal = bucketAndFacetInfo.get(0); + Map facetInfo = (Map) bucketAndFacetInfo.get(1); + + bucketList.add( refineBucket(bucketVal, false, facetInfo ) ); + } + + /*** special buckets + if (freq.missing) { + Map bucketFacetInfo = (Map)fcontext.facetInfo.get("missing"); + + if (bucketFacetInfo != null || !skipThisFacet) { + SimpleOrderedMap missingBucket = new SimpleOrderedMap<>(); + fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null, skipThisFacet, bucketFacetInfo); + res.add("missing", missingBucket); + } + } + **********/ + + + // If there are just a couple of leaves, and if the domain is large, then + // going by term is likely the most efficient? + // If the domain is small, or if the number of leaves is large, then doing + // the normal collection method may be best. + + return res; + } + + private SimpleOrderedMap refineBucket(Object bucketVal, boolean skip, Map facetInfo) throws IOException { + // TODO: refactor this repeated code from above + Comparable start = calc.getValue(bucketVal.toString()); + Comparable end = calc.getValue(freq.end.toString()); + EnumSet include = freq.include; + + String gap = freq.gap.toString(); + + Comparable low = calc.getValue(bucketVal.toString()); + Comparable high = calc.addGap(low, gap); + if (end.compareTo(high) < 0) { + if (freq.hardend) { + high = end; + } else { + end = high; + } + } + if (high.compareTo(low) < 0) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "range facet infinite loop (is gap negative? did the math overflow?)"); + } + if (high.compareTo(low) == 0) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, + "range facet infinite loop: gap is either zero, or too small relative start/end and caused underflow: " + low + " + " + gap + " = " + high ); + } + + boolean incLower = + (include.contains(FacetParams.FacetRangeInclude.LOWER) || + (include.contains(FacetParams.FacetRangeInclude.EDGE) && + 0 == low.compareTo(start))); + boolean incUpper = + (include.contains(FacetParams.FacetRangeInclude.UPPER) || + (include.contains(FacetParams.FacetRangeInclude.EDGE) && + 0 == high.compareTo(end))); + + Range range = new Range(low, low, high, incLower, incUpper); + + + // now refine this range + + SimpleOrderedMap bucket = new SimpleOrderedMap<>(); + FieldType ft = sf.getType(); + bucket.add("val", bucketVal); + // String internal = ft.toInternal( tobj.toString() ); // TODO - we need a better way to get from object to query... + + Query domainQ = sf.getType().getRangeQuery(null, sf, range.low == null ? null : calc.formatValue(range.low), range.high==null ? null : calc.formatValue(range.high), range.includeLower, range.includeUpper); + fillBucket(bucket, domainQ, null, skip, facetInfo); + + return bucket; + } } diff --git a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java index ad4fabf6f19..b8dbf2a6263 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java +++ b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java @@ -84,8 +84,7 @@ public class DebugAgg extends AggValueSource { this.numSlots = numSlots; creates.addAndGet(1); sub = new CountSlotArrAcc(fcontext, numSlots); - - new RuntimeException("DEBUG Acc numSlots=" + numSlots).printStackTrace(); +// new RuntimeException("DEBUG Acc numSlots=" + numSlots).printStackTrace(); } @Override diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java index 9087d30a241..7cf14280c66 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.solr.JSONTestUtil; import org.apache.solr.SolrTestCaseHS; +import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.SimpleOrderedMap; @@ -32,7 +33,7 @@ import org.junit.Test; import org.noggit.JSONParser; import org.noggit.ObjectBuilder; - +@SolrTestCaseJ4.SuppressPointFields public class TestJsonFacetRefinement extends SolrTestCaseHS { private static SolrInstances servers; // for distributed testing @@ -314,6 +315,32 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS { "}" ); + // basic refining test through/under a query facet + client.testJQ(params(p, "q", "*:*", + "json.facet", "{" + + "q1 : { type:query, q:'*:*', facet:{" + + "cat0:{${terms} type:terms, field:${cat_s}, sort:'count desc', limit:1, overrequest:0, refine:true}" + + "}}" + + "}" + ) + , "facets=={ count:8" + + ", q1:{ count:8, cat0:{ buckets:[ {val:A,count:4} ] } }" + + "}" + ); + + // basic refining test through/under a range facet + client.testJQ(params(p, "q", "*:*", + "json.facet", "{" + + "r1 : { type:range, field:${num_d} start:-20, end:20, gap:40 , facet:{" + + "cat0:{${terms} type:terms, field:${cat_s}, sort:'count desc', limit:1, overrequest:0, refine:true}" + + "}}" + + "}" + ) + , "facets=={ count:8" + + ", r1:{ buckets:[{val:-20.0,count:8, cat0:{buckets:[{val:A,count:4}]} }] }" + + "}" + ); + // test that basic stats work for refinement client.testJQ(params(p, "q", "*:*", "json.facet", "{" + From 1921b61ba8f3c7579bc04975b7ce90167a74e51e Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Tue, 6 Jun 2017 21:51:17 -0400 Subject: [PATCH 010/131] Ref Guide: Remove place holders --- .../solr-ref-guide/src/stream-decorators.adoc | 7 ----- .../solr-ref-guide/src/stream-evaluators.adoc | 26 ------------------- 2 files changed, 33 deletions(-) diff --git a/solr/solr-ref-guide/src/stream-decorators.adoc b/solr/solr-ref-guide/src/stream-decorators.adoc index 15948330f36..abd7c10145c 100644 --- a/solr/solr-ref-guide/src/stream-decorators.adoc +++ b/solr/solr-ref-guide/src/stream-decorators.adoc @@ -20,10 +20,6 @@ // specific language governing permissions and limitations // under the License. -== cartesianProduct -//TODO - - == classify The `classify` function classifies tuples using a logistic regression text classification model. It was designed specifically to work with models trained using the <>. The `classify` function uses the <> to retrieve a stored model and then scores a stream of tuples using the model. The tuples read by the classifier must contain a text field that can be used for classification. The classify function uses a Lucene analyzer to extract the features from the text so the model can be applied. By default the `classify` function looks for the analyzer using the name of text field in the tuple. If the Solr schema on the worker node does not contain this field, the analyzer can be looked up in another field by specifying the `analyzerField` parameter. @@ -534,9 +530,6 @@ merge( on="fieldA asc") ---- -== list -// TODO - == null The null expression is a useful utility function for understanding bottlenecks when performing parallel relational algebra (joins, intersections, rollups etc.). The null function reads all the tuples from an underlying stream and returns a single tuple with the count and processing time. Because the null stream adds minimal overhead of it's own, it can be used to isolate the performance of Solr's /export handler. If the /export handlers performance is not the bottleneck, then the bottleneck is likely occurring in the workers where the stream decorators are running. diff --git a/solr/solr-ref-guide/src/stream-evaluators.adoc b/solr/solr-ref-guide/src/stream-evaluators.adoc index 3927b8f934f..2c2a2ae2664 100644 --- a/solr/solr-ref-guide/src/stream-evaluators.adoc +++ b/solr/solr-ref-guide/src/stream-evaluators.adoc @@ -412,30 +412,6 @@ or(and(fieldA,fieldB),fieldC) // (fieldA && fieldB) || fieldC or(fieldA,fieldB,fieldC,and(fieldD,fieldE),fieldF) ---- -== analyze -//TODO - -== second -//TODO - -== minute -//TODO - -== hour -//TODO - -== day -//TODO - -== month -//TODO - -== year -//TODO - -== convert -//TODO - == raw The `raw` function will return whatever raw value is the parameter. This is useful for cases where you want to use a string as part of another evaluator. @@ -457,5 +433,3 @@ raw(true) // "true" (note: this returns the string "true" and not the boolean tr eq(raw(fieldA), fieldA) // true if the value of fieldA equals the string "fieldA" ---- -== UUID -//TODO From 23adc2b0a0a6d6be7990e93030af9f5600e1e784 Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Wed, 7 Jun 2017 08:04:29 +0530 Subject: [PATCH 011/131] Adding 6.6.0 release version number --- dev-tools/doap/lucene.rdf | 7 +++++++ dev-tools/doap/solr.rdf | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/dev-tools/doap/lucene.rdf b/dev-tools/doap/lucene.rdf index e47eaf32702..166688690e2 100644 --- a/dev-tools/doap/lucene.rdf +++ b/dev-tools/doap/lucene.rdf @@ -66,6 +66,13 @@ + + + lucene-6.6.0 + 2017-06-06 + 6.6.0 + + lucene-6.5.1 diff --git a/dev-tools/doap/solr.rdf b/dev-tools/doap/solr.rdf index eb92a96de08..a6000faf854 100644 --- a/dev-tools/doap/solr.rdf +++ b/dev-tools/doap/solr.rdf @@ -66,6 +66,13 @@ + + + solr-6.6.0 + 2017-06-06 + 6.6.0 + + solr-6.5.1 From fe176b601b41f46404caa195839f1780e59aa3d1 Mon Sep 17 00:00:00 2001 From: jpgilaberte Date: Mon, 29 May 2017 13:06:05 +0200 Subject: [PATCH 012/131] LUCENE-7855: The advanced parameters of the Wikipedia tokenizer are added to the factory Closes #209 --- lucene/CHANGES.txt | 5 ++ .../wikipedia/WikipediaTokenizerFactory.java | 21 +++-- .../TestWikipediaTokenizerFactory.java | 84 +++++++++++++++---- 3 files changed, 90 insertions(+), 20 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index e4287223b15..2e8431938d1 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -119,6 +119,11 @@ Other ======================= Lucene 6.7.0 ======================= +New Features + +* LUCENE-7855: Added advanced options of the Wikipedia tokenizer to its factory. + (Juan Pedro via Adrien Grand) + Other * LUCENE-7800: Remove code that potentially rethrows checked exceptions diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java index 3a570967c8f..83e08aa4d4f 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java @@ -16,9 +16,9 @@ */ package org.apache.lucene.analysis.wikipedia; - import java.util.Collections; import java.util.Map; +import java.util.Set; import org.apache.lucene.analysis.util.TokenizerFactory; import org.apache.lucene.util.AttributeFactory; @@ -33,19 +33,28 @@ import org.apache.lucene.util.AttributeFactory; * </fieldType> */ public class WikipediaTokenizerFactory extends TokenizerFactory { - + public static final String TOKEN_OUTPUT = "tokenOutput"; + public static final String UNTOKENIZED_TYPES = "untokenizedTypes"; + + protected final int tokenOutput; + protected Set untokenizedTypes; + /** Creates a new WikipediaTokenizerFactory */ public WikipediaTokenizerFactory(Map args) { super(args); + tokenOutput = getInt(args, TOKEN_OUTPUT, WikipediaTokenizer.TOKENS_ONLY); + untokenizedTypes = getSet(args, UNTOKENIZED_TYPES); + + if (untokenizedTypes == null) { + untokenizedTypes = Collections.emptySet(); + } if (!args.isEmpty()) { throw new IllegalArgumentException("Unknown parameters: " + args); } } - - // TODO: add support for WikipediaTokenizer's advanced options. + @Override public WikipediaTokenizer create(AttributeFactory factory) { - return new WikipediaTokenizer(factory, WikipediaTokenizer.TOKENS_ONLY, - Collections.emptySet()); + return new WikipediaTokenizer(factory, tokenOutput, untokenizedTypes); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java index ec345f9bcdc..74396205c1a 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java @@ -17,34 +17,90 @@ package org.apache.lucene.analysis.wikipedia; -import java.io.Reader; import java.io.StringReader; +import java.util.HashSet; +import java.util.Set; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase; -import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer; /** * Simple tests to ensure the wikipedia tokenizer is working. */ public class TestWikipediaTokenizerFactory extends BaseTokenStreamFactoryTestCase { + + private final String WIKIPEDIA = "Wikipedia"; + private final String TOKEN_OUTPUT = "tokenOutput"; + private final String UNTOKENIZED_TYPES = "untokenizedTypes"; + public void testTokenizer() throws Exception { - Reader reader = new StringReader("This is a [[Category:foo]]"); - Tokenizer tokenizer = tokenizerFactory("Wikipedia").create(newAttributeFactory()); - tokenizer.setReader(reader); - assertTokenStreamContents(tokenizer, - new String[] { "This", "is", "a", "foo" }, - new int[] { 0, 5, 8, 21 }, - new int[] { 4, 7, 9, 24 }, - new String[] { "", "", "", WikipediaTokenizer.CATEGORY }, - new int[] { 1, 1, 1, 1, }); + String text = "This is a [[Category:foo]]"; + Tokenizer tf = tokenizerFactory(WIKIPEDIA).create(newAttributeFactory()); + tf.setReader(new StringReader(text)); + assertTokenStreamContents(tf, + new String[] { "This", "is", "a", "foo" }, + new int[] { 0, 5, 8, 21 }, + new int[] { 4, 7, 9, 24 }, + new String[] { "", "", "", WikipediaTokenizer.CATEGORY }, + new int[] { 1, 1, 1, 1, }, + text.length()); } - + + public void testTokenizerTokensOnly() throws Exception { + String text = "This is a [[Category:foo]]"; + Tokenizer tf = tokenizerFactory(WIKIPEDIA, TOKEN_OUTPUT, new Integer( WikipediaTokenizer.TOKENS_ONLY).toString()).create(newAttributeFactory()); + tf.setReader(new StringReader(text)); + assertTokenStreamContents(tf, + new String[] { "This", "is", "a", "foo" }, + new int[] { 0, 5, 8, 21 }, + new int[] { 4, 7, 9, 24 }, + new String[] { "", "", "", WikipediaTokenizer.CATEGORY }, + new int[] { 1, 1, 1, 1, }, + text.length()); + } + + public void testTokenizerUntokenizedOnly() throws Exception { + String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]"; + Set untoks = new HashSet<>(); + untoks.add(WikipediaTokenizer.CATEGORY); + untoks.add(WikipediaTokenizer.ITALICS); + Tokenizer tf = tokenizerFactory(WIKIPEDIA, TOKEN_OUTPUT, new Integer(WikipediaTokenizer.UNTOKENIZED_ONLY).toString(), UNTOKENIZED_TYPES, WikipediaTokenizer.CATEGORY + ", " + WikipediaTokenizer.ITALICS).create(newAttributeFactory()); + tf.setReader(new StringReader(test)); + assertTokenStreamContents(tf, + new String[] { "a b c d", "e f g", "link", "here", "link", + "there", "italics here", "something", "more italics", "h i j" }, + new int[] { 11, 32, 42, 47, 56, 61, 71, 86, 98, 124 }, + new int[] { 18, 37, 46, 51, 60, 66, 83, 95, 110, 133 }, + new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } + ); + } + + public void testTokenizerBoth() throws Exception { + String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]"; + Tokenizer tf = tokenizerFactory(WIKIPEDIA, TOKEN_OUTPUT, new Integer(WikipediaTokenizer.BOTH).toString(), UNTOKENIZED_TYPES, WikipediaTokenizer.CATEGORY + ", " + WikipediaTokenizer.ITALICS).create(newAttributeFactory()); + tf.setReader(new StringReader(test)); + assertTokenStreamContents(tf, + new String[] { "a b c d", "a", "b", "c", "d", "e f g", "e", "f", "g", + "link", "here", "link", "there", "italics here", "italics", "here", + "something", "more italics", "more", "italics", "h i j", "h", "i", "j" }, + new int[] { 11, 11, 13, 15, 17, 32, 32, 34, 36, 42, 47, 56, 61, 71, 71, 79, 86, 98, 98, 103, 124, 124, 128, 132 }, + new int[] { 18, 12, 14, 16, 18, 37, 33, 35, 37, 46, 51, 60, 66, 83, 78, 83, 95, 110, 102, 110, 133, 125, 129, 133 }, + new int[] { 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1 } + ); + } + /** Test that bogus arguments result in exception */ public void testBogusArguments() throws Exception { IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - tokenizerFactory("Wikipedia", "bogusArg", "bogusValue"); + tokenizerFactory(WIKIPEDIA, "bogusArg", "bogusValue").create(newAttributeFactory()); }); assertTrue(expected.getMessage().contains("Unknown parameters")); } -} + + public void testIllegalArguments() throws Exception { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + Tokenizer tf = tokenizerFactory(WIKIPEDIA, TOKEN_OUTPUT, "-1").create(newAttributeFactory()); + }); + assertTrue(expected.getMessage().contains("tokenOutput must be TOKENS_ONLY, UNTOKENIZED_ONLY or BOTH")); + } +} \ No newline at end of file From b25dda0b209cddaf6165042dc0e860068e9aacc5 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 7 Jun 2017 13:26:18 +0100 Subject: [PATCH 013/131] SOLR-10800: Factor out HttpShardHandler.transformReplicasToShardUrls from HttpShardHandler.prepDistributed. (Domenico Fabio Marino, Christine Poerschke) --- solr/CHANGES.txt | 3 ++ .../handler/component/HttpShardHandler.java | 31 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d7eb46f41ea..7e45c3bcdfd 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -253,6 +253,9 @@ Other Changes * SOLR-10419: All collection APIs should use the new Policy framework for replica placement. (Noble Paul, shalin) +* SOLR-10800: Factor out HttpShardHandler.transformReplicasToShardUrls from HttpShardHandler.prepDistributed. + (Domenico Fabio Marino, Christine Poerschke) + ================== 6.7.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index 2954cff54da..1c98f5814c1 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -379,10 +379,11 @@ public class HttpShardHandler extends ShardHandler { for (int i=0; i shardUrls; if (rb.shards[i] != null) { - shardUrls = StrUtils.splitSmart(rb.shards[i], "|", true); + final List shardUrls = StrUtils.splitSmart(rb.shards[i], "|", true); replicaListTransformer.transform(shardUrls); + // And now recreate the | delimited list of equivalent servers + rb.shards[i] = createSliceShardsStr(shardUrls); } else { if (clusterState == null) { clusterState = zkController.getClusterState(); @@ -424,15 +425,11 @@ public class HttpShardHandler extends ShardHandler { final List eligibleSliceReplicas = collectEligibleReplicas(slice, clusterState, onlyNrtReplicas, isShardLeader); - replicaListTransformer.transform(eligibleSliceReplicas); + final List shardUrls = transformReplicasToShardUrls(replicaListTransformer, eligibleSliceReplicas); - shardUrls = new ArrayList<>(eligibleSliceReplicas.size()); - for (Replica replica : eligibleSliceReplicas) { - String url = ZkCoreNodeProps.getCoreUrl(replica); - shardUrls.add(url); - } - - if (shardUrls.isEmpty()) { + // And now recreate the | delimited list of equivalent servers + final String sliceShardsStr = createSliceShardsStr(shardUrls); + if (sliceShardsStr.isEmpty()) { boolean tolerant = rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false); if (!tolerant) { // stop the check when there are no replicas available for a shard @@ -440,9 +437,8 @@ public class HttpShardHandler extends ShardHandler { "no servers hosting shard: " + rb.slices[i]); } } + rb.shards[i] = sliceShardsStr; } - // And now recreate the | delimited list of equivalent servers - rb.shards[i] = createSliceShardsStr(shardUrls); } } String shards_rows = params.get(ShardParams.SHARDS_ROWS); @@ -475,6 +471,17 @@ public class HttpShardHandler extends ShardHandler { return eligibleSliceReplicas; } + private static List transformReplicasToShardUrls(final ReplicaListTransformer replicaListTransformer, final List eligibleSliceReplicas) { + replicaListTransformer.transform(eligibleSliceReplicas); + + final List shardUrls = new ArrayList<>(eligibleSliceReplicas.size()); + for (Replica replica : eligibleSliceReplicas) { + String url = ZkCoreNodeProps.getCoreUrl(replica); + shardUrls.add(url); + } + return shardUrls; + } + private static String createSliceShardsStr(final List shardUrls) { final StringBuilder sliceShardsStr = new StringBuilder(); boolean first = true; From 528899d845cc9ac73622cc0775667bd0c52cc694 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 7 Jun 2017 17:49:33 +0200 Subject: [PATCH 014/131] LUCENE-7828: Speed up range queries on range fields by improving how we compute the relation between the query and inner nodes of the BKD tree. --- lucene/CHANGES.txt | 6 + .../lucene/document/RangeFieldQuery.java | 377 +++++++++++------- 2 files changed, 232 insertions(+), 151 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 2e8431938d1..1e89390aa90 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -134,6 +134,12 @@ Improvements * LUCENE-7841: Normalize ґ to г in Ukrainian analyzer. (Andriy Rysin via Dawid Weiss) +Optimizations + +* LUCENE-7828: Speed up range queries on range fields by improving how we + compute the relation between the query and inner nodes of the BKD tree. + (Adrien Grand) + ======================= Lucene 6.6.0 ======================= New Features diff --git a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java index 750189d1507..fd3da1e6efd 100644 --- a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java @@ -19,22 +19,20 @@ package org.apache.lucene.document; import java.io.IOException; import java.util.Arrays; import java.util.Objects; -import java.util.function.IntPredicate; -import java.util.function.Predicate; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; -import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.index.PointValues.IntersectVisitor; +import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; import org.apache.lucene.search.Weight; import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.StringHelper; @@ -60,13 +58,167 @@ abstract class RangeFieldQuery extends Query { /** Used by {@code RangeFieldQuery} to check how each internal or leaf node relates to the query. */ enum QueryType { /** Use this for intersects queries. */ - INTERSECTS, + INTERSECTS { + + @Override + Relation compare(byte[] queryPackedValue, byte[] minPackedValue, byte[] maxPackedValue, + int numDims, int bytesPerDim, int dim) { + int minOffset = dim * bytesPerDim; + int maxOffset = minOffset + bytesPerDim * numDims; + + if (StringHelper.compare(bytesPerDim, queryPackedValue, maxOffset, minPackedValue, minOffset) < 0 + || StringHelper.compare(bytesPerDim, queryPackedValue, minOffset, maxPackedValue, maxOffset) > 0) { + // disjoint + return Relation.CELL_OUTSIDE_QUERY; + } + + if (StringHelper.compare(bytesPerDim, queryPackedValue, maxOffset, maxPackedValue, minOffset) >= 0 + && StringHelper.compare(bytesPerDim, queryPackedValue, minOffset, minPackedValue, maxOffset) <= 0) { + return Relation.CELL_INSIDE_QUERY; + } + + return Relation.CELL_CROSSES_QUERY; + } + + @Override + boolean matches(byte[] queryPackedValue, byte[] packedValue, int numDims, int bytesPerDim, int dim) { + int minOffset = dim * bytesPerDim; + int maxOffset = minOffset + bytesPerDim * numDims; + return StringHelper.compare(bytesPerDim, queryPackedValue, maxOffset, packedValue, minOffset) >= 0 + && StringHelper.compare(bytesPerDim, queryPackedValue, minOffset, packedValue, maxOffset) <= 0; + } + + }, /** Use this for within queries. */ - WITHIN, + WITHIN { + + @Override + Relation compare(byte[] queryPackedValue, byte[] minPackedValue, byte[] maxPackedValue, + int numDims, int bytesPerDim, int dim) { + int minOffset = dim * bytesPerDim; + int maxOffset = minOffset + bytesPerDim * numDims; + + if (StringHelper.compare(bytesPerDim, queryPackedValue, maxOffset, minPackedValue, maxOffset) < 0 + || StringHelper.compare(bytesPerDim, queryPackedValue, minOffset, maxPackedValue, minOffset) > 0) { + // all ranges have at least one point outside of the query + return Relation.CELL_OUTSIDE_QUERY; + } + + if (StringHelper.compare(bytesPerDim, queryPackedValue, maxOffset, maxPackedValue, maxOffset) >= 0 + && StringHelper.compare(bytesPerDim, queryPackedValue, minOffset, minPackedValue, minOffset) <= 0) { + return Relation.CELL_INSIDE_QUERY; + } + + return Relation.CELL_CROSSES_QUERY; + } + + @Override + boolean matches(byte[] queryPackedValue, byte[] packedValue, int numDims, int bytesPerDim, int dim) { + int minOffset = dim * bytesPerDim; + int maxOffset = minOffset + bytesPerDim * numDims; + return StringHelper.compare(bytesPerDim, queryPackedValue, minOffset, packedValue, minOffset) <= 0 + && StringHelper.compare(bytesPerDim, queryPackedValue, maxOffset, packedValue, maxOffset) >= 0; + } + + }, /** Use this for contains */ - CONTAINS, + CONTAINS { + + @Override + Relation compare(byte[] queryPackedValue, byte[] minPackedValue, byte[] maxPackedValue, + int numDims, int bytesPerDim, int dim) { + int minOffset = dim * bytesPerDim; + int maxOffset = minOffset + bytesPerDim * numDims; + + if (StringHelper.compare(bytesPerDim, queryPackedValue, maxOffset, maxPackedValue, maxOffset) > 0 + || StringHelper.compare(bytesPerDim, queryPackedValue, minOffset, minPackedValue, minOffset) < 0) { + // all ranges are either less than the query max or greater than the query min + return Relation.CELL_OUTSIDE_QUERY; + } + + if (StringHelper.compare(bytesPerDim, queryPackedValue, maxOffset, minPackedValue, maxOffset) <= 0 + && StringHelper.compare(bytesPerDim, queryPackedValue, minOffset, maxPackedValue, minOffset) >= 0) { + return Relation.CELL_INSIDE_QUERY; + } + + return Relation.CELL_CROSSES_QUERY; + } + + @Override + boolean matches(byte[] queryPackedValue, byte[] packedValue, int numDims, int bytesPerDim, int dim) { + int minOffset = dim * bytesPerDim; + int maxOffset = minOffset + bytesPerDim * numDims; + return StringHelper.compare(bytesPerDim, queryPackedValue, minOffset, packedValue, minOffset) >= 0 + && StringHelper.compare(bytesPerDim, queryPackedValue, maxOffset, packedValue, maxOffset) <= 0; + } + + }, /** Use this for crosses queries */ - CROSSES + CROSSES { + + @Override + Relation compare(byte[] queryPackedValue, byte[] minPackedValue, byte[] maxPackedValue, + int numDims, int bytesPerDim, int dim) { + throw new UnsupportedOperationException(); + } + + @Override + boolean matches(byte[] queryPackedValue, byte[] packedValue, int numDims, int bytesPerDim, int dim) { + throw new UnsupportedOperationException(); + } + + @Override + Relation compare(byte[] queryPackedValue, byte[] minPackedValue, byte[] maxPackedValue, + int numDims, int bytesPerDim) { + Relation intersectRelation = QueryType.INTERSECTS.compare(queryPackedValue, minPackedValue, maxPackedValue, numDims, bytesPerDim); + if (intersectRelation == Relation.CELL_OUTSIDE_QUERY) { + return Relation.CELL_OUTSIDE_QUERY; + } + + Relation withinRelation = QueryType.WITHIN.compare(queryPackedValue, minPackedValue, maxPackedValue, numDims, bytesPerDim); + if (withinRelation == Relation.CELL_INSIDE_QUERY) { + return Relation.CELL_OUTSIDE_QUERY; + } + + if (intersectRelation == Relation.CELL_INSIDE_QUERY && withinRelation == Relation.CELL_OUTSIDE_QUERY) { + return Relation.CELL_INSIDE_QUERY; + } + + return Relation.CELL_CROSSES_QUERY; + } + + boolean matches(byte[] queryPackedValue, byte[] packedValue, int numDims, int bytesPerDim) { + return INTERSECTS.matches(queryPackedValue, packedValue, numDims, bytesPerDim) + && WITHIN.matches(queryPackedValue, packedValue, numDims, bytesPerDim) == false; + } + + }; + + abstract Relation compare(byte[] queryPackedValue, byte[] minPackedValue, byte[] maxPackedValue, int numDims, int bytesPerDim, int dim); + + Relation compare(byte[] queryPackedValue, byte[] minPackedValue, byte[] maxPackedValue, int numDims, int bytesPerDim) { + boolean inside = true; + for (int dim = 0; dim < numDims; ++dim) { + Relation relation = compare(queryPackedValue, minPackedValue, maxPackedValue, numDims, bytesPerDim, dim); + if (relation == Relation.CELL_OUTSIDE_QUERY) { + return Relation.CELL_OUTSIDE_QUERY; + } else if (relation != Relation.CELL_INSIDE_QUERY) { + inside = false; + } + } + return inside ? Relation.CELL_INSIDE_QUERY : Relation.CELL_CROSSES_QUERY; + } + + abstract boolean matches(byte[] queryPackedValue, byte[] packedValue, int numDims, int bytesPerDim, int dim); + + boolean matches(byte[] queryPackedValue, byte[] packedValue, int numDims, int bytesPerDim) { + for (int dim = 0; dim < numDims; ++dim) { + if (matches(queryPackedValue, packedValue, numDims, bytesPerDim, dim) == false) { + return false; + } + } + return true; + } } /** @@ -111,54 +263,33 @@ abstract class RangeFieldQuery extends Query { @Override public final Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - final RangeFieldComparator target = new RangeFieldComparator(); - private DocIdSet buildMatchingDocIdSet(LeafReader reader, PointValues values) throws IOException { - DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field); - values.intersect( - new IntersectVisitor() { - DocIdSetBuilder.BulkAdder adder; - @Override - public void grow(int count) { - adder = result.grow(count); - } - @Override - public void visit(int docID) throws IOException { - adder.add(docID); - } - @Override - public void visit(int docID, byte[] leaf) throws IOException { - if (target.matches(leaf)) { - adder.add(docID); - } - } - @Override - public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - return compareRange(minPackedValue, maxPackedValue); - } - }); - return result.build(); - } - - private Relation compareRange(byte[] minPackedValue, byte[] maxPackedValue) { - byte[] node = getInternalRange(minPackedValue, maxPackedValue); - // compute range relation for BKD traversal - if (target.intersects(node) == false) { - return Relation.CELL_OUTSIDE_QUERY; - } else if (target.within(node)) { - // target within cell; continue traversing: - return Relation.CELL_CROSSES_QUERY; - } else if (target.contains(node)) { - // target contains cell; add iff queryType is not a CONTAINS or CROSSES query: - return (queryType == QueryType.CONTAINS || queryType == QueryType.CROSSES) ? - Relation.CELL_OUTSIDE_QUERY : Relation.CELL_INSIDE_QUERY; - } - // target intersects cell; continue traversing: - return Relation.CELL_CROSSES_QUERY; + private IntersectVisitor getIntersectVisitor(DocIdSetBuilder result) { + return new IntersectVisitor() { + DocIdSetBuilder.BulkAdder adder; + @Override + public void grow(int count) { + adder = result.grow(count); + } + @Override + public void visit(int docID) throws IOException { + adder.add(docID); + } + @Override + public void visit(int docID, byte[] leaf) throws IOException { + if (queryType.matches(ranges, leaf, numDims, bytesPerDim)) { + adder.add(docID); + } + } + @Override + public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + return queryType.compare(ranges, minPackedValue, maxPackedValue, numDims, bytesPerDim); + } + }; } @Override - public Scorer scorer(LeafReaderContext context) throws IOException { + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); PointValues values = reader.getPointValues(field); if (values == null) { @@ -173,115 +304,59 @@ abstract class RangeFieldQuery extends Query { checkFieldInfo(fieldInfo); boolean allDocsMatch = false; if (values.getDocCount() == reader.maxDoc() - && compareRange(values.getMinPackedValue(), values.getMaxPackedValue()) == Relation.CELL_INSIDE_QUERY) { + && queryType.compare(ranges, values.getMinPackedValue(), values.getMaxPackedValue(), numDims, bytesPerDim) == Relation.CELL_INSIDE_QUERY) { allDocsMatch = true; } - DocIdSetIterator iterator = allDocsMatch == true ? - DocIdSetIterator.all(reader.maxDoc()) : buildMatchingDocIdSet(reader, values).iterator(); - return new ConstantScoreScorer(this, score(), iterator); - } + final Weight weight = this; + if (allDocsMatch) { + return new ScorerSupplier() { + @Override + public Scorer get(boolean randomAccess) { + return new ConstantScoreScorer(weight, score(), DocIdSetIterator.all(reader.maxDoc())); + } - /** get an encoded byte representation of the internal node; this is - * the lower half of the min array and the upper half of the max array */ - private byte[] getInternalRange(byte[] min, byte[] max) { - byte[] range = new byte[min.length]; - final int dimSize = numDims * bytesPerDim; - System.arraycopy(min, 0, range, 0, dimSize); - System.arraycopy(max, dimSize, range, dimSize, dimSize); - return range; - } - }; - } + @Override + public long cost() { + return reader.maxDoc(); + } + }; + } else { + return new ScorerSupplier() { - /** - * RangeFieldComparator class provides the core comparison logic for accepting or rejecting indexed - * {@code RangeField} types based on the defined query range and relation. - */ - class RangeFieldComparator { - final Predicate predicate; + final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field); + final IntersectVisitor visitor = getIntersectVisitor(result); + long cost = -1; - /** constructs the comparator based on the query type */ - RangeFieldComparator() { - switch (queryType) { - case INTERSECTS: - predicate = this::intersects; - break; - case WITHIN: - predicate = this::contains; - break; - case CONTAINS: - predicate = this::within; - break; - case CROSSES: - // crosses first checks intersection (disjoint automatic fails), - // then ensures the query doesn't wholly contain the leaf: - predicate = (byte[] leaf) -> this.intersects(leaf) - && this.contains(leaf) == false; - break; - default: - throw new IllegalArgumentException("invalid queryType [" + queryType + "] found."); - } - } + @Override + public Scorer get(boolean randomAccess) throws IOException { + values.intersect(visitor); + DocIdSetIterator iterator = result.build().iterator(); + return new ConstantScoreScorer(weight, score(), iterator); + } - /** determines if the candidate range matches the query request */ - private boolean matches(final byte[] candidate) { - return (Arrays.equals(ranges, candidate) && queryType != QueryType.CROSSES) - || predicate.test(candidate); - } - - /** check if query intersects candidate range */ - private boolean intersects(final byte[] candidate) { - return relate((int d) -> compareMinMax(candidate, d) > 0 || compareMaxMin(candidate, d) < 0); - } - - /** check if query is within candidate range */ - private boolean within(final byte[] candidate) { - return relate((int d) -> compareMinMin(candidate, d) < 0 || compareMaxMax(candidate, d) > 0); - } - - /** check if query contains candidate range */ - private boolean contains(final byte[] candidate) { - return relate((int d) -> compareMinMin(candidate, d) > 0 || compareMaxMax(candidate, d) < 0); - } - - /** internal method used by each relation method to test range relation logic */ - private boolean relate(IntPredicate predicate) { - for (int d=0; d= 0; + } + return cost; + } + }; } } - return true; - } - /** compare the encoded min value (for the defined query dimension) with the encoded min value in the byte array */ - private int compareMinMin(byte[] b, int dimension) { - // convert dimension to offset: - dimension *= bytesPerDim; - return StringHelper.compare(bytesPerDim, ranges, dimension, b, dimension); - } - - /** compare the encoded min value (for the defined query dimension) with the encoded max value in the byte array */ - private int compareMinMax(byte[] b, int dimension) { - // convert dimension to offset: - dimension *= bytesPerDim; - return StringHelper.compare(bytesPerDim, ranges, dimension, b, numDims * bytesPerDim + dimension); - } - - /** compare the encoded max value (for the defined query dimension) with the encoded min value in the byte array */ - private int compareMaxMin(byte[] b, int dimension) { - // convert dimension to offset: - dimension *= bytesPerDim; - return StringHelper.compare(bytesPerDim, ranges, numDims * bytesPerDim + dimension, b, dimension); - } - - /** compare the encoded max value (for the defined query dimension) with the encoded max value in the byte array */ - private int compareMaxMax(byte[] b, int dimension) { - // convert dimension to max offset: - dimension = numDims * bytesPerDim + dimension * bytesPerDim; - return StringHelper.compare(bytesPerDim, ranges, dimension, b, dimension); - } + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + ScorerSupplier scorerSupplier = scorerSupplier(context); + if (scorerSupplier == null) { + return null; + } + return scorerSupplier.get(false); + } + }; } @Override From bcce49c160d356435f6e47a92172ae136ae49acf Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 7 Jun 2017 19:38:17 +0100 Subject: [PATCH 015/131] SOLR-10174: fix @Ignore in TestMultipleAdditiveTreesModel --- .../model/TestMultipleAdditiveTreesModel.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestMultipleAdditiveTreesModel.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestMultipleAdditiveTreesModel.java index 3c817d289de..edc0e24e251 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestMultipleAdditiveTreesModel.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestMultipleAdditiveTreesModel.java @@ -22,7 +22,6 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.ltr.TestRerankBase; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; public class TestMultipleAdditiveTreesModel extends TestRerankBase { @@ -47,10 +46,16 @@ public class TestMultipleAdditiveTreesModel extends TestRerankBase { @Test - public void testMultipleAdditiveTreesScoringWithAndWithoutEfiFeatureMatches() throws Exception { + public void testMultipleAdditiveTrees() throws Exception { loadFeatures("multipleadditivetreesmodel_features.json"); loadModels("multipleadditivetreesmodel.json"); + doTestMultipleAdditiveTreesScoringWithAndWithoutEfiFeatureMatches(); + doTestMultipleAdditiveTreesExplain(); + } + + private void doTestMultipleAdditiveTreesScoringWithAndWithoutEfiFeatureMatches() throws Exception { + final SolrQuery query = new SolrQuery(); query.setQuery("*:*"); query.add("rows", "3"); @@ -79,9 +84,8 @@ public class TestMultipleAdditiveTreesModel extends TestRerankBase { assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==-120.0"); } - @Ignore - @Test - public void multipleAdditiveTreesTestExplain() throws Exception { + private void doTestMultipleAdditiveTreesExplain() throws Exception { + final SolrQuery query = new SolrQuery(); query.setQuery("*:*"); query.add("fl", "*,score,[fv]"); @@ -103,7 +107,7 @@ public class TestMultipleAdditiveTreesModel extends TestRerankBase { qryResult = qryResult.substring(qryResult.indexOf("explain")); assertThat(qryResult, containsString("multipleadditivetreesmodel")); - assertThat(qryResult, containsString(MultipleAdditiveTreesModel.class.getCanonicalName())); + assertThat(qryResult, containsString(MultipleAdditiveTreesModel.class.getSimpleName())); assertThat(qryResult, containsString("-100.0 = tree 0")); assertThat(qryResult, containsString("50.0 = tree 0")); @@ -113,7 +117,6 @@ public class TestMultipleAdditiveTreesModel extends TestRerankBase { assertThat(qryResult, containsString(" Go Right ")); assertThat(qryResult, containsString(" Go Left ")); - assertThat(qryResult, containsString("'this_feature_doesnt_exist' does not exist in FV")); } @Test From 6c3ece2b9f97926769849b4008ed7f10276f0b14 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 7 Jun 2017 15:43:56 -0400 Subject: [PATCH 016/131] LUCENE-7854: restore the IllegalArgumentException if you index too many tokens in one field --- .../java/org/apache/lucene/index/DefaultIndexingChain.java | 6 +++++- .../test/org/apache/lucene/index/TestCustomTermFreq.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java index df739d85061..f2c3de11a4c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java +++ b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java @@ -770,7 +770,11 @@ final class DefaultIndexingChain extends DocConsumer { } invertState.lastStartOffset = startOffset; - invertState.length = Math.addExact(invertState.length, invertState.termFreqAttribute.getTermFrequency()); + try { + invertState.length = Math.addExact(invertState.length, invertState.termFreqAttribute.getTermFrequency()); + } catch (ArithmeticException ae) { + throw new IllegalArgumentException("too many tokens for field \"" + field.name() + "\""); + } //System.out.println(" term=" + invertState.termAttribute); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java index 5b38f576644..d2eff257648 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java @@ -303,7 +303,7 @@ public class TestCustomTermFreq extends LuceneTestCase { new int[] {3, Integer.MAX_VALUE}), fieldType); doc2.add(field); - expectThrows(ArithmeticException.class, () -> {w.addDocument(doc2);}); + expectThrows(IllegalArgumentException.class, () -> {w.addDocument(doc2);}); IndexReader r = DirectoryReader.open(w); assertEquals(1, r.numDocs()); From e4ab04ab7d6813c9e1ee7e0f835488ae68d5ab2f Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Wed, 7 Jun 2017 19:21:10 -0400 Subject: [PATCH 017/131] SOLR-10501: Test sortMissing{First,Last} with points fields. --- solr/CHANGES.txt | 2 + .../solr/collection1/conf/schema-point.xml | 64 +- .../apache/solr/schema/TestPointFields.java | 574 ++++++++++-------- 3 files changed, 374 insertions(+), 266 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 7e45c3bcdfd..dc17616213e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -385,6 +385,8 @@ Other Changes * SOLR-8762: return child docs in DIH debug (Gopikannan Venugopalsamy via Mikhail Khludnev) +* SOLR-10501: Test sortMissing{First,Last} with points fields. (Steve Rowe) + ================== 6.6.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-point.xml b/solr/core/src/test-files/solr/collection1/conf/schema-point.xml index 8bf545a44d8..c024cb6d0a7 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-point.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-point.xml @@ -60,6 +60,18 @@ + + + + + + + + + + + + @@ -73,7 +85,19 @@ - + + + + + + + + + + + + + @@ -86,7 +110,19 @@ - + + + + + + + + + + + + + @@ -99,6 +135,18 @@ + + + + + + + + + + + + @@ -112,6 +160,18 @@ + + + + + + + + + + + + diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java index b93b260ed91..a1d2260207d 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java +++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java @@ -18,9 +18,11 @@ package org.apache.solr.schema; import java.io.IOException; import java.text.SimpleDateFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashSet; import java.util.LinkedHashSet; @@ -29,6 +31,9 @@ import java.util.Locale; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoublePoint; @@ -71,6 +76,13 @@ import com.google.common.collect.ImmutableMap; */ public class TestPointFields extends SolrTestCaseJ4 { + private static final String[] FIELD_SUFFIXES = new String[] { + "", "_dv", "_mv", "_mv_dv", "_ni", "_ni_dv", "_ni_dv_ns", "_ni_dv_ns_mv", + "_ni_mv", "_ni_mv_dv", "_ni_ns", "_ni_ns_mv", "_dv_ns", "_ni_ns_dv", "_dv_ns_mv", + "_smf", "_dv_smf", "_mv_smf", "_mv_dv_smf", "_ni_dv_smf", "_ni_mv_dv_smf", + "_sml", "_dv_sml", "_mv_sml", "_mv_dv_sml", "_ni_dv_sml", "_ni_mv_dv_sml" + }; + @BeforeClass public static void beforeClass() throws Exception { initCore("solrconfig.xml","schema-point.xml"); @@ -126,30 +138,44 @@ public class TestPointFields extends SolrTestCaseJ4 { public void testIntPointFieldSortAndFunction() throws Exception { final SortedSet regexToTest = dynFieldRegexesForType(IntPointField.class); - final String[] sequential = new String[]{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}; + final List sequential = Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"); + final List randomInts = getRandomInts(10, false); + final List randomIntsMissing = getRandomInts(10, true); for (String r : Arrays.asList("*_p_i", "*_p_i_dv", "*_p_i_dv_ns", "*_p_i_ni_dv", "*_p_i_ni_dv_ns", "*_p_i_ni_ns_dv")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSort(r.replace("*","number"), sequential); - // TODO: test some randomly generated (then sorted) arrays (with dups and/or missing values) - - doTestIntPointFunctionQuery(r.replace("*","number"), "int"); + String field = r.replace("*", "number"); + doTestPointFieldSort(field, sequential); + doTestPointFieldSort(field, randomInts); + doTestIntPointFunctionQuery(field, "int"); + } + for (String r : Arrays.asList("*_p_i_smf", "*_p_i_dv_smf", "*_p_i_ni_dv_smf", + "*_p_i_sml", "*_p_i_dv_sml", "*_p_i_ni_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestPointFieldSort(field, sequential); + doTestPointFieldSort(field, randomIntsMissing); + doTestIntPointFunctionQuery(field, "int"); } for (String r : Arrays.asList("*_p_i_ni", "*_p_i_ni_ns")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSortError(r.replace("*","number"), "w/o docValues", "42"); - doTestPointFieldFunctionQueryError(r.replace("*","number"), "w/o docValues", "42"); + String field = r.replace("*", "number"); + doTestPointFieldSortError(field, "w/o docValues", "42"); + doTestPointFieldFunctionQueryError(field, "w/o docValues", "42"); } for (String r : Arrays.asList("*_p_i_mv", "*_p_i_ni_mv", "*_p_i_ni_mv_dv", "*_p_i_ni_dv_ns_mv", - "*_p_i_ni_ns_mv", "*_p_i_dv_ns_mv", "*_p_i_mv_dv")) { + "*_p_i_ni_ns_mv", "*_p_i_dv_ns_mv", "*_p_i_mv_dv", + "*_p_i_mv_smf", "*_p_i_mv_dv_smf", "*_p_i_ni_mv_dv_smf", + "*_p_i_mv_sml", "*_p_i_mv_dv_sml", "*_p_i_ni_mv_dv_sml")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSortError(r.replace("*","number"), "multivalued", "42"); - doTestPointFieldSortError(r.replace("*","number"), "multivalued", "42", "666"); - doTestPointFieldFunctionQueryError(r.replace("*","number"), "multivalued", "42"); - doTestPointFieldFunctionQueryError(r.replace("*","number"), "multivalued", "42", "666"); + String field = r.replace("*", "number"); + doTestPointFieldSortError(field, "multivalued", "42"); + doTestPointFieldSortError(field, "multivalued", "42", "666"); + doTestPointFieldFunctionQueryError(field, "multivalued", "42"); + doTestPointFieldFunctionQueryError(field, "multivalued", "42", "666"); } assertEquals("Missing types in the test", Collections.emptySet(), regexToTest); @@ -209,7 +235,7 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testIntPointFieldMultiValuedFacetField() throws Exception { testPointFieldMultiValuedFacetField("number_p_i_mv", "number_p_i_mv_dv", getSequentialStringArrayWithInts(20)); - testPointFieldMultiValuedFacetField("number_p_i_mv", "number_p_i_mv_dv", getRandomStringArrayWithInts(20, false)); + testPointFieldMultiValuedFacetField("number_p_i_mv", "number_p_i_mv_dv", toStringArray(getRandomInts(20, false))); } @Test @@ -241,12 +267,78 @@ public class TestPointFields extends SolrTestCaseJ4 { testMultiValuedIntPointFieldsAtomicUpdates("number_p_i_ni_mv_dv", "int"); testMultiValuedIntPointFieldsAtomicUpdates("number_p_i_dv_ns_mv", "int"); } - + + private String[] toStringArray(List list) { + return list.stream().map(String::valueOf).collect(Collectors.toList()).toArray(new String[list.size()]); + } + + private class PosVal > { + int pos; + T val; + + PosVal(int pos, T val) { + this.pos = pos; + this.val = val; + } + } + + /** Primary sort by value, with nulls either first or last as specified, and then secondary sort by position. */ + private > + Comparator> getPosValComparator(final boolean ascending, final boolean nullsFirst) { + return (o1, o2) -> { + if (o1.val == null) { + if (o2.val == null) { + return ascending ? Integer.compare(o1.pos, o2.pos) : Integer.compare(o2.pos, o1.pos); + } else { + return nullsFirst ? -1 : 1; + } + } else if (o2.val == null) { + return nullsFirst ? 1 : -1; + } else { + return ascending ? o1.val.compareTo(o2.val) : o2.val.compareTo(o1.val); + } + }; + } + + /** + * Primary ascending sort by value, with missing values (represented as null) either first or last as specified, + * and then secondary ascending sort by position. + */ + private > String[] toAscendingStringArray(List list, boolean missingFirst) { + return toStringArray(toAscendingPosVals(list, missingFirst).stream().map(pv -> pv.val).collect(Collectors.toList())); + } + + /** + * Primary ascending sort by value, with missing values (represented as null) either first or last as specified, + * and then secondary ascending sort by position. + * + * @return a list of the (originally) positioned values sorted as described above. + */ + private > List> toAscendingPosVals(List list, boolean missingFirst) { + List> posVals = IntStream.range(0, list.size()) + .mapToObj(i -> new PosVal<>(i, list.get(i))).collect(Collectors.toList()); + posVals.sort(getPosValComparator(true, missingFirst)); + return posVals; + } + + /** + * Primary descending sort by value, with missing values (represented as null) either first or last as specified, + * and then secondary descending sort by position. + * + * @return a list of the (originally) positioned values sorted as described above. + */ + private > List> toDescendingPosVals(List list, boolean missingFirst) { + List> posVals = IntStream.range(0, list.size()) + .mapToObj(i -> new PosVal<>(i, list.get(i))).collect(Collectors.toList()); + posVals.sort(getPosValComparator(false, missingFirst)); + return posVals; + } + @Test public void testIntPointSetQuery() throws Exception { - doTestSetQueries("number_p_i", getRandomStringArrayWithInts(20, false), false); - doTestSetQueries("number_p_i_mv", getRandomStringArrayWithInts(20, false), true); - doTestSetQueries("number_p_i_ni_dv", getRandomStringArrayWithInts(20, false), false); + doTestSetQueries("number_p_i", toStringArray(getRandomInts(20, false)), false); + doTestSetQueries("number_p_i_mv", toStringArray(getRandomInts(20, false)), true); + doTestSetQueries("number_p_i_ni_dv", toStringArray(getRandomInts(20, false)), false); } // DoublePointField @@ -300,38 +392,48 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testDoublePointFieldSortAndFunction() throws Exception { final SortedSet regexToTest = dynFieldRegexesForType(DoublePointField.class); - final String[] sequential = new String[]{"0.0", "1.0", "2.0", "3.0", "4.0", "5.0", "6.0", "7.0", "8.0", "9.0"}; - final String[] randstrs = getRandomStringArrayWithDoubles(10, true); + final List sequential = Arrays.asList("0.0", "1.0", "2.0", "3.0", "4.0", "5.0", "6.0", "7.0", "8.0", "9.0"); + List randomDoubles = getRandomDoubles(10, false); + List randomDoublesMissing = getRandomDoubles(10, true); for (String r : Arrays.asList("*_p_d", "*_p_d_dv", "*_p_d_dv_ns", "*_p_d_ni_dv", "*_p_d_ni_dv_ns", "*_p_d_ni_ns_dv")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSort(r.replace("*","number"), sequential); - doTestPointFieldSort(r.replace("*","number"), randstrs); - // TODO: test some randomly generated (then sorted) arrays (with dups and/or missing values) + String field = r.replace("*", "number"); + doTestPointFieldSort(field, sequential); + doTestPointFieldSort(field, randomDoubles); + doTestFloatPointFunctionQuery(field, "double"); + } - doTestFloatPointFunctionQuery(r.replace("*","number"), "double"); + for (String r : Arrays.asList("*_p_d_smf", "*_p_d_dv_smf", "*_p_d_ni_dv_smf", + "*_p_d_sml", "*_p_d_dv_sml", "*_p_d_ni_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestPointFieldSort(field, sequential); + doTestPointFieldSort(field, randomDoublesMissing); + doTestFloatPointFunctionQuery(field, "double"); } for (String r : Arrays.asList("*_p_d_ni", "*_p_d_ni_ns")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSortError(r.replace("*","number"), "w/o docValues", "42.34"); - - doTestPointFieldFunctionQueryError(r.replace("*","number"), "w/o docValues", "42.34"); + String field = r.replace("*", "number"); + doTestPointFieldSortError(field, "w/o docValues", "42.34"); + doTestPointFieldFunctionQueryError(field, "w/o docValues", "42.34"); } for (String r : Arrays.asList("*_p_d_mv", "*_p_d_ni_mv", "*_p_d_ni_mv_dv", "*_p_d_ni_dv_ns_mv", - "*_p_d_ni_ns_mv", "*_p_d_dv_ns_mv", "*_p_d_mv_dv")) { + "*_p_d_ni_ns_mv", "*_p_d_dv_ns_mv", "*_p_d_mv_dv", + "*_p_d_mv_smf", "*_p_d_mv_dv_smf", "*_p_d_ni_mv_dv_smf", + "*_p_d_mv_sml", "*_p_d_mv_dv_sml", "*_p_d_ni_mv_dv_sml")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSortError(r.replace("*","number"), "multivalued", "42.34"); - doTestPointFieldSortError(r.replace("*","number"), "multivalued", "42.34", "66.6"); - - doTestPointFieldFunctionQueryError(r.replace("*","number"), "multivalued", "42.34"); - doTestPointFieldFunctionQueryError(r.replace("*","number"), "multivalued", "42.34", "66.6"); + String field = r.replace("*", "number"); + doTestPointFieldSortError(field, "multivalued", "42.34"); + doTestPointFieldSortError(field, "multivalued", "42.34", "66.6"); + doTestPointFieldFunctionQueryError(field, "multivalued", "42.34"); + doTestPointFieldFunctionQueryError(field, "multivalued", "42.34", "66.6"); } assertEquals("Missing types in the test", Collections.emptySet(), regexToTest); - } @Test @@ -339,7 +441,7 @@ public class TestPointFields extends SolrTestCaseJ4 { testPointFieldFacetField("number_p_d", "number_p_d_dv", getSequentialStringArrayWithDoubles(10)); clearIndex(); assertU(commit()); - testPointFieldFacetField("number_p_d", "number_p_d_dv", getRandomStringArrayWithDoubles(10, false)); + testPointFieldFacetField("number_p_d", "number_p_d_dv", toStringArray(getRandomDoubles(10, false))); } @Test @@ -357,14 +459,14 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testDoublePointFieldMultiValuedExactQuery() throws Exception { - testPointFieldMultiValuedExactQuery("number_p_d_mv", getRandomStringArrayWithDoubles(20, false)); - testPointFieldMultiValuedExactQuery("number_p_d_ni_mv_dv", getRandomStringArrayWithDoubles(20, false)); + testPointFieldMultiValuedExactQuery("number_p_d_mv", toStringArray(getRandomDoubles(20, false))); + testPointFieldMultiValuedExactQuery("number_p_d_ni_mv_dv", toStringArray(getRandomDoubles(20, false))); } @Test public void testDoublePointFieldMultiValuedNonSearchableExactQuery() throws Exception { - testPointFieldMultiValuedExactQuery("number_p_d_ni_mv", getRandomStringArrayWithDoubles(20, false), false); - testPointFieldMultiValuedExactQuery("number_p_d_ni_ns_mv", getRandomStringArrayWithDoubles(20, false), false); + testPointFieldMultiValuedExactQuery("number_p_d_ni_mv", toStringArray(getRandomDoubles(20, false)), false); + testPointFieldMultiValuedExactQuery("number_p_d_ni_ns_mv", toStringArray(getRandomDoubles(20, false)), false); } @Test @@ -384,7 +486,7 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testDoublePointFieldMultiValuedFacetField() throws Exception { testPointFieldMultiValuedFacetField("number_p_d_mv", "number_p_d_mv_dv", getSequentialStringArrayWithDoubles(20)); - testPointFieldMultiValuedFacetField("number_p_d_mv", "number_p_d_mv_dv", getRandomStringArrayWithDoubles(20, false)); + testPointFieldMultiValuedFacetField("number_p_d_mv", "number_p_d_mv_dv", toStringArray(getRandomDoubles(20, false))); } @Test @@ -395,7 +497,7 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testDoublePointMultiValuedFunctionQuery() throws Exception { testPointMultiValuedFunctionQuery("number_p_d_mv", "number_p_d_mv_dv", "double", getSequentialStringArrayWithDoubles(20)); - testPointMultiValuedFunctionQuery("number_p_d_mv", "number_p_d_mv_dv", "double", getRandomStringArrayWithFloats(20, true)); + testPointMultiValuedFunctionQuery("number_p_d_mv", "number_p_d_mv_dv", "double", toAscendingStringArray(getRandomFloats(20, false), true)); } @Test @@ -460,9 +562,9 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testDoublePointSetQuery() throws Exception { - doTestSetQueries("number_p_d", getRandomStringArrayWithDoubles(20, false), false); - doTestSetQueries("number_p_d_mv", getRandomStringArrayWithDoubles(20, false), true); - doTestSetQueries("number_p_d_ni_dv", getRandomStringArrayWithDoubles(20, false), false); + doTestSetQueries("number_p_d", toStringArray(getRandomDoubles(20, false)), false); + doTestSetQueries("number_p_d_mv", toStringArray(getRandomDoubles(20, false)), true); + doTestSetQueries("number_p_d_ni_dv", toStringArray(getRandomDoubles(20, false)), false); } // Float @@ -514,38 +616,48 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testFloatPointFieldSortAndFunction() throws Exception { final SortedSet regexToTest = dynFieldRegexesForType(FloatPointField.class); - final String[] sequential = new String[]{"0.0", "1.0", "2.0", "3.0", "4.0", "5.0", "6.0", "7.0", "8.0", "9.0"}; - final String[] randstrs = getRandomStringArrayWithFloats(10, true); + final List sequential = Arrays.asList("0.0", "1.0", "2.0", "3.0", "4.0", "5.0", "6.0", "7.0", "8.0", "9.0"); + final List randomFloats = getRandomFloats(10, false); + final List randomFloatsMissing = getRandomFloats(10, true); - for (String r : Arrays.asList("*_p_f", "*_p_f_dv", "*_p_f_dv_ns", "*_p_f_ni_dv", + for (String r : Arrays.asList("*_p_f", "*_p_f_dv", "*_p_f_dv_ns", "*_p_f_ni_dv", "*_p_f_ni_dv_ns", "*_p_f_ni_ns_dv")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSort(r.replace("*","number"), sequential); - doTestPointFieldSort(r.replace("*","number"), randstrs); - // TODO: test some randomly generated (then sorted) arrays (with dups and/or missing values) + String field = r.replace("*", "number"); + doTestPointFieldSort(field, sequential); + doTestPointFieldSort(field, randomFloats); - doTestFloatPointFunctionQuery(r.replace("*","number"), "float"); + doTestFloatPointFunctionQuery(field, "float"); + } + for (String r : Arrays.asList("*_p_f_smf", "*_p_f_dv_smf", "*_p_f_ni_dv_smf", + "*_p_f_sml", "*_p_f_dv_sml", "*_p_f_ni_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestPointFieldSort(field, sequential); + doTestPointFieldSort(field, randomFloatsMissing); + doTestFloatPointFunctionQuery(field, "float"); } for (String r : Arrays.asList("*_p_f_ni", "*_p_f_ni_ns")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSortError(r.replace("*","number"), "w/o docValues", "42.34"); - - doTestPointFieldFunctionQueryError(r.replace("*","number"), "w/o docValues", "42.34"); + String field = r.replace("*", "number"); + doTestPointFieldSortError(field, "w/o docValues", "42.34"); + doTestPointFieldFunctionQueryError(field, "w/o docValues", "42.34"); } for (String r : Arrays.asList("*_p_f_mv", "*_p_f_ni_mv", "*_p_f_ni_mv_dv", "*_p_f_ni_dv_ns_mv", - "*_p_f_ni_ns_mv", "*_p_f_dv_ns_mv", "*_p_f_mv_dv")) { + "*_p_f_ni_ns_mv", "*_p_f_dv_ns_mv", "*_p_f_mv_dv", + "*_p_f_mv_smf", "*_p_f_mv_dv_smf", "*_p_f_ni_mv_dv_smf", + "*_p_f_mv_sml", "*_p_f_mv_dv_sml", "*_p_f_ni_mv_dv_sml")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSortError(r.replace("*","number"), "multivalued", "42.34"); - doTestPointFieldSortError(r.replace("*","number"), "multivalued", "42.34", "66.6"); - - doTestPointFieldFunctionQueryError(r.replace("*","number"), "multivalued", "42.34"); - doTestPointFieldFunctionQueryError(r.replace("*","number"), "multivalued", "42.34", "66.6"); + String field = r.replace("*", "number"); + doTestPointFieldSortError(field, "multivalued", "42.34"); + doTestPointFieldSortError(field, "multivalued", "42.34", "66.6"); + doTestPointFieldFunctionQueryError(field, "multivalued", "42.34"); + doTestPointFieldFunctionQueryError(field, "multivalued", "42.34", "66.6"); } assertEquals("Missing types in the test", Collections.emptySet(), regexToTest); - } @Test @@ -553,7 +665,7 @@ public class TestPointFields extends SolrTestCaseJ4 { testPointFieldFacetField("number_p_f", "number_p_f_dv", getSequentialStringArrayWithDoubles(10)); clearIndex(); assertU(commit()); - testPointFieldFacetField("number_p_f", "number_p_f_dv", getRandomStringArrayWithFloats(10, false)); + testPointFieldFacetField("number_p_f", "number_p_f_dv", toStringArray(getRandomFloats(10, false))); } @Test @@ -571,14 +683,14 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testFloatPointFieldMultiValuedExactQuery() throws Exception { - testPointFieldMultiValuedExactQuery("number_p_f_mv", getRandomStringArrayWithFloats(20, false)); - testPointFieldMultiValuedExactQuery("number_p_f_ni_mv_dv", getRandomStringArrayWithFloats(20, false)); + testPointFieldMultiValuedExactQuery("number_p_f_mv", toStringArray(getRandomFloats(20, false))); + testPointFieldMultiValuedExactQuery("number_p_f_ni_mv_dv", toStringArray(getRandomFloats(20, false))); } @Test public void testFloatPointFieldMultiValuedNonSearchableExactQuery() throws Exception { - testPointFieldMultiValuedExactQuery("number_p_f_ni_mv", getRandomStringArrayWithFloats(20, false), false); - testPointFieldMultiValuedExactQuery("number_p_f_ni_ns_mv", getRandomStringArrayWithFloats(20, false), false); + testPointFieldMultiValuedExactQuery("number_p_f_ni_mv", toStringArray(getRandomFloats(20, false)), false); + testPointFieldMultiValuedExactQuery("number_p_f_ni_ns_mv", toStringArray(getRandomFloats(20, false)), false); } @Test @@ -603,13 +715,13 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testFloatPointFieldMultiValuedFacetField() throws Exception { testPointFieldMultiValuedFacetField("number_p_f_mv", "number_p_f_mv_dv", getSequentialStringArrayWithDoubles(20)); - testPointFieldMultiValuedFacetField("number_p_f_mv", "number_p_f_mv_dv", getRandomStringArrayWithFloats(20, false)); + testPointFieldMultiValuedFacetField("number_p_f_mv", "number_p_f_mv_dv", toStringArray(getRandomFloats(20, false))); } @Test public void testFloatPointMultiValuedFunctionQuery() throws Exception { testPointMultiValuedFunctionQuery("number_p_f_mv", "number_p_f_mv_dv", "float", getSequentialStringArrayWithDoubles(20)); - testPointMultiValuedFunctionQuery("number_p_f_mv", "number_p_f_mv_dv", "float", getRandomStringArrayWithFloats(20, true)); + testPointMultiValuedFunctionQuery("number_p_f_mv", "number_p_f_mv_dv", "float", toAscendingStringArray(getRandomFloats(20, false), true)); } @@ -635,9 +747,9 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testFloatPointSetQuery() throws Exception { - doTestSetQueries("number_p_f", getRandomStringArrayWithFloats(20, false), false); - doTestSetQueries("number_p_f_mv", getRandomStringArrayWithFloats(20, false), true); - doTestSetQueries("number_p_f_ni_dv", getRandomStringArrayWithFloats(20, false), false); + doTestSetQueries("number_p_f", toStringArray(getRandomFloats(20, false)), false); + doTestSetQueries("number_p_f_mv", toStringArray(getRandomFloats(20, false)), true); + doTestSetQueries("number_p_f_ni_dv", toStringArray(getRandomFloats(20, false)), false); } @Test @@ -689,36 +801,50 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testLongPointFieldSortAndFunction() throws Exception { final SortedSet regexToTest = dynFieldRegexesForType(LongPointField.class); - final String[] vals = new String[]{ String.valueOf(Integer.MIN_VALUE), - "1", "2", "3", "4", "5", "6", "7", - String.valueOf(Integer.MAX_VALUE), String.valueOf(Long.MAX_VALUE)}; + final List vals = Arrays.asList((long)Integer.MIN_VALUE, + 1L, 2L, 3L, 4L, 5L, 6L, 7L, + (long)Integer.MAX_VALUE, Long.MAX_VALUE); + final List randomLongs = getRandomLongs(10, false); + final List randomLongsMissing = getRandomLongs(10, true); for (String r : Arrays.asList("*_p_l", "*_p_l_dv", "*_p_l_dv_ns", "*_p_l_ni_dv", "*_p_l_ni_dv_ns", "*_p_l_ni_ns_dv")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSort(r.replace("*","number"), vals); - // TODO: test some randomly generated (then sorted) arrays (with dups and/or missing values) - - doTestIntPointFunctionQuery(r.replace("*","number"), "long"); + String field = r.replace("*", "number"); + doTestPointFieldSort(field, vals); + doTestPointFieldSort(field, randomLongs); + doTestIntPointFunctionQuery(field, "long"); } - + + for (String r : Arrays.asList("*_p_l_smf", "*_p_l_dv_smf", "*_p_l_ni_dv_smf", + "*_p_l_sml", "*_p_l_dv_sml", "*_p_l_ni_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestPointFieldSort(field, vals); + doTestPointFieldSort(field, randomLongsMissing); + doTestIntPointFunctionQuery(field, "long"); + } + for (String r : Arrays.asList("*_p_l_ni", "*_p_l_ni_ns")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSortError(r.replace("*","number"), "w/o docValues", "4234"); - doTestPointFieldFunctionQueryError(r.replace("*","number"), "w/o docValues", "4234"); + String field = r.replace("*", "number"); + doTestPointFieldSortError(field, "w/o docValues", "4234"); + doTestPointFieldFunctionQueryError(field, "w/o docValues", "4234"); } for (String r : Arrays.asList("*_p_l_mv", "*_p_l_ni_mv", "*_p_l_ni_mv_dv", "*_p_l_ni_dv_ns_mv", - "*_p_l_ni_ns_mv", "*_p_l_dv_ns_mv", "*_p_l_mv_dv")) { + "*_p_l_ni_ns_mv", "*_p_l_dv_ns_mv", "*_p_l_mv_dv", + "*_p_l_mv_smf", "*_p_l_mv_dv_smf", "*_p_l_ni_mv_dv_smf", + "*_p_l_mv_sml", "*_p_l_mv_dv_sml", "*_p_l_ni_mv_dv_sml")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSortError(r.replace("*","number"), "multivalued", "4234"); - doTestPointFieldSortError(r.replace("*","number"), "multivalued", "4234", "66666666"); - doTestPointFieldFunctionQueryError(r.replace("*","number"), "multivalued", "4234"); - doTestPointFieldFunctionQueryError(r.replace("*","number"), "multivalued", "4234", "66666666"); + String field = r.replace("*", "number"); + doTestPointFieldSortError(field, "multivalued", "4234"); + doTestPointFieldSortError(field, "multivalued", "4234", "66666666"); + doTestPointFieldFunctionQueryError(field, "multivalued", "4234"); + doTestPointFieldFunctionQueryError(field, "multivalued", "4234", "66666666"); } assertEquals("Missing types in the test", Collections.emptySet(), regexToTest); - } @Test @@ -726,7 +852,7 @@ public class TestPointFields extends SolrTestCaseJ4 { testPointFieldFacetField("number_p_l", "number_p_l_dv", getSequentialStringArrayWithInts(10)); clearIndex(); assertU(commit()); - testPointFieldFacetField("number_p_l", "number_p_l_dv", getRandomStringArrayWithLongs(10, true)); + testPointFieldFacetField("number_p_l", "number_p_l_dv", toStringArray(getRandomLongs(10, false))); } @Test @@ -771,7 +897,7 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testLongPointFieldMultiValuedFacetField() throws Exception { testPointFieldMultiValuedFacetField("number_p_l_mv", "number_p_l_mv_dv", getSequentialStringArrayWithInts(20)); - testPointFieldMultiValuedFacetField("number_p_l_mv", "number_p_l_mv_dv", getRandomStringArrayWithLongs(20, false)); + testPointFieldMultiValuedFacetField("number_p_l_mv", "number_p_l_mv_dv", toStringArray(getRandomLongs(20, false))); } @Test @@ -806,9 +932,9 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testLongPointSetQuery() throws Exception { - doTestSetQueries("number_p_l", getRandomStringArrayWithLongs(20, false), false); - doTestSetQueries("number_p_l_mv", getRandomStringArrayWithLongs(20, false), true); - doTestSetQueries("number_p_l_ni_dv", getRandomStringArrayWithLongs(20, false), false); + doTestSetQueries("number_p_l", toStringArray(getRandomLongs(20, false)), false); + doTestSetQueries("number_p_l_mv", toStringArray(getRandomLongs(20, false)), true); + doTestSetQueries("number_p_l_ni_dv", toStringArray(getRandomLongs(20, false)), false); } @Test @@ -862,37 +988,48 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testDatePointFieldSortAndFunction() throws Exception { final SortedSet regexToTest = dynFieldRegexesForType(DatePointField.class); - final String[] sequential = getSequentialStringArrayWithDates(10); + final List sequential = Arrays.asList(getSequentialStringArrayWithDates(10)); + final List randomDates = getRandomInstants(10, false); + final List randomDatesMissing = getRandomInstants(10, true); for (String r : Arrays.asList("*_p_dt", "*_p_dt_dv", "*_p_dt_dv_ns", "*_p_dt_ni_dv", "*_p_dt_ni_dv_ns", "*_p_dt_ni_ns_dv")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSort(r.replace("*","number"), sequential); - // TODO: test some randomly generated (then sorted) arrays (with dups and/or missing values) - - doTestDatePointFunctionQuery(r.replace("*","number"), "date"); + String field = r.replace("*", "number"); + doTestPointFieldSort(field, sequential); + doTestPointFieldSort(field, randomDates); + doTestDatePointFunctionQuery(field, "date"); + } + for (String r : Arrays.asList("*_p_dt_smf", "*_p_dt_dv_smf", "*_p_dt_ni_dv_smf", + "*_p_dt_sml", "*_p_dt_dv_sml", "*_p_dt_ni_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestPointFieldSort(field, sequential); + doTestPointFieldSort(field, randomDatesMissing); + doTestDatePointFunctionQuery(field, "date"); } for (String r : Arrays.asList("*_p_dt_ni", "*_p_dt_ni_ns")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSortError(r.replace("*","number"), "w/o docValues", "1995-12-31T23:59:59Z"); - - doTestPointFieldFunctionQueryError(r.replace("*","number"), "w/o docValues", "1995-12-31T23:59:59Z"); + String field = r.replace("*", "number"); + doTestPointFieldSortError(field, "w/o docValues", "1995-12-31T23:59:59Z"); + doTestPointFieldFunctionQueryError(field, "w/o docValues", "1995-12-31T23:59:59Z"); } for (String r : Arrays.asList("*_p_dt_mv", "*_p_dt_ni_mv", "*_p_dt_ni_mv_dv", "*_p_dt_ni_dv_ns_mv", - "*_p_dt_ni_ns_mv", "*_p_dt_dv_ns_mv", "*_p_dt_mv_dv")) { + "*_p_dt_ni_ns_mv", "*_p_dt_dv_ns_mv", "*_p_dt_mv_dv", + "*_p_dt_mv_smf", "*_p_dt_mv_dv_smf", "*_p_dt_ni_mv_dv_smf", + "*_p_dt_mv_sml", "*_p_dt_mv_dv_sml", "*_p_dt_ni_mv_dv_sml")) { assertTrue(r, regexToTest.remove(r)); - doTestPointFieldSortError(r.replace("*","number"), "multivalued", "1995-12-31T23:59:59Z"); - doTestPointFieldSortError(r.replace("*","number"), "multivalued", "1995-12-31T23:59:59Z", "2000-12-31T23:59:59Z"); - - doTestPointFieldFunctionQueryError(r.replace("*","number"), "multivalued", "1995-12-31T23:59:59Z"); - doTestPointFieldFunctionQueryError(r.replace("*","number"), "multivalued", "1995-12-31T23:59:59Z", "2000-12-31T23:59:59Z"); + String field = r.replace("*", "number"); + doTestPointFieldSortError(field, "multivalued", "1995-12-31T23:59:59Z"); + doTestPointFieldSortError(field, "multivalued", "1995-12-31T23:59:59Z", "2000-12-31T23:59:59Z"); + doTestPointFieldFunctionQueryError(field, "multivalued", "1995-12-31T23:59:59Z"); + doTestPointFieldFunctionQueryError(field, "multivalued", "1995-12-31T23:59:59Z", "2000-12-31T23:59:59Z"); } assertEquals("Missing types in the test", Collections.emptySet(), regexToTest); - } @Test @@ -942,7 +1079,7 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testDatePointFieldMultiValuedFacetField() throws Exception { testPointFieldMultiValuedFacetField("number_p_dt_mv", "number_p_dt_mv_dv", getSequentialStringArrayWithDates(20)); - testPointFieldMultiValuedFacetField("number_p_dt_mv", "number_p_dt_mv_dv", getRandomStringArrayWithDates(20, false)); + testPointFieldMultiValuedFacetField("number_p_dt_mv", "number_p_dt_mv_dv", toStringArray(getRandomInstants(20, false))); } @Test @@ -977,9 +1114,9 @@ public class TestPointFields extends SolrTestCaseJ4 { @Test public void testDatePointSetQuery() throws Exception { - doTestSetQueries("number_p_dt", getRandomStringArrayWithDates(20, false), false); - doTestSetQueries("number_p_dt_mv", getRandomStringArrayWithDates(20, false), true); - doTestSetQueries("number_p_dt_ni_dv", getRandomStringArrayWithDates(20, false), false); + doTestSetQueries("number_p_dt", toStringArray(getRandomInstants(20, false)), false); + doTestSetQueries("number_p_dt_mv", toStringArray(getRandomInstants(20, false)), true); + doTestSetQueries("number_p_dt_ni_dv", toStringArray(getRandomInstants(20, false)), false); } @@ -1012,15 +1149,14 @@ public class TestPointFields extends SolrTestCaseJ4 { public void testInternals() throws IOException { String[] types = new String[]{"i", "l", "f", "d"}; - String[] suffixes = new String[]{"", "_dv", "_mv", "_mv_dv", "_ni", "_ni_dv", "_ni_dv_ns", "_ni_dv_ns_mv", "_ni_mv", "_ni_mv_dv", "_ni_ns", "_ni_ns_mv", "_dv_ns", "_ni_ns_dv", "_dv_ns_mv"}; Set typesTested = new HashSet<>(); for (String type:types) { - for (String suffix:suffixes) { + for (String suffix:FIELD_SUFFIXES) { doTestInternals("number_p_" + type + suffix, getSequentialStringArrayWithInts(10)); typesTested.add("*_p_" + type + suffix); } } - for (String suffix:suffixes) { + for (String suffix:FIELD_SUFFIXES) { doTestInternals("number_p_dt" + suffix, getSequentialStringArrayWithDates(10)); typesTested.add("*_p_dt" + suffix); } @@ -1047,50 +1183,39 @@ public class TestPointFields extends SolrTestCaseJ4 { return typesToTest; } - private String[] getRandomStringArrayWithDoubles(int length, boolean sorted) { - Set set; - if (sorted) { - set = new TreeSet<>(); - } else { - set = new HashSet<>(); - } - while (set.size() < length) { - double f = random().nextDouble() * (Double.MAX_VALUE/2); - if (random().nextBoolean()) { - f = f * -1; + private List getRandomList(int length, boolean missingVals, Supplier randomVal) { + List list = new ArrayList<>(length); + for (int i = 0 ; i < length ; ++i) { + T val = null; + // Sometimes leave val as null when we're producing missing values + if (missingVals == false || usually()) { + val = randomVal.get(); } - set.add(f); + list.add(val); } - String[] stringArr = new String[length]; - int i = 0; - for (double val:set) { - stringArr[i] = String.valueOf(val); - i++; - } - return stringArr; + return list; } - - private String[] getRandomStringArrayWithFloats(int length, boolean sorted) { - Set set; - if (sorted) { - set = new TreeSet<>(); - } else { - set = new HashSet<>(); - } - while (set.size() < length) { - float f = random().nextFloat() * (Float.MAX_VALUE/2); - if (random().nextBoolean()) { - f = f * -1; - } - set.add(f); - } - String[] stringArr = new String[length]; - int i = 0; - for (float val:set) { - stringArr[i] = String.valueOf(val); - i++; - } - return stringArr; + + private List getRandomDoubles(int length, boolean missingVals) { + return getRandomList(length, missingVals, + () -> random().nextDouble() * Double.MAX_VALUE * (random().nextBoolean() ? 1.D : -1.D)); + } + + private List getRandomFloats(int length, boolean missingVals) { + return getRandomList(length, missingVals, + () -> random().nextFloat() * Float.MAX_VALUE * (random().nextBoolean() ? 1.f : -1.f)); + } + + private List getRandomInts(int length, boolean missingVals) { + return getRandomList(length, missingVals, () -> random().nextInt()); + } + + private List getRandomLongs(int length, boolean missingVals){ + return getRandomList(length, missingVals, () -> random().nextLong()); + } + + private List getRandomInstants(int length, boolean missingVals){ + return getRandomList(length, missingVals, () -> Instant.ofEpochMilli(random().nextLong())); } private String[] getSequentialStringArrayWithInts(int length) { @@ -1117,74 +1242,7 @@ public class TestPointFields extends SolrTestCaseJ4 { } return arr; } - - private String[] getRandomStringArrayWithInts(int length, boolean sorted) { - Set set; - if (sorted) { - set = new TreeSet<>(); - } else { - set = new HashSet<>(); - } - while (set.size() < length) { - int number = random().nextInt(100); - if (random().nextBoolean()) { - number = number * -1; - } - set.add(number); - } - String[] stringArr = new String[length]; - int i = 0; - for (int val:set) { - stringArr[i] = String.valueOf(val); - i++; - } - return stringArr; - } - - private String[] getRandomStringArrayWithLongs(int length, boolean sorted) { - Set set; - if (sorted) { - set = new TreeSet<>(); - } else { - set = new HashSet<>(); - } - while (set.size() < length) { - long number = random().nextLong(); - if (random().nextBoolean()) { - number = number * -1; - } - set.add(number); - } - String[] stringArr = new String[length]; - int i = 0; - for (long val:set) { - stringArr[i] = String.valueOf(val); - i++; - } - return stringArr; - } - private String[] getRandomStringArrayWithDates(int length, boolean sorted) { - assert length < 60; - Set set; - if (sorted) { - set = new TreeSet<>(); - } else { - set = new HashSet<>(); - } - while (set.size() < length) { - int number = random().nextInt(60); - set.add(number); - } - String[] stringArr = new String[length]; - int i = 0; - for (int val:set) { - stringArr[i] = String.format(Locale.ROOT, "1995-12-11T19:59:%02dZ", val); - i++; - } - return stringArr; - } - private void doTestFieldNotIndexed(String field, String[] values) throws IOException { assert values.length == 10; // test preconditions @@ -1379,9 +1437,9 @@ public class TestPointFields extends SolrTestCaseJ4 { String[] arr; if (testLong) { - arr = getRandomStringArrayWithLongs(100, true); + arr = toAscendingStringArray(getRandomLongs(100, false), true); } else { - arr = getRandomStringArrayWithInts(100, true); + arr = toAscendingStringArray(getRandomInts(100, false), true); } for (int i = 0; i < arr.length; i++) { assertU(adoc("id", String.valueOf(i), fieldName, arr[i])); @@ -1423,14 +1481,6 @@ public class TestPointFields extends SolrTestCaseJ4 { "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + docValuesField +"']/int[@name='" + numbers[2] + "'][.='1']", "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + docValuesField +"']/int[@name='" + numbers[3] + "'][.='1']"); -// assertU(commit()); -// assertQ(req("q", "id:0", "fl", "id, " + docValuesField, "facet", "true", "facet.field", docValuesField, "facet.mincount", "0"), -// "//*[@numFound='1']", -// "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + docValuesField +"']/int[@name='" + numbers[0] + "'][.='1']", -// "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + docValuesField +"']/int[@name='" + numbers[1] + "'][.='0']", -// "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + docValuesField +"']/int[@name='" + numbers[2] + "'][.='0']", -// "count(//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + docValuesField +"']/int))==10"); - assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); assertTrue(h.getCore().getLatestSchema().getField(nonDocValuesField).getType() instanceof PointField); assertQEx("Expecting Exception", @@ -1875,13 +1925,6 @@ public class TestPointFields extends SolrTestCaseJ4 { assertTrue(h.getCore().getLatestSchema().getField(docValuesField).getType() instanceof PointField); String function = "field(" + docValuesField + ", min)"; -// assertQ(req("q", "*:*", "fl", "id, " + function), -// "//*[@numFound='10']", -// "//result/doc[1]/" + type + "[@name='" + function + "'][.='" + numbers[0] + "']", -// "//result/doc[2]/" + type + "[@name='" + function + "'][.='" + numbers[1] + "']", -// "//result/doc[3]/" + type + "[@name='" + function + "'][.='" + numbers[2] + "']", -// "//result/doc[10]/" + type + "[@name='" + function + "'][.='" + numbers[9] + "']"); - assertQ(req("q", "*:*", "fl", "id, " + docValuesField, "sort", function + " desc"), "//*[@numFound='10']", "//result/doc[1]/str[@name='id'][.='9']", @@ -2063,27 +2106,32 @@ public class TestPointFields extends SolrTestCaseJ4 { * @param field name of field to sort on * @param values list of values in ascending order */ - private void doTestPointFieldSort(String field, String... values) throws Exception { - assert values != null && 2 <= values.length; - - // TODO: need to add sort missing coverage... - // - // idea: accept "null" as possible value for sort missing tests ? - // - // need to account for possibility that multiple nulls will be in non deterministic order - // always using secondary sort on id seems prudent ... handles any "dups" in values[] + private > void doTestPointFieldSort(String field, List values) throws Exception { + assert values != null && 2 <= values.size(); + + final List docs = new ArrayList<>(values.size()); + final String[] ascXpathChecks = new String[values.size() + 1]; + final String[] descXpathChecks = new String[values.size() + 1]; + ascXpathChecks[values.size()] = "//*[@numFound='" + values.size() + "']"; + descXpathChecks[values.size()] = "//*[@numFound='" + values.size() + "']"; - final List docs = new ArrayList<>(values.length); - final String[] ascXpathChecks = new String[values.length + 1]; - final String[] descXpathChecks = new String[values.length + 1]; - ascXpathChecks[values.length] = "//*[@numFound='" + values.length + "']"; - descXpathChecks[values.length] = "//*[@numFound='" + values.length + "']"; + boolean missingFirst = field.endsWith("_sml") == false; - for (int i = values.length-1; i >= 0; i--) { - docs.add(sdoc("id", String.valueOf(i), field, String.valueOf(values[i]))); + List> ascendingPosVals = toAscendingPosVals(values, missingFirst); + for (int i = ascendingPosVals.size() - 1 ; i >= 0 ; --i) { + T value = ascendingPosVals.get(i).val; + if (value == null) { + docs.add(sdoc("id", String.valueOf(i))); // null => missing value + } else { + docs.add(sdoc("id", String.valueOf(i), field, String.valueOf(value))); + } // reminder: xpath array indexes start at 1 ascXpathChecks[i]= "//result/doc["+ (1 + i)+"]/str[@name='id'][.='"+i+"']"; - descXpathChecks[i]= "//result/doc["+ (values.length - i) +"]/str[@name='id'][.='"+i+"']"; + } + List> descendingPosVals = toDescendingPosVals + (ascendingPosVals.stream().map(pv->pv.val).collect(Collectors.toList()), missingFirst); + for (int i = descendingPosVals.size() - 1 ; i >= 0 ; --i) { + descXpathChecks[i]= "//result/doc[" + (i + 1) + "]/str[@name='id'][.='" + descendingPosVals.get(i).pos + "']"; } // ensure doc add order doesn't affect results @@ -2093,11 +2141,10 @@ public class TestPointFields extends SolrTestCaseJ4 { } assertU(commit()); - assertQ(req("q", "*:*", "fl", "id", "sort", field + " asc"), + assertQ(req("q", "*:*", "fl", "id, " + field, "sort", field + " asc, id asc"), ascXpathChecks); - assertQ(req("q", "*:*", "fl", "id", "sort", field + " desc"), + assertQ(req("q", "*:*", "fl", "id, " + field, "sort", field + " desc, id desc"), descXpathChecks); - clearIndex(); assertU(commit()); @@ -2200,9 +2247,9 @@ public class TestPointFields extends SolrTestCaseJ4 { String[] arr; if (testDouble) { - arr = getRandomStringArrayWithDoubles(10, true); + arr = toAscendingStringArray(getRandomDoubles(10, false), true); } else { - arr = getRandomStringArrayWithFloats(10, true); + arr = toAscendingStringArray(getRandomFloats(10, false), true); } for (int i = 0; i < arr.length; i++) { assertU(adoc("id", String.valueOf(i), fieldName, arr[i])); @@ -2316,7 +2363,7 @@ public class TestPointFields extends SolrTestCaseJ4 { StringBuilder builder = new StringBuilder(fieldName + ":("); for (int i = 0; i < numTerms; i++) { if (sf.getType().getNumberType() == NumberType.DATE) { - builder.append(String.valueOf(values[i]).replace(":", "\\:") + ' '); + builder.append(values[i].replaceAll("(:|^[-+])", "\\\\$1") + ' '); } else { builder.append(String.valueOf(values[i]).replace("-", "\\-") + ' '); } @@ -2327,7 +2374,7 @@ public class TestPointFields extends SolrTestCaseJ4 { "//*[@numFound='" + numTerms + "']", "//*[@name='parsed_filter_queries']/str[.='(" + getSetQueryToString(fieldName, values, numTerms) + ")']"); } else { - // Won't use PointInSetQuery if the fiels is not indexed, but should match the same docs + // Won't use PointInSetQuery if the field is not indexed, but should match the same docs assertQ(req(CommonParams.DEBUG, CommonParams.QUERY, "q", "*:*", "fq", builder.toString(), "fl", "id," + fieldName), "//*[@numFound='" + numTerms + "']"); } @@ -2990,7 +3037,6 @@ public class TestPointFields extends SolrTestCaseJ4 { public void testWhiteboxCreateFields() throws Exception { String[] typeNames = new String[]{"i", "l", "f", "d", "dt"}; - String[] suffixes = new String[]{"", "_dv", "_mv", "_mv_dv", "_ni", "_ni_dv", "_ni_dv_ns", "_ni_dv_ns_mv", "_ni_mv", "_ni_mv_dv", "_ni_ns", "_ni_ns_mv", "_dv_ns", "_ni_ns_dv", "_dv_ns_mv"}; Class[] expectedClasses = new Class[]{IntPoint.class, LongPoint.class, FloatPoint.class, DoublePoint.class, LongPoint.class}; Date dateToTest = new Date(); @@ -3004,7 +3050,7 @@ public class TestPointFields extends SolrTestCaseJ4 { Set typesTested = new HashSet<>(); for (int i = 0; i < typeNames.length; i++) { - for (String suffix:suffixes) { + for (String suffix:FIELD_SUFFIXES) { doWhiteboxCreateFields("whitebox_p_" + typeNames[i] + suffix, expectedClasses[i], values[i]); typesTested.add("*_p_" + typeNames[i] + suffix); } From ce5b184b17fb000f9d3469807872df30e1550397 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Thu, 8 Jun 2017 15:57:14 +0930 Subject: [PATCH 018/131] SOLR-9530: documentation added --- .../src/update-request-processors.adoc | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/update-request-processors.adoc b/solr/solr-ref-guide/src/update-request-processors.adoc index eb69d3b1040..a4f3e0c0b90 100644 --- a/solr/solr-ref-guide/src/update-request-processors.adoc +++ b/solr/solr-ref-guide/src/update-request-processors.adoc @@ -379,13 +379,14 @@ These are listed for completeness, but are part of the Solr infrastructure, part [[UpdateRequestProcessors-UpdateProcessorsThatCanBeUsedatRuntime]] === Update Processors That Can Be Used at Runtime +These Update processors do not need any configuration is your `solrconfig.xml` . They are automatically initialized when their name is added to the `processor` parameter. Multiple processors can be used by appending multiple processor names (comma separated) [[UpdateRequestProcessors-TemplateUpdateProcessorFactory]] ==== TemplateUpdateProcessorFactory The `TemplateUpdateProcessorFactory` can be used to add new fields to documents based on a template pattern. -This can be used directly in a request without any configuration. To enable this processor, use the parameter `processor=Template`. The template parameter `Template.field` (multivalued) define the field to add and the pattern. Templates may contain placeholders which refer to other fields in the document. You can have multiple `Template.field` parameters in a single request. +Use the parameter `processor=Template` to use it. The template parameter `Template.field` (multivalued) define the field to add and the pattern. Templates may contain placeholders which refer to other fields in the document. You can have multiple `Template.field` parameters in a single request. For example: @@ -395,3 +396,22 @@ processor=Template&Template.field=fullName:Mr. ${firstName} ${lastName} ---- The above example would add a new field to the document called `fullName`. The fields `firstName and` `lastName` are supplied from the document fields. If either of them is missing, that part is replaced with an empty string. If those fields are multi-valued, only the first value is used. + +==== AtomicUpdateProcessorFactory + + +Use it to convert your normal `update` operations to atomic update operations. This is particularly useful when you use endpoints such as `/update/csv` or `/update/json/docs` which does not support syntax for atomic operations. + +example: + +[source,bash] +---- +processor=Atomic&Atomic.field1=add&Atomic.field2=set&Atomic.field3=inc&Atomic.field4=remove&Atomic.field4=remove +---- + +The above parameters convert a normal `update` operation on + +* `field1` to an atomic `add` operation +* `field2` to an atomic `set` operation +* `field3` to an atomic `inc` operation +* `field4` to an atomic `remove` operation \ No newline at end of file From b0020634ac19ce36d2edcee8684406a51654eff7 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Thu, 8 Jun 2017 08:35:18 +0200 Subject: [PATCH 019/131] LUCENE-7864: IndexMergeTool is not using intermediate hard links (even if possible) --- lucene/CHANGES.txt | 5 +++++ .../java/org/apache/lucene/misc/IndexMergeTool.java | 11 ++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 1e89390aa90..0bc032bf92f 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -124,6 +124,11 @@ New Features * LUCENE-7855: Added advanced options of the Wikipedia tokenizer to its factory. (Juan Pedro via Adrien Grand) +Bug Fixes + +* LUCENE-7864: IndexMergeTool is not using intermediate hard links (even + if possible). (Dawid Weiss) + Other * LUCENE-7800: Remove code that potentially rethrows checked exceptions diff --git a/lucene/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java b/lucene/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java index cbb11cdd749..09eb850a3f7 100644 --- a/lucene/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java +++ b/lucene/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java @@ -39,15 +39,16 @@ public class IndexMergeTool { System.err.println("Usage: IndexMergeTool [index3] ..."); System.exit(1); } - FSDirectory mergedIndex = FSDirectory.open(Paths.get(args[0])); - IndexWriter writer = new IndexWriter(mergedIndex, new IndexWriterConfig(null) - .setOpenMode(OpenMode.CREATE)); + // Try to use hardlinks to source segments, if possible. + Directory mergedIndex = new HardlinkCopyDirectoryWrapper(FSDirectory.open(Paths.get(args[0]))); + + IndexWriter writer = new IndexWriter(mergedIndex, + new IndexWriterConfig(null).setOpenMode(OpenMode.CREATE)); Directory[] indexes = new Directory[args.length - 1]; for (int i = 1; i < args.length; i++) { - // try to use hardlinks if possible - indexes[i - 1] = new HardlinkCopyDirectoryWrapper(FSDirectory.open(Paths.get(args[i]))); + indexes[i - 1] = FSDirectory.open(Paths.get(args[i])); } System.out.println("Merging..."); From cf9b70b25365b194261cf7c3b6e32ec3330295dc Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Thu, 8 Jun 2017 10:15:54 +0100 Subject: [PATCH 020/131] SOLR-10174: reduce makeFeatureWeights code duplication in tests --- .../apache/solr/ltr/TestLTRScoringQuery.java | 20 +++++-------------- .../solr/ltr/TestSelectiveWeightCreation.java | 14 ++----------- .../solr/ltr/model/TestLinearModel.java | 10 ++++++++++ 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTRScoringQuery.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTRScoringQuery.java index 0576c999771..e20f7e1684b 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTRScoringQuery.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTRScoringQuery.java @@ -89,16 +89,6 @@ public class TestLTRScoringQuery extends LuceneTestCase { return features; } - private static Map makeFeatureWeights(List features) { - final Map nameParams = new HashMap(); - final HashMap modelWeights = new HashMap(); - for (final Feature feat : features) { - modelWeights.put(feat.getName(), 0.1); - } - nameParams.put("weights", modelWeights); - return nameParams; - } - private LTRScoringQuery.ModelWeight performQuery(TopDocs hits, IndexSearcher searcher, int docid, LTRScoringQuery model) throws IOException, ModelException { @@ -132,7 +122,7 @@ public class TestLTRScoringQuery extends LuceneTestCase { Collections.nCopies(features.size(),IdentityNormalizer.INSTANCE)); final List allFeatures = makeFeatures( new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); - final Map modelParams = makeFeatureWeights(features); + final Map modelParams = TestLinearModel.makeFeatureWeights(features); final LTRScoringModel algorithm1 = TestLinearModel.createLinearModel( "testModelName", @@ -222,7 +212,7 @@ public class TestLTRScoringQuery extends LuceneTestCase { Collections.nCopies(features.size(),IdentityNormalizer.INSTANCE)); LTRScoringModel ltrScoringModel = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, - makeFeatureWeights(features)); + TestLinearModel.makeFeatureWeights(features)); LTRScoringQuery.ModelWeight modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(ltrScoringModel)); @@ -248,7 +238,7 @@ public class TestLTRScoringQuery extends LuceneTestCase { new ArrayList( Collections.nCopies(features.size(),IdentityNormalizer.INSTANCE)); ltrScoringModel = TestLinearModel.createLinearModel("test", - features, norms, "test", allFeatures, makeFeatureWeights(features)); + features, norms, "test", allFeatures, TestLinearModel.makeFeatureWeights(features)); modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(ltrScoringModel)); @@ -268,7 +258,7 @@ public class TestLTRScoringQuery extends LuceneTestCase { Collections.nCopies(features.size(),IdentityNormalizer.INSTANCE)); try { ltrScoringModel = TestLinearModel.createLinearModel("test", - features, norms, "test", allFeatures, makeFeatureWeights(features)); + features, norms, "test", allFeatures, TestLinearModel.makeFeatureWeights(features)); fail("unexpectedly got here instead of catching "+expectedModelException); modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(ltrScoringModel)); @@ -301,7 +291,7 @@ public class TestLTRScoringQuery extends LuceneTestCase { Collections.nCopies(features.size(),norm)); final LTRScoringModel normMeta = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, - makeFeatureWeights(features)); + TestLinearModel.makeFeatureWeights(features)); modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(normMeta)); diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java index cbd0e2389da..b2cbec907b0 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java @@ -70,16 +70,6 @@ public class TestSelectiveWeightCreation extends TestRerankBase { return features; } - private static Map makeFeatureWeights(List features) { - final Map nameParams = new HashMap(); - final HashMap modelWeights = new HashMap(); - for (final Feature feat : features) { - modelWeights.put(feat.getName(), 0.1); - } - nameParams.put("weights", modelWeights); - return nameParams; - } - private LTRScoringQuery.ModelWeight performQuery(TopDocs hits, IndexSearcher searcher, int docid, LTRScoringQuery model) throws IOException, ModelException { @@ -168,7 +158,7 @@ public class TestSelectiveWeightCreation extends TestRerankBase { // when features are NOT requested in the response, only the modelFeature weights should be created final LTRScoringModel ltrScoringModel1 = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, - makeFeatureWeights(features)); + TestLinearModel.makeFeatureWeights(features)); LTRScoringQuery.ModelWeight modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(ltrScoringModel1, false)); // features not requested in response LTRScoringQuery.FeatureInfo[] featuresInfo = modelWeight.getFeaturesInfo(); @@ -185,7 +175,7 @@ public class TestSelectiveWeightCreation extends TestRerankBase { // when features are requested in the response, weights should be created for all features final LTRScoringModel ltrScoringModel2 = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, - makeFeatureWeights(features)); + TestLinearModel.makeFeatureWeights(features)); modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(ltrScoringModel2, true)); // features requested in response featuresInfo = modelWeight.getFeaturesInfo(); diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestLinearModel.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestLinearModel.java index 067bd277de8..46496244db8 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestLinearModel.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestLinearModel.java @@ -45,6 +45,16 @@ public class TestLinearModel extends TestRerankBase { return model; } + public static Map makeFeatureWeights(List features) { + final Map nameParams = new HashMap(); + final HashMap modelWeights = new HashMap(); + for (final Feature feat : features) { + modelWeights.put(feat.getName(), 0.1); + } + nameParams.put("weights", modelWeights); + return nameParams; + } + static ManagedModelStore store = null; static FeatureStore fstore = null; From 5a34d419b9afea443e56a775f3b7f2d08e18add7 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 7 Jun 2017 19:55:32 +0200 Subject: [PATCH 021/131] LUCENE-7869: Changed MemoryIndex to sort 1d points. In case of 1d points, the PointInSetQuery.MergePointVisitor expects that these points are visited in ascending order. Prior to this change the memory index doesn't do this and this can result in document with multiple points that should match to not match. --- lucene/CHANGES.txt | 4 +++ .../lucene/index/memory/MemoryIndex.java | 35 +++++++++++-------- .../lucene/index/memory/TestMemoryIndex.java | 24 +++++++++++++ 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 0bc032bf92f..02512434896 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -129,6 +129,10 @@ Bug Fixes * LUCENE-7864: IndexMergeTool is not using intermediate hard links (even if possible). (Dawid Weiss) +* LUCENE-7869: Changed MemoryIndex to sort 1d points. In case of 1d points, the PointInSetQuery.MergePointVisitor expects + that these points are visited in ascending order. The memory index doesn't do this and this can result in document + with multiple points that should match to not match. (Martijn van Groningen) + Other * LUCENE-7800: Remove code that potentially rethrows checked exceptions diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java index a409a17c631..92b9359c948 100644 --- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java +++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java @@ -868,20 +868,27 @@ public class MemoryIndex { final int numDimensions = fieldInfo.getPointDimensionCount(); final int numBytesPerDimension = fieldInfo.getPointNumBytes(); - minPackedValue = pointValues[0].bytes.clone(); - maxPackedValue = pointValues[0].bytes.clone(); - - for (int i = 0; i < pointValuesCount; i++) { - BytesRef pointValue = pointValues[i]; - assert pointValue.bytes.length == pointValue.length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this"; - - for (int dim = 0; dim < numDimensions; ++dim) { - int offset = dim * numBytesPerDimension; - if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, minPackedValue, offset) < 0) { - System.arraycopy(pointValue.bytes, offset, minPackedValue, offset, numBytesPerDimension); - } - if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, maxPackedValue, offset) > 0) { - System.arraycopy(pointValue.bytes, offset, maxPackedValue, offset, numBytesPerDimension); + if (numDimensions == 1) { + // PointInSetQuery.MergePointVisitor expects values to be visited in increasing order, + // this is a 1d optimization which has to be done here too. Otherwise we emit values + // out of order which causes mismatches. + Arrays.sort(pointValues, 0, pointValuesCount); + minPackedValue = pointValues[0].bytes.clone(); + maxPackedValue = pointValues[pointValuesCount - 1].bytes.clone(); + } else { + minPackedValue = pointValues[0].bytes.clone(); + maxPackedValue = pointValues[0].bytes.clone(); + for (int i = 0; i < pointValuesCount; i++) { + BytesRef pointValue = pointValues[i]; + assert pointValue.bytes.length == pointValue.length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this"; + for (int dim = 0; dim < numDimensions; ++dim) { + int offset = dim * numBytesPerDimension; + if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, minPackedValue, offset) < 0) { + System.arraycopy(pointValue.bytes, offset, minPackedValue, offset, numBytesPerDimension); + } + if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, maxPackedValue, offset) > 0) { + System.arraycopy(pointValue.bytes, offset, maxPackedValue, offset, numBytesPerDimension); + } } } } diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java index 3d3a12c97ce..fa62262ddd4 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java @@ -512,6 +512,30 @@ public class TestMemoryIndex extends LuceneTestCase { assertEquals(1, s.count(DoublePoint.newRangeQuery("doubles", new double[] {10D, 10D}, new double[] {30D, 30D}))); } + public void testMultiValuedPointsSortedCorrectly() throws Exception { + Document doc = new Document(); + doc.add(new IntPoint("ints", 3)); + doc.add(new IntPoint("ints", 2)); + doc.add(new IntPoint("ints", 1)); + doc.add(new LongPoint("longs", 3L)); + doc.add(new LongPoint("longs", 2L)); + doc.add(new LongPoint("longs", 1L)); + doc.add(new FloatPoint("floats", 3F)); + doc.add(new FloatPoint("floats", 2F)); + doc.add(new FloatPoint("floats", 1F)); + doc.add(new DoublePoint("doubles", 3D)); + doc.add(new DoublePoint("doubles", 2D)); + doc.add(new DoublePoint("doubles", 1D)); + + MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer); + IndexSearcher s = mi.createSearcher(); + + assertEquals(1, s.count(IntPoint.newSetQuery("ints", 2))); + assertEquals(1, s.count(LongPoint.newSetQuery("longs", 2))); + assertEquals(1, s.count(FloatPoint.newSetQuery("floats", 2))); + assertEquals(1, s.count(DoublePoint.newSetQuery("doubles", 2))); + } + public void testIndexingPointsAndDocValues() throws Exception { FieldType type = new FieldType(); type.setDimensions(1, 4); From c5c05b46c43896d0c3ab74818e6c0847b10d8e99 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Thu, 8 Jun 2017 19:39:55 +0930 Subject: [PATCH 022/131] SOLR-10647: move the spec files to solrj --- .../src/java/org/apache/solr/api/Api.java | 1 + .../src/java/org/apache/solr/api/ApiBag.java | 20 ++++++------------- .../java/org/apache/solr/api/V2HttpCall.java | 18 ++++++++--------- .../solr/handler/UpdateRequestHandlerApi.java | 4 ++-- .../handler/admin/BaseHandlerApiSupport.java | 3 +-- .../solr/handler/admin/InfoHandler.java | 2 +- .../handler/admin/SecurityConfHandler.java | 10 +++++----- .../apache/solr/security/BasicAuthPlugin.java | 2 +- .../RuleBasedAuthorizationPlugin.java | 6 +++--- .../Sha256AuthenticationProvider.java | 4 ++-- .../solr/handler/admin/TestApiFramework.java | 2 +- .../apache/solr/util/JsonValidatorTest.java | 5 ++--- .../org/apache/solr/common}/SpecProvider.java | 2 +- .../solr/common/params/CommonParams.java | 7 +++++-- .../apache/solr/common}/util/PathTrie.java | 2 +- .../org/apache/solr/common/util/Utils.java | 8 ++++++++ .../apispec/autoscaling.Commands.json | 0 .../resources/apispec/cluster.Commands.json | 0 .../resources/apispec/cluster.aliases.json | 0 .../apispec/cluster.commandstatus.delete.json | 0 .../apispec/cluster.commandstatus.json | 0 .../apispec/cluster.configs.Commands.json | 0 .../apispec/cluster.configs.delete.json | 0 .../resources/apispec/cluster.configs.json | 0 .../src/resources/apispec/cluster.json | 0 .../src/resources/apispec/cluster.nodes.json | 0 .../cluster.security.BasicAuth.Commands.json | 0 ...uster.security.RuleBasedAuthorization.json | 0 ...ster.security.authentication.Commands.json | 0 .../cluster.security.authentication.json | 0 ...uster.security.authorization.Commands.json | 0 .../cluster.security.authorization.json | 0 .../apispec/collections.Commands.json | 0 .../collections.collection.Commands.json | 0 ...ollections.collection.Commands.modify.json | 0 ...ollections.collection.Commands.reload.json | 0 .../collections.collection.delete.json | 0 .../apispec/collections.collection.json | 0 ...ollections.collection.shards.Commands.json | 0 ...ions.collection.shards.shard.Commands.json | 0 ...ctions.collection.shards.shard.delete.json | 0 ...ollection.shards.shard.replica.delete.json | 0 .../src/resources/apispec/collections.json | 0 .../resources/apispec/core.RealtimeGet.json | 0 .../apispec/core.SchemaEdit.addCopyField.json | 0 .../apispec/core.SchemaEdit.addField.json | 0 ...ore.SchemaEdit.addFieldType.analyzers.json | 0 .../apispec/core.SchemaEdit.addFieldType.json | 0 .../core.SchemaEdit.deleteCopyField.json | 0 .../core.SchemaEdit.deleteDynamicField.json | 0 .../apispec/core.SchemaEdit.deleteField.json | 0 .../core.SchemaEdit.deleteFieldType.json | 0 .../resources/apispec/core.SchemaEdit.json | 0 .../apispec/core.SchemaRead.copyFields.json | 0 ...e.SchemaRead.dynamicFields_fieldTypes.json | 0 .../apispec/core.SchemaRead.fields.json | 0 .../resources/apispec/core.SchemaRead.json | 0 .../src/resources/apispec/core.Update.json | 0 ...Commands.addRequestHandler.properties.json | 0 .../apispec/core.config.Commands.generic.json | 0 .../apispec/core.config.Commands.json | 0 .../core.config.Commands.runtimeLib.json | 0 .../apispec/core.config.Params.Commands.json | 0 .../resources/apispec/core.config.Params.json | 0 .../src/resources/apispec/core.config.json | 0 .../resources/apispec/core.system.blob.json | 0 .../apispec/core.system.blob.upload.json | 0 .../src/resources/apispec/cores.Commands.json | 0 .../src/resources/apispec/cores.Status.json | 0 .../apispec/cores.core.Commands.json | 0 .../apispec/cores.core.Commands.split.json | 0 .../src/resources/apispec/emptySpec.json | 0 .../src/resources/apispec/node.Commands.json | 0 .../src/resources/apispec/node.Info.json | 0 .../src/resources/apispec/node.invoke.json | 0 .../solr/common/util}/TestPathTrie.java | 4 ++-- 76 files changed, 51 insertions(+), 49 deletions(-) rename solr/{core/src/java/org/apache/solr/api => solrj/src/java/org/apache/solr/common}/SpecProvider.java (96%) rename solr/{core/src/java/org/apache/solr => solrj/src/java/org/apache/solr/common}/util/PathTrie.java (99%) rename solr/{core => solrj}/src/resources/apispec/autoscaling.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.aliases.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.commandstatus.delete.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.commandstatus.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.configs.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.configs.delete.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.configs.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.nodes.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.security.BasicAuth.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.security.RuleBasedAuthorization.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.security.authentication.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.security.authentication.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.security.authorization.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/cluster.security.authorization.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.collection.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.collection.Commands.modify.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.collection.Commands.reload.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.collection.delete.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.collection.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.collection.shards.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.collection.shards.shard.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.collection.shards.shard.delete.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.collection.shards.shard.replica.delete.json (100%) rename solr/{core => solrj}/src/resources/apispec/collections.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.RealtimeGet.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaEdit.addCopyField.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaEdit.addField.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaEdit.addFieldType.analyzers.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaEdit.addFieldType.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaEdit.deleteCopyField.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaEdit.deleteDynamicField.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaEdit.deleteField.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaEdit.deleteFieldType.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaEdit.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaRead.copyFields.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaRead.dynamicFields_fieldTypes.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaRead.fields.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.SchemaRead.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.Update.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.config.Commands.addRequestHandler.properties.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.config.Commands.generic.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.config.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.config.Commands.runtimeLib.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.config.Params.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.config.Params.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.config.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.system.blob.json (100%) rename solr/{core => solrj}/src/resources/apispec/core.system.blob.upload.json (100%) rename solr/{core => solrj}/src/resources/apispec/cores.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/cores.Status.json (100%) rename solr/{core => solrj}/src/resources/apispec/cores.core.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/cores.core.Commands.split.json (100%) rename solr/{core => solrj}/src/resources/apispec/emptySpec.json (100%) rename solr/{core => solrj}/src/resources/apispec/node.Commands.json (100%) rename solr/{core => solrj}/src/resources/apispec/node.Info.json (100%) rename solr/{core => solrj}/src/resources/apispec/node.invoke.json (100%) rename solr/{core/src/test/org/apache/solr/api => solrj/src/test/org/apache/solr/common/util}/TestPathTrie.java (96%) diff --git a/solr/core/src/java/org/apache/solr/api/Api.java b/solr/core/src/java/org/apache/solr/api/Api.java index 8512c89c956..edc2206b4a6 100644 --- a/solr/core/src/java/org/apache/solr/api/Api.java +++ b/solr/core/src/java/org/apache/solr/api/Api.java @@ -20,6 +20,7 @@ package org.apache.solr.api; import java.util.Map; import com.google.common.collect.ImmutableMap; +import org.apache.solr.common.SpecProvider; import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; diff --git a/solr/core/src/java/org/apache/solr/api/ApiBag.java b/solr/core/src/java/org/apache/solr/api/ApiBag.java index adff11b6a9f..7f30602045a 100644 --- a/solr/core/src/java/org/apache/solr/api/ApiBag.java +++ b/solr/core/src/java/org/apache/solr/api/ApiBag.java @@ -33,6 +33,8 @@ import java.util.stream.Collectors; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SpecProvider; +import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.core.PluginBag; @@ -42,9 +44,8 @@ import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.PermissionNameProvider; -import org.apache.solr.common.util.CommandOperation; import org.apache.solr.util.JsonSchemaValidator; -import org.apache.solr.util.PathTrie; +import org.apache.solr.common.util.PathTrie; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -227,12 +228,6 @@ public class ApiBag { } } - public static SpecProvider getSpec(final String name) { - return () -> { - return ValidatingJsonMap.parse(APISPEC_LOCATION + name + ".json", APISPEC_LOCATION); - }; - } - public static class ReqHandlerToApi extends Api implements PermissionNameProvider { SolrRequestHandler rh; @@ -257,13 +252,10 @@ public class ApiBag { public static List wrapRequestHandlers(final SolrRequestHandler rh, String... specs) { ImmutableList.Builder b = ImmutableList.builder(); - for (String spec : specs) b.add(new ReqHandlerToApi(rh, ApiBag.getSpec(spec))); + for (String spec : specs) b.add(new ReqHandlerToApi(rh, Utils.getSpec(spec))); return b.build(); } - public static final String APISPEC_LOCATION = "apispec/"; - public static final String INTROSPECT = "/_introspect"; - public static final SpecProvider EMPTY_SPEC = () -> ValidatingJsonMap.EMPTY; public static final String HANDLER_NAME = "handlerName"; @@ -276,7 +268,7 @@ public class ApiBag { public void registerLazy(PluginBag.PluginHolder holder, PluginInfo info) { String specName = info.attributes.get("spec"); if (specName == null) specName = "emptySpec"; - register(new LazyLoadedApi(ApiBag.getSpec(specName), holder), Collections.singletonMap(HANDLER_NAME, info.attributes.get(NAME))); + register(new LazyLoadedApi(Utils.getSpec(specName), holder), Collections.singletonMap(HANDLER_NAME, info.attributes.get(NAME))); } public static SpecProvider constructSpec(PluginInfo info) { @@ -286,7 +278,7 @@ public class ApiBag { Map map = (Map) specObj; return () -> ValidatingJsonMap.getDeepCopy(map, 4, false); } else { - return ApiBag.getSpec((String) specObj); + return Utils.getSpec((String) specObj); } } diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java index 76cba9c2ad0..36ca4867eb1 100644 --- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java +++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java @@ -49,7 +49,7 @@ import org.apache.solr.servlet.HttpSolrCall; import org.apache.solr.servlet.SolrDispatchFilter; import org.apache.solr.servlet.SolrRequestParsers; import org.apache.solr.util.JsonSchemaValidator; -import org.apache.solr.util.PathTrie; +import org.apache.solr.common.util.PathTrie; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +58,7 @@ import static org.apache.solr.common.params.CommonParams.WT; import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN; import static org.apache.solr.servlet.SolrDispatchFilter.Action.PASSTHROUGH; import static org.apache.solr.servlet.SolrDispatchFilter.Action.PROCESS; -import static org.apache.solr.util.PathTrie.getPathSegments; +import static org.apache.solr.common.util.PathTrie.getPathSegments; // class that handle the '/v2' path public class V2HttpCall extends HttpSolrCall { @@ -102,7 +102,7 @@ public class V2HttpCall extends HttpSolrCall { String collectionName = origCorename = corename = pieces.get(1); DocCollection collection = getDocCollection(collectionName); if (collection == null) { - if ( ! path.endsWith(ApiBag.INTROSPECT)) { + if ( ! path.endsWith(CommonParams.INTROSPECT)) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no such collection or alias"); } } else { @@ -124,7 +124,7 @@ public class V2HttpCall extends HttpSolrCall { } if (core == null) { log.error(">> path: '" + path + "'"); - if (path.endsWith(ApiBag.INTROSPECT)) { + if (path.endsWith(CommonParams.INTROSPECT)) { initAdminRequest(path); return; } else { @@ -193,7 +193,7 @@ public class V2HttpCall extends HttpSolrCall { Map parts) { fullPath = fullPath == null ? path : fullPath; Api api = requestHandlers.v2lookup(path, method, parts); - if (api == null && path.endsWith(ApiBag.INTROSPECT)) { + if (api == null && path.endsWith(CommonParams.INTROSPECT)) { // the particular http method does not have any , // just try if any other method has this path api = requestHandlers.v2lookup(path, null, parts); @@ -234,7 +234,7 @@ public class V2HttpCall extends HttpSolrCall { private static CompositeApi getSubPathApi(PluginBag requestHandlers, String path, String fullPath, CompositeApi compositeApi) { - String newPath = path.endsWith(ApiBag.INTROSPECT) ? path.substring(0, path.length() - ApiBag.INTROSPECT.length()) : path; + String newPath = path.endsWith(CommonParams.INTROSPECT) ? path.substring(0, path.length() - CommonParams.INTROSPECT.length()) : path; Map> subpaths = new LinkedHashMap<>(); getSubPaths(newPath, requestHandlers.getApiBag(), subpaths); @@ -244,12 +244,12 @@ public class V2HttpCall extends HttpSolrCall { @Override public void call(SolrQueryRequest req1, SolrQueryResponse rsp) { String prefix = null; - prefix = fullPath.endsWith(ApiBag.INTROSPECT) ? - fullPath.substring(0, fullPath.length() - ApiBag.INTROSPECT.length()) : + prefix = fullPath.endsWith(CommonParams.INTROSPECT) ? + fullPath.substring(0, fullPath.length() - CommonParams.INTROSPECT.length()) : fullPath; LinkedHashMap> result = new LinkedHashMap<>(subPaths.size()); for (Map.Entry> e : subPaths.entrySet()) { - if (e.getKey().endsWith(ApiBag.INTROSPECT)) continue; + if (e.getKey().endsWith(CommonParams.INTROSPECT)) continue; result.put(prefix + e.getKey(), e.getValue()); } diff --git a/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandlerApi.java b/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandlerApi.java index 6ba3229c2f2..f7bc14072e5 100644 --- a/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandlerApi.java +++ b/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandlerApi.java @@ -24,7 +24,7 @@ import java.util.Map; import com.google.common.collect.ImmutableMap; import org.apache.solr.common.SolrException; import org.apache.solr.api.Api; -import org.apache.solr.api.ApiBag; +import org.apache.solr.common.util.Utils; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -38,7 +38,7 @@ public class UpdateRequestHandlerApi extends UpdateRequestHandler { } private Api getApiImpl() { - return new Api(ApiBag.getSpec("core.Update")) { + return new Api(Utils.getSpec("core.Update")) { @Override public void call(SolrQueryRequest req, SolrQueryResponse rsp) { String path = req.getPath(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java b/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java index 083ec335f90..087c6f1d0a7 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java @@ -28,7 +28,6 @@ import java.util.Objects; import com.google.common.collect.ImmutableList; import org.apache.solr.api.Api; -import org.apache.solr.api.ApiBag; import org.apache.solr.api.ApiSupport; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionApiMapping.CommandMeta; @@ -73,7 +72,7 @@ public abstract class BaseHandlerApiSupport implements ApiSupport { private Api getApi(final V2EndPoint op) { final BaseHandlerApiSupport apiHandler = this; - return new Api(ApiBag.getSpec(op.getSpecName())) { + return new Api(Utils.getSpec(op.getSpecName())) { @Override public void call(SolrQueryRequest req, SolrQueryResponse rsp) { SolrParams params = req.getParams(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java index c7cd0520be9..a2bfb5ba75e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java @@ -32,7 +32,7 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.api.Api; import static java.util.Collections.singletonList; -import static org.apache.solr.api.ApiBag.getSpec; +import static org.apache.solr.common.util.Utils.getSpec; import static org.apache.solr.common.params.CommonParams.PATH; public class InfoHandler extends RequestHandlerBase { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java index 63b93185697..6e81ccfb3bf 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java @@ -45,7 +45,7 @@ import org.apache.solr.security.PermissionNameProvider; import org.apache.solr.common.util.CommandOperation; import org.apache.solr.api.Api; import org.apache.solr.api.ApiBag.ReqHandlerToApi; -import org.apache.solr.api.SpecProvider; +import org.apache.solr.common.SpecProvider; import org.apache.solr.util.JsonSchemaValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -264,10 +264,10 @@ public abstract class SecurityConfHandler extends RequestHandlerBase implements synchronized (this) { if (apis == null) { Collection apis = new ArrayList<>(); - final SpecProvider authcCommands = ApiBag.getSpec("cluster.security.authentication.Commands"); - final SpecProvider authzCommands = ApiBag.getSpec("cluster.security.authorization.Commands"); - apis.add(new ReqHandlerToApi(this, ApiBag.getSpec("cluster.security.authentication"))); - apis.add(new ReqHandlerToApi(this, ApiBag.getSpec("cluster.security.authorization"))); + final SpecProvider authcCommands = Utils.getSpec("cluster.security.authentication.Commands"); + final SpecProvider authzCommands = Utils.getSpec("cluster.security.authorization.Commands"); + apis.add(new ReqHandlerToApi(this, Utils.getSpec("cluster.security.authentication"))); + apis.add(new ReqHandlerToApi(this, Utils.getSpec("cluster.security.authorization"))); SpecProvider authcSpecProvider = () -> { AuthenticationPlugin authcPlugin = cores.getAuthenticationPlugin(); return authcPlugin != null && authcPlugin instanceof SpecProvider ? diff --git a/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java index 296a2a658b5..eab89e3ca4a 100644 --- a/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java @@ -39,7 +39,7 @@ import org.apache.http.message.BasicHeader; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.common.util.CommandOperation; -import org.apache.solr.api.SpecProvider; +import org.apache.solr.common.SpecProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPlugin.java b/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPlugin.java index c54453dd9a1..6bf2822d624 100644 --- a/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPlugin.java @@ -27,8 +27,8 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; -import org.apache.solr.api.ApiBag; -import org.apache.solr.api.SpecProvider; +import org.apache.solr.common.SpecProvider; +import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.common.util.CommandOperation; import org.slf4j.Logger; @@ -238,7 +238,7 @@ public class RuleBasedAuthorizationPlugin implements AuthorizationPlugin, Config @Override public ValidatingJsonMap getSpec() { - return ApiBag.getSpec("cluster.security.RuleBasedAuthorization").getSpec(); + return Utils.getSpec("cluster.security.RuleBasedAuthorization").getSpec(); } } diff --git a/solr/core/src/java/org/apache/solr/security/Sha256AuthenticationProvider.java b/solr/core/src/java/org/apache/solr/security/Sha256AuthenticationProvider.java index 10ed71a4e1a..4b85c4558c7 100644 --- a/solr/core/src/java/org/apache/solr/security/Sha256AuthenticationProvider.java +++ b/solr/core/src/java/org/apache/solr/security/Sha256AuthenticationProvider.java @@ -31,9 +31,9 @@ import java.util.Set; import com.google.common.collect.ImmutableSet; import org.apache.commons.codec.binary.Base64; import org.apache.solr.common.util.CommandOperation; +import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.ValidatingJsonMap; -import org.apache.solr.api.ApiBag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -161,7 +161,7 @@ public class Sha256AuthenticationProvider implements ConfigEditablePlugin, Basi @Override public ValidatingJsonMap getSpec() { - return ApiBag.getSpec("cluster.security.BasicAuth.Commands").getSpec(); + return Utils.getSpec("cluster.security.BasicAuth.Commands").getSpec(); } static final Set supported_ops = ImmutableSet.of("set-user", "delete-user"); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java b/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java index 2f4fcda97d2..83b1cf34010 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java @@ -42,7 +42,7 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.api.Api; import org.apache.solr.api.V2HttpCall; import org.apache.solr.common.util.CommandOperation; -import org.apache.solr.util.PathTrie; +import org.apache.solr.common.util.PathTrie; import static org.apache.solr.api.ApiBag.EMPTY_SPEC; import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET; diff --git a/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java b/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java index 02e95f7de27..a5e50dd1a2b 100644 --- a/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java +++ b/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.api.ApiBag; import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; @@ -48,7 +47,7 @@ public class JsonValidatorTest extends SolrTestCaseJ4 { public void testSchemaValidation() { - ValidatingJsonMap spec = ApiBag.getSpec("collections.Commands").getSpec(); + ValidatingJsonMap spec = Utils.getSpec("collections.Commands").getSpec(); Map createSchema = spec.getMap("commands", NOT_NULL).getMap("create-alias", NOT_NULL); JsonSchemaValidator validator = new JsonSchemaValidator(createSchema); List errs = validator.validateJson(Utils.fromJSONString("{name : x, collections: [ c1 , c2]}")); @@ -181,7 +180,7 @@ public class JsonValidatorTest extends SolrTestCaseJ4 { } private void checkSchema(String name) { - ValidatingJsonMap spec = ApiBag.getSpec(name).getSpec(); + ValidatingJsonMap spec = Utils.getSpec(name).getSpec(); Map commands = (Map) spec.get("commands"); for (Object o : commands.entrySet()) { Map.Entry cmd = (Map.Entry) o; diff --git a/solr/core/src/java/org/apache/solr/api/SpecProvider.java b/solr/solrj/src/java/org/apache/solr/common/SpecProvider.java similarity index 96% rename from solr/core/src/java/org/apache/solr/api/SpecProvider.java rename to solr/solrj/src/java/org/apache/solr/common/SpecProvider.java index c373c999a3c..d111604487e 100644 --- a/solr/core/src/java/org/apache/solr/api/SpecProvider.java +++ b/solr/solrj/src/java/org/apache/solr/common/SpecProvider.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.api; +package org.apache.solr.common; import org.apache.solr.common.util.ValidatingJsonMap; /**A generic interface for any class that is capable of providing its specification as a json schema diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java index dad586e60aa..3b24f224cd5 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java @@ -17,6 +17,7 @@ package org.apache.solr.common.params; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Locale; import java.util.Set; @@ -184,7 +185,7 @@ public interface CommonParams { String AUTOSCALING_PATH = "/admin/autoscaling"; String AUTOSCALING_DIAGNOSTICS_PATH = "/admin/autoscaling/diagnostics"; - Set ADMIN_PATHS = new HashSet<>(Arrays.asList( + Set ADMIN_PATHS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( CORES_HANDLER_PATH, COLLECTIONS_HANDLER_PATH, CONFIGSETS_HANDLER_PATH, @@ -192,7 +193,9 @@ public interface CommonParams { AUTHZ_PATH, METRICS_PATH, AUTOSCALING_PATH, - AUTOSCALING_DIAGNOSTICS_PATH)); + AUTOSCALING_DIAGNOSTICS_PATH))); + String APISPEC_LOCATION = "apispec/"; + String INTROSPECT = "/_introspect"; /** valid values for: echoParams */ enum EchoParamStyle { diff --git a/solr/core/src/java/org/apache/solr/util/PathTrie.java b/solr/solrj/src/java/org/apache/solr/common/util/PathTrie.java similarity index 99% rename from solr/core/src/java/org/apache/solr/util/PathTrie.java rename to solr/solrj/src/java/org/apache/solr/common/util/PathTrie.java index ceaa5de4fbe..b02a3237d48 100644 --- a/solr/core/src/java/org/apache/solr/util/PathTrie.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/PathTrie.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.solr.util; +package org.apache.solr.common.util; import java.util.ArrayList; import java.util.HashSet; diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java index ce2c305e87d..b64f9712b28 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java @@ -42,8 +42,10 @@ import org.apache.http.util.EntityUtils; import org.apache.solr.common.IteratorWriter; import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SpecProvider; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkOperation; +import org.apache.solr.common.params.CommonParams; import org.apache.zookeeper.KeeperException; import org.noggit.CharArr; import org.noggit.JSONParser; @@ -302,4 +304,10 @@ public class Utils { public static final Pattern ARRAY_ELEMENT_INDEX = Pattern .compile("(\\S*?)\\[(\\d+)\\]"); + + public static SpecProvider getSpec(final String name) { + return () -> { + return ValidatingJsonMap.parse(CommonParams.APISPEC_LOCATION + name + ".json", CommonParams.APISPEC_LOCATION); + }; + } } diff --git a/solr/core/src/resources/apispec/autoscaling.Commands.json b/solr/solrj/src/resources/apispec/autoscaling.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/autoscaling.Commands.json rename to solr/solrj/src/resources/apispec/autoscaling.Commands.json diff --git a/solr/core/src/resources/apispec/cluster.Commands.json b/solr/solrj/src/resources/apispec/cluster.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.Commands.json rename to solr/solrj/src/resources/apispec/cluster.Commands.json diff --git a/solr/core/src/resources/apispec/cluster.aliases.json b/solr/solrj/src/resources/apispec/cluster.aliases.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.aliases.json rename to solr/solrj/src/resources/apispec/cluster.aliases.json diff --git a/solr/core/src/resources/apispec/cluster.commandstatus.delete.json b/solr/solrj/src/resources/apispec/cluster.commandstatus.delete.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.commandstatus.delete.json rename to solr/solrj/src/resources/apispec/cluster.commandstatus.delete.json diff --git a/solr/core/src/resources/apispec/cluster.commandstatus.json b/solr/solrj/src/resources/apispec/cluster.commandstatus.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.commandstatus.json rename to solr/solrj/src/resources/apispec/cluster.commandstatus.json diff --git a/solr/core/src/resources/apispec/cluster.configs.Commands.json b/solr/solrj/src/resources/apispec/cluster.configs.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.configs.Commands.json rename to solr/solrj/src/resources/apispec/cluster.configs.Commands.json diff --git a/solr/core/src/resources/apispec/cluster.configs.delete.json b/solr/solrj/src/resources/apispec/cluster.configs.delete.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.configs.delete.json rename to solr/solrj/src/resources/apispec/cluster.configs.delete.json diff --git a/solr/core/src/resources/apispec/cluster.configs.json b/solr/solrj/src/resources/apispec/cluster.configs.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.configs.json rename to solr/solrj/src/resources/apispec/cluster.configs.json diff --git a/solr/core/src/resources/apispec/cluster.json b/solr/solrj/src/resources/apispec/cluster.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.json rename to solr/solrj/src/resources/apispec/cluster.json diff --git a/solr/core/src/resources/apispec/cluster.nodes.json b/solr/solrj/src/resources/apispec/cluster.nodes.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.nodes.json rename to solr/solrj/src/resources/apispec/cluster.nodes.json diff --git a/solr/core/src/resources/apispec/cluster.security.BasicAuth.Commands.json b/solr/solrj/src/resources/apispec/cluster.security.BasicAuth.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.security.BasicAuth.Commands.json rename to solr/solrj/src/resources/apispec/cluster.security.BasicAuth.Commands.json diff --git a/solr/core/src/resources/apispec/cluster.security.RuleBasedAuthorization.json b/solr/solrj/src/resources/apispec/cluster.security.RuleBasedAuthorization.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.security.RuleBasedAuthorization.json rename to solr/solrj/src/resources/apispec/cluster.security.RuleBasedAuthorization.json diff --git a/solr/core/src/resources/apispec/cluster.security.authentication.Commands.json b/solr/solrj/src/resources/apispec/cluster.security.authentication.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.security.authentication.Commands.json rename to solr/solrj/src/resources/apispec/cluster.security.authentication.Commands.json diff --git a/solr/core/src/resources/apispec/cluster.security.authentication.json b/solr/solrj/src/resources/apispec/cluster.security.authentication.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.security.authentication.json rename to solr/solrj/src/resources/apispec/cluster.security.authentication.json diff --git a/solr/core/src/resources/apispec/cluster.security.authorization.Commands.json b/solr/solrj/src/resources/apispec/cluster.security.authorization.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.security.authorization.Commands.json rename to solr/solrj/src/resources/apispec/cluster.security.authorization.Commands.json diff --git a/solr/core/src/resources/apispec/cluster.security.authorization.json b/solr/solrj/src/resources/apispec/cluster.security.authorization.json similarity index 100% rename from solr/core/src/resources/apispec/cluster.security.authorization.json rename to solr/solrj/src/resources/apispec/cluster.security.authorization.json diff --git a/solr/core/src/resources/apispec/collections.Commands.json b/solr/solrj/src/resources/apispec/collections.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/collections.Commands.json rename to solr/solrj/src/resources/apispec/collections.Commands.json diff --git a/solr/core/src/resources/apispec/collections.collection.Commands.json b/solr/solrj/src/resources/apispec/collections.collection.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/collections.collection.Commands.json rename to solr/solrj/src/resources/apispec/collections.collection.Commands.json diff --git a/solr/core/src/resources/apispec/collections.collection.Commands.modify.json b/solr/solrj/src/resources/apispec/collections.collection.Commands.modify.json similarity index 100% rename from solr/core/src/resources/apispec/collections.collection.Commands.modify.json rename to solr/solrj/src/resources/apispec/collections.collection.Commands.modify.json diff --git a/solr/core/src/resources/apispec/collections.collection.Commands.reload.json b/solr/solrj/src/resources/apispec/collections.collection.Commands.reload.json similarity index 100% rename from solr/core/src/resources/apispec/collections.collection.Commands.reload.json rename to solr/solrj/src/resources/apispec/collections.collection.Commands.reload.json diff --git a/solr/core/src/resources/apispec/collections.collection.delete.json b/solr/solrj/src/resources/apispec/collections.collection.delete.json similarity index 100% rename from solr/core/src/resources/apispec/collections.collection.delete.json rename to solr/solrj/src/resources/apispec/collections.collection.delete.json diff --git a/solr/core/src/resources/apispec/collections.collection.json b/solr/solrj/src/resources/apispec/collections.collection.json similarity index 100% rename from solr/core/src/resources/apispec/collections.collection.json rename to solr/solrj/src/resources/apispec/collections.collection.json diff --git a/solr/core/src/resources/apispec/collections.collection.shards.Commands.json b/solr/solrj/src/resources/apispec/collections.collection.shards.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/collections.collection.shards.Commands.json rename to solr/solrj/src/resources/apispec/collections.collection.shards.Commands.json diff --git a/solr/core/src/resources/apispec/collections.collection.shards.shard.Commands.json b/solr/solrj/src/resources/apispec/collections.collection.shards.shard.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/collections.collection.shards.shard.Commands.json rename to solr/solrj/src/resources/apispec/collections.collection.shards.shard.Commands.json diff --git a/solr/core/src/resources/apispec/collections.collection.shards.shard.delete.json b/solr/solrj/src/resources/apispec/collections.collection.shards.shard.delete.json similarity index 100% rename from solr/core/src/resources/apispec/collections.collection.shards.shard.delete.json rename to solr/solrj/src/resources/apispec/collections.collection.shards.shard.delete.json diff --git a/solr/core/src/resources/apispec/collections.collection.shards.shard.replica.delete.json b/solr/solrj/src/resources/apispec/collections.collection.shards.shard.replica.delete.json similarity index 100% rename from solr/core/src/resources/apispec/collections.collection.shards.shard.replica.delete.json rename to solr/solrj/src/resources/apispec/collections.collection.shards.shard.replica.delete.json diff --git a/solr/core/src/resources/apispec/collections.json b/solr/solrj/src/resources/apispec/collections.json similarity index 100% rename from solr/core/src/resources/apispec/collections.json rename to solr/solrj/src/resources/apispec/collections.json diff --git a/solr/core/src/resources/apispec/core.RealtimeGet.json b/solr/solrj/src/resources/apispec/core.RealtimeGet.json similarity index 100% rename from solr/core/src/resources/apispec/core.RealtimeGet.json rename to solr/solrj/src/resources/apispec/core.RealtimeGet.json diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.addCopyField.json b/solr/solrj/src/resources/apispec/core.SchemaEdit.addCopyField.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaEdit.addCopyField.json rename to solr/solrj/src/resources/apispec/core.SchemaEdit.addCopyField.json diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.addField.json b/solr/solrj/src/resources/apispec/core.SchemaEdit.addField.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaEdit.addField.json rename to solr/solrj/src/resources/apispec/core.SchemaEdit.addField.json diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.addFieldType.analyzers.json b/solr/solrj/src/resources/apispec/core.SchemaEdit.addFieldType.analyzers.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaEdit.addFieldType.analyzers.json rename to solr/solrj/src/resources/apispec/core.SchemaEdit.addFieldType.analyzers.json diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.addFieldType.json b/solr/solrj/src/resources/apispec/core.SchemaEdit.addFieldType.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaEdit.addFieldType.json rename to solr/solrj/src/resources/apispec/core.SchemaEdit.addFieldType.json diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.deleteCopyField.json b/solr/solrj/src/resources/apispec/core.SchemaEdit.deleteCopyField.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaEdit.deleteCopyField.json rename to solr/solrj/src/resources/apispec/core.SchemaEdit.deleteCopyField.json diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.deleteDynamicField.json b/solr/solrj/src/resources/apispec/core.SchemaEdit.deleteDynamicField.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaEdit.deleteDynamicField.json rename to solr/solrj/src/resources/apispec/core.SchemaEdit.deleteDynamicField.json diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.deleteField.json b/solr/solrj/src/resources/apispec/core.SchemaEdit.deleteField.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaEdit.deleteField.json rename to solr/solrj/src/resources/apispec/core.SchemaEdit.deleteField.json diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.deleteFieldType.json b/solr/solrj/src/resources/apispec/core.SchemaEdit.deleteFieldType.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaEdit.deleteFieldType.json rename to solr/solrj/src/resources/apispec/core.SchemaEdit.deleteFieldType.json diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.json b/solr/solrj/src/resources/apispec/core.SchemaEdit.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaEdit.json rename to solr/solrj/src/resources/apispec/core.SchemaEdit.json diff --git a/solr/core/src/resources/apispec/core.SchemaRead.copyFields.json b/solr/solrj/src/resources/apispec/core.SchemaRead.copyFields.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaRead.copyFields.json rename to solr/solrj/src/resources/apispec/core.SchemaRead.copyFields.json diff --git a/solr/core/src/resources/apispec/core.SchemaRead.dynamicFields_fieldTypes.json b/solr/solrj/src/resources/apispec/core.SchemaRead.dynamicFields_fieldTypes.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaRead.dynamicFields_fieldTypes.json rename to solr/solrj/src/resources/apispec/core.SchemaRead.dynamicFields_fieldTypes.json diff --git a/solr/core/src/resources/apispec/core.SchemaRead.fields.json b/solr/solrj/src/resources/apispec/core.SchemaRead.fields.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaRead.fields.json rename to solr/solrj/src/resources/apispec/core.SchemaRead.fields.json diff --git a/solr/core/src/resources/apispec/core.SchemaRead.json b/solr/solrj/src/resources/apispec/core.SchemaRead.json similarity index 100% rename from solr/core/src/resources/apispec/core.SchemaRead.json rename to solr/solrj/src/resources/apispec/core.SchemaRead.json diff --git a/solr/core/src/resources/apispec/core.Update.json b/solr/solrj/src/resources/apispec/core.Update.json similarity index 100% rename from solr/core/src/resources/apispec/core.Update.json rename to solr/solrj/src/resources/apispec/core.Update.json diff --git a/solr/core/src/resources/apispec/core.config.Commands.addRequestHandler.properties.json b/solr/solrj/src/resources/apispec/core.config.Commands.addRequestHandler.properties.json similarity index 100% rename from solr/core/src/resources/apispec/core.config.Commands.addRequestHandler.properties.json rename to solr/solrj/src/resources/apispec/core.config.Commands.addRequestHandler.properties.json diff --git a/solr/core/src/resources/apispec/core.config.Commands.generic.json b/solr/solrj/src/resources/apispec/core.config.Commands.generic.json similarity index 100% rename from solr/core/src/resources/apispec/core.config.Commands.generic.json rename to solr/solrj/src/resources/apispec/core.config.Commands.generic.json diff --git a/solr/core/src/resources/apispec/core.config.Commands.json b/solr/solrj/src/resources/apispec/core.config.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/core.config.Commands.json rename to solr/solrj/src/resources/apispec/core.config.Commands.json diff --git a/solr/core/src/resources/apispec/core.config.Commands.runtimeLib.json b/solr/solrj/src/resources/apispec/core.config.Commands.runtimeLib.json similarity index 100% rename from solr/core/src/resources/apispec/core.config.Commands.runtimeLib.json rename to solr/solrj/src/resources/apispec/core.config.Commands.runtimeLib.json diff --git a/solr/core/src/resources/apispec/core.config.Params.Commands.json b/solr/solrj/src/resources/apispec/core.config.Params.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/core.config.Params.Commands.json rename to solr/solrj/src/resources/apispec/core.config.Params.Commands.json diff --git a/solr/core/src/resources/apispec/core.config.Params.json b/solr/solrj/src/resources/apispec/core.config.Params.json similarity index 100% rename from solr/core/src/resources/apispec/core.config.Params.json rename to solr/solrj/src/resources/apispec/core.config.Params.json diff --git a/solr/core/src/resources/apispec/core.config.json b/solr/solrj/src/resources/apispec/core.config.json similarity index 100% rename from solr/core/src/resources/apispec/core.config.json rename to solr/solrj/src/resources/apispec/core.config.json diff --git a/solr/core/src/resources/apispec/core.system.blob.json b/solr/solrj/src/resources/apispec/core.system.blob.json similarity index 100% rename from solr/core/src/resources/apispec/core.system.blob.json rename to solr/solrj/src/resources/apispec/core.system.blob.json diff --git a/solr/core/src/resources/apispec/core.system.blob.upload.json b/solr/solrj/src/resources/apispec/core.system.blob.upload.json similarity index 100% rename from solr/core/src/resources/apispec/core.system.blob.upload.json rename to solr/solrj/src/resources/apispec/core.system.blob.upload.json diff --git a/solr/core/src/resources/apispec/cores.Commands.json b/solr/solrj/src/resources/apispec/cores.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/cores.Commands.json rename to solr/solrj/src/resources/apispec/cores.Commands.json diff --git a/solr/core/src/resources/apispec/cores.Status.json b/solr/solrj/src/resources/apispec/cores.Status.json similarity index 100% rename from solr/core/src/resources/apispec/cores.Status.json rename to solr/solrj/src/resources/apispec/cores.Status.json diff --git a/solr/core/src/resources/apispec/cores.core.Commands.json b/solr/solrj/src/resources/apispec/cores.core.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/cores.core.Commands.json rename to solr/solrj/src/resources/apispec/cores.core.Commands.json diff --git a/solr/core/src/resources/apispec/cores.core.Commands.split.json b/solr/solrj/src/resources/apispec/cores.core.Commands.split.json similarity index 100% rename from solr/core/src/resources/apispec/cores.core.Commands.split.json rename to solr/solrj/src/resources/apispec/cores.core.Commands.split.json diff --git a/solr/core/src/resources/apispec/emptySpec.json b/solr/solrj/src/resources/apispec/emptySpec.json similarity index 100% rename from solr/core/src/resources/apispec/emptySpec.json rename to solr/solrj/src/resources/apispec/emptySpec.json diff --git a/solr/core/src/resources/apispec/node.Commands.json b/solr/solrj/src/resources/apispec/node.Commands.json similarity index 100% rename from solr/core/src/resources/apispec/node.Commands.json rename to solr/solrj/src/resources/apispec/node.Commands.json diff --git a/solr/core/src/resources/apispec/node.Info.json b/solr/solrj/src/resources/apispec/node.Info.json similarity index 100% rename from solr/core/src/resources/apispec/node.Info.json rename to solr/solrj/src/resources/apispec/node.Info.json diff --git a/solr/core/src/resources/apispec/node.invoke.json b/solr/solrj/src/resources/apispec/node.invoke.json similarity index 100% rename from solr/core/src/resources/apispec/node.invoke.json rename to solr/solrj/src/resources/apispec/node.invoke.json diff --git a/solr/core/src/test/org/apache/solr/api/TestPathTrie.java b/solr/solrj/src/test/org/apache/solr/common/util/TestPathTrie.java similarity index 96% rename from solr/core/src/test/org/apache/solr/api/TestPathTrie.java rename to solr/solrj/src/test/org/apache/solr/common/util/TestPathTrie.java index d4cbf32b102..2eea2db7ca8 100644 --- a/solr/core/src/test/org/apache/solr/api/TestPathTrie.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/TestPathTrie.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.solr.api; +package org.apache.solr.common.util; import java.util.HashMap; import java.util.HashSet; @@ -23,7 +23,7 @@ import java.util.Set; import com.google.common.collect.ImmutableSet; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.util.PathTrie; +import org.apache.solr.common.util.PathTrie; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; From 9c90cae97e6916bb871c9617e8bb028394ee31fb Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 8 Jun 2017 06:40:02 -0400 Subject: [PATCH 023/131] remove some dead code and refactor a bit --- .../blocktree/BlockTreeTermsReader.java | 37 ++++++++----------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java index 8d31f18f51a..0e14bf74182 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java @@ -121,12 +121,6 @@ public final class BlockTreeTermsReader extends FieldsProducer { private final TreeMap fields = new TreeMap<>(); - /** File offset where the directory starts in the terms file. */ - private long dirOffset; - - /** File offset where the directory starts in the index file. */ - private long indexDirOffset; - final String segment; final int version; @@ -167,8 +161,8 @@ public final class BlockTreeTermsReader extends FieldsProducer { CodecUtil.retrieveChecksum(termsIn); // Read per-field details - seekDir(termsIn, dirOffset); - seekDir(indexIn, indexDirOffset); + seekDir(termsIn); + seekDir(indexIn); final int numFields = termsIn.readVInt(); if (numFields < 0) { @@ -181,13 +175,7 @@ public final class BlockTreeTermsReader extends FieldsProducer { if (numTerms <= 0) { throw new CorruptIndexException("Illegal numTerms for field number: " + field, termsIn); } - final int numBytes = termsIn.readVInt(); - if (numBytes < 0) { - throw new CorruptIndexException("invalid rootCode for field number: " + field + ", numBytes=" + numBytes, termsIn); - } - final BytesRef rootCode = new BytesRef(new byte[numBytes]); - termsIn.readBytes(rootCode.bytes, 0, numBytes); - rootCode.length = numBytes; + final BytesRef rootCode = readBytesRef(termsIn); final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field); if (fieldInfo == null) { throw new CorruptIndexException("invalid field number: " + field, termsIn); @@ -230,19 +218,24 @@ public final class BlockTreeTermsReader extends FieldsProducer { } private static BytesRef readBytesRef(IndexInput in) throws IOException { + int numBytes = in.readVInt(); + if (numBytes < 0) { + throw new CorruptIndexException("invalid bytes length: " + numBytes, in); + } + BytesRef bytes = new BytesRef(); - bytes.length = in.readVInt(); - bytes.bytes = new byte[bytes.length]; - in.readBytes(bytes.bytes, 0, bytes.length); + bytes.length = numBytes; + bytes.bytes = new byte[numBytes]; + in.readBytes(bytes.bytes, 0, numBytes); + return bytes; } /** Seek {@code input} to the directory offset. */ - private void seekDir(IndexInput input, long dirOffset) - throws IOException { + private static void seekDir(IndexInput input) throws IOException { input.seek(input.length() - CodecUtil.footerLength() - 8); - dirOffset = input.readLong(); - input.seek(dirOffset); + long offset = input.readLong(); + input.seek(offset); } // for debugging From eefa32860c68b47d98df3fdbda26b276a1d89c1f Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Wed, 7 Jun 2017 14:58:47 -0500 Subject: [PATCH 024/131] Ref Guide: Change PDF description list font to bold instead of italic for better readability --- solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml b/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml index a313c94520d..e0ce2f24cfe 100644 --- a/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml +++ b/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml @@ -194,7 +194,7 @@ thematic_break: margin_top: $vertical_rhythm * 0.5 margin_bottom: $vertical_rhythm * 1.5 description_list: - term_font_style: italic + term_font_style: bold term_spacing: $vertical_rhythm / 4 description_indent: $horizontal_rhythm * 1.25 outline_list: From ccb1987127a1c96c4f69f7cd807bfe4df5eb0224 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Wed, 7 Jun 2017 14:59:20 -0500 Subject: [PATCH 025/131] Ref Guide: remove errant + in TIP formatting --- solr/solr-ref-guide/meta-docs/publish.adoc | 1 - 1 file changed, 1 deletion(-) diff --git a/solr/solr-ref-guide/meta-docs/publish.adoc b/solr/solr-ref-guide/meta-docs/publish.adoc index 642143a1887..85c9719b6c4 100644 --- a/solr/solr-ref-guide/meta-docs/publish.adoc +++ b/solr/solr-ref-guide/meta-docs/publish.adoc @@ -161,7 +161,6 @@ Tip:: + //TODO update Jenkins link If you do not have the required dependencies, and don't choose to install them, you can download the files from the Jenkins (https://builds.apache.org/job/Solr-reference-guide-jira-SOLR-10290/lastSuccessfulBuild/artifact/solr/build/solr-ref-guide/html-site/[Solr Reference Guide job]). But these HTML pages will have the `DRAFT` status noted in them and will not be suitable for publishing. -+ === Publish a New Guide // A lot of this was copied from https://wiki.apache.org/lucene-java/ReleaseTodo#Website_.2B-.3D_javadocs. See that section for explanations for why some steps are required. From 0eb4b575cc15296360e06f6f92136463bdc7a291 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Thu, 8 Jun 2017 09:18:03 -0500 Subject: [PATCH 026/131] Ref Guide: fix incorrect admonition type style definition in theme --- solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml b/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml index e0ce2f24cfe..e8366ccf7e4 100644 --- a/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml +++ b/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml @@ -144,7 +144,7 @@ admonition: caution: name: fa-fire stroke_color: '#bf6900' - information: + important: name: fa-bolt stroke_color: '#eeea74' blockquote: From f0bd43582acc24fa698b156a9d16f62b833011d4 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Thu, 8 Jun 2017 09:55:41 -0500 Subject: [PATCH 027/131] Ref Guide: reduce size of quote blocks --- solr/solr-ref-guide/src/css/ref-guide.css | 2 +- solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/solr/solr-ref-guide/src/css/ref-guide.css b/solr/solr-ref-guide/src/css/ref-guide.css index a17a07dbc8f..617bc1822cc 100644 --- a/solr/solr-ref-guide/src/css/ref-guide.css +++ b/solr/solr-ref-guide/src/css/ref-guide.css @@ -1746,7 +1746,7 @@ table.pyhltable .linenodiv word-spacing: .1em; letter-spacing: 0; font-style: italic; - font-size: 1.15rem; + font-size: inherit; line-height: 1.75; } diff --git a/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml b/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml index e8366ccf7e4..843dfc3bc90 100644 --- a/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml +++ b/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml @@ -149,7 +149,8 @@ admonition: stroke_color: '#eeea74' blockquote: font_color: $base_font_color - font_size: $base_font_size_large + font_size: $base_font_size + font_style: italic border_color: $base_border_color border_width: 5 padding: [$vertical_rhythm / 2, $horizontal_rhythm, $vertical_rhythm / -2, $horizontal_rhythm + $blockquote_border_width / 2] From d411dceaec7db6837e39565b3f431a510fa7f4e1 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Thu, 8 Jun 2017 10:53:22 -0500 Subject: [PATCH 028/131] Ref Guide: several minor typos and format fixes for 6.6 Guide --- .../src/basic-authentication-plugin.adoc | 2 +- solr/solr-ref-guide/src/collections-api.adoc | 21 ++- solr/solr-ref-guide/src/configsets-api.adoc | 2 +- .../src/configuring-solrconfig-xml.adoc | 1 + .../detecting-languages-during-indexing.adoc | 2 +- solr/solr-ref-guide/src/enabling-ssl.adoc | 6 +- solr/solr-ref-guide/src/faceting.adoc | 4 +- solr/solr-ref-guide/src/function-queries.adoc | 134 +++++++++++++++--- solr/solr-ref-guide/src/jvm-settings.adoc | 2 +- .../src/kerberos-authentication-plugin.adoc | 4 +- .../solr-ref-guide/src/language-analysis.adoc | 6 +- .../src/making-and-restoring-backups.adoc | 4 +- solr/solr-ref-guide/src/morelikethis.adoc | 2 +- solr/solr-ref-guide/src/other-parsers.adoc | 23 ++- .../src/replication-screen.adoc | 6 +- .../solr-ref-guide/src/result-clustering.adoc | 1 - solr/solr-ref-guide/src/schema-api.adoc | 2 +- solr/solr-ref-guide/src/schemaless-mode.adoc | 2 +- .../solr-ref-guide/src/stream-decorators.adoc | 4 +- .../src/taking-solr-to-production.adoc | 18 ++- ...ding-analyzers-tokenizers-and-filters.adoc | 8 +- .../src/updatehandlers-in-solrconfig.adoc | 2 +- ...ing-with-external-files-and-processes.adoc | 26 ++-- .../src/zookeeper-access-control.adoc | 15 +- 24 files changed, 201 insertions(+), 96 deletions(-) diff --git a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc index 1143d586cc0..057d9dfd72d 100644 --- a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc @@ -113,7 +113,7 @@ curl --user solr:SolrRocks http://localhost:8983/solr/admin/authentication -H 'C ---- [[BasicAuthenticationPlugin-Setaproperty]] -=== Set a property +=== Set a Property Set arbitrary properties for authentication plugin. The only supported property is `'blockUnknown'` diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc index 25df02f17b3..78869b69884 100644 --- a/solr/solr-ref-guide/src/collections-api.adoc +++ b/solr/solr-ref-guide/src/collections-api.adoc @@ -32,7 +32,7 @@ The Collections API is used to enable you to create, remove, or reload collectio // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="20,15,10,15,40",options="header"] +[cols="25,10,10,15,40",options="header"] |=== |Key |Type |Required |Default |Description |name |string |Yes | |The name of the collection to be created. @@ -225,7 +225,7 @@ Shard splitting can be a long running process. In order to avoid timeouts, you s // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="20,15,10,55",options="header"] +[cols="25,15,10,50",options="header"] |=== |Key |Type |Required |Description |collection |string |Yes |The name of the collection that includes the shard to be split. @@ -329,7 +329,7 @@ Shards can only created with this API for collections that use the 'implicit' ro // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="20,15,10,55",options="header"] +[cols="25,15,10,50",options="header"] |=== |Key |Type |Required |Description |collection |string |Yes |The name of the collection that includes the shard that will be splitted. @@ -380,7 +380,7 @@ Deleting a shard will unload all replicas of the shard, remove them from `cluste *Query Parameters* -[cols=",,,",options="header",] +[cols="25,15,10,50",options="header",] |=== |Key |Type |Required |Description |collection |string |Yes |The name of the collection that includes the shard to be deleted. @@ -692,7 +692,7 @@ Add a replica to a shard in a collection. The node name can be specified if the // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="20,15,10,55",options="header"] +[cols="25,15,10,50",options="header"] |=== |Key |Type |Required |Description |collection |string |Yes |The name of the collection. @@ -814,7 +814,7 @@ Please note that the migrate API does not perform any de-duplication on the docu // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="20,15,10,55",options="header"] +[cols="25,15,10,50",options="header"] |=== |Key |Type |Required |Description |collection |string |Yes |The name of the source collection from which documents will be split. @@ -1474,10 +1474,10 @@ The property to add. Note: this will have the literal 'property.' prepended to d and `property=property.special` - -There is one pre-defined property "preferredLeader" for which shardUnique is forced to 'true' and an error returned if shardUnique is explicitly set to 'false'. PreferredLeader is a boolean property, any value assigned that is not equal (case insensitive) to 'true' will be interpreted as 'false' for preferredLeader. |property.value |string |Yes |The value to assign to the property. -|shardUnique (1) |Boolean |No |default: false. If true, then setting this property in one replica will remove the property from all other replicas in that shard. +|shardUnique |Boolean |No |default: false. If true, then setting this property in one replica will remove the property from all other replicas in that shard. + +There is one pre-defined property `preferredLeader` for which `shardUnique` is forced to 'true' and an error returned if `shardUnique` is explicitly set to 'false'. `PreferredLeader` is a boolean property, any value assigned that is not equal (case insensitive) to 'true' will be interpreted as 'false' for `preferredLeader`. |=== [[CollectionsAPI-Output.13]] @@ -1559,7 +1559,6 @@ The property to add. Note: this will have the literal 'property.' prepended to d and `property=property.special` - |=== [[CollectionsAPI-Output.14]] @@ -1854,7 +1853,7 @@ Additionally, there are several parameters that can be overridden: // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="20,15,10,55",options="header"] +[cols="25,15,10,50",options="header"] |=== |Key |Type |Required |Description |collection.configName |String |No |Defines the name of the configurations to use for this collection. These must already be stored in ZooKeeper. If not provided, Solr will default to the collection name as the configuration name. diff --git a/solr/solr-ref-guide/src/configsets-api.adoc b/solr/solr-ref-guide/src/configsets-api.adoc index dcc3c334d3a..2bd50ffc4ba 100644 --- a/solr/solr-ref-guide/src/configsets-api.adoc +++ b/solr/solr-ref-guide/src/configsets-api.adoc @@ -48,7 +48,7 @@ Create a ConfigSet, based on an existing ConfigSet. // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="20,15,10,15,40",options="header"] +[cols="25,10,10,10,45",options="header"] |=== |Key |Type |Required |Default |Description |name |String |Yes | |ConfigSet to be created diff --git a/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc b/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc index 61f78b42a41..c3a3021f0ff 100644 --- a/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc +++ b/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc @@ -125,6 +125,7 @@ The path and name of the `solrcore.properties` file can be overridden using the Every Solr core has a `core.properties` file, automatically created when using the APIs. When you create a SolrCloud collection, you can pass through custom parameters to go into each core.properties that will be created, by prefixing the parameter name with "property." as a URL parameter. Example: +[source,bash] http://localhost:8983/solr/admin/collections?action=CREATE&name=gettingstarted&numShards=1&property.my.custom.prop=edismax That would create a `core.properties` file that has at least the following properties (others omitted for brevity): diff --git a/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc b/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc index 1bcddecffda..26786e425e9 100644 --- a/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc +++ b/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc @@ -73,7 +73,7 @@ As previously mentioned, both implementations of the `langid` UpdateRequestProce // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="20,10,10,10,50",options="header"] +[cols="30,10,10,10,40",options="header"] |=== |Parameter |Type |Default |Required |Description |langid |Boolean |true |no |Enables and disables language detection. diff --git a/solr/solr-ref-guide/src/enabling-ssl.adoc b/solr/solr-ref-guide/src/enabling-ssl.adoc index ec6b59d079c..c26e4af8423 100644 --- a/solr/solr-ref-guide/src/enabling-ssl.adoc +++ b/solr/solr-ref-guide/src/enabling-ssl.adoc @@ -74,7 +74,7 @@ openssl pkcs12 -nokeys -in solr-ssl.keystore.p12 -out solr-ssl.cacert.pem ---- [[EnablingSSL-SetcommonSSLrelatedsystemproperties]] -=== Set common SSL related system properties +=== Set Common SSL-Related System Properties The Solr Control Script is already setup to pass SSL-related Java system properties to the JVM. To activate the SSL settings, uncomment and update the set of properties beginning with SOLR_SSL_* in `bin/solr.in.sh`. (or `bin\solr.in.cmd` on Windows). @@ -134,7 +134,7 @@ bin\solr.cmd -p 8984 ---- [[EnablingSSL-SolrCloud]] -== SolrCloud +== SSL with SolrCloud This section describes how to run a two-node SolrCloud cluster with no initial collections and a single-node external ZooKeeper. The commands below assume you have already created the keystore described above. @@ -167,7 +167,7 @@ If you have set up your ZooKeeper cluster to use a < will include values greater or equal to 1 and lower than 10 * [1,10] -> will include values greater or equal to 1 and lower or equal to 10 -The initial and end values cannot be empty. If the interval needs to be unbounded, the special character `*` can be used for both, start and end limit. +The initial and end values cannot be empty. -When using `\*`, `(` and `[`, and `)` and `]` will be treated equal. `[*,*]` will include all documents with a value in the field. +If the interval needs to be unbounded, the special character `*` can be used for both, start and end limit. When using this special character, the start syntax options (`(` anAd `[`), and end syntax options (`)` and `]`) will be treated the same. `[*,*]` will include all documents with a value in the field. The interval limits may be strings but there is no need to add quotes. All the text until the comma will be treated as the start limit, and the text after that will be the end limit. For example: `[Buenos Aires,New York]`. Keep in mind that a string-like comparison will be done to match documents in string intervals (case-sensitive). The comparator can't be changed. diff --git a/solr/solr-ref-guide/src/function-queries.adoc b/solr/solr-ref-guide/src/function-queries.adoc index 7e03ae3d34f..86a7d02ff13 100644 --- a/solr/solr-ref-guide/src/function-queries.adoc +++ b/solr/solr-ref-guide/src/function-queries.adoc @@ -86,16 +86,36 @@ The table below summarizes the functions available for function queries. |=== |Function |Description |Syntax Examples |abs |Returns the absolute value of the specified value or function. |`abs(x)` `abs(-5)` -|childfield(field)|Returns the value of the given field for one of the matched child docs when searching by <>. It can be used only in `sort` parameter| + +|childfield(field) |Returns the value of the given field for one of the matched child docs when searching by <>. It can be used only in `sort` parameter. a| * `sort=childfield(name) asc` implies `$q` as a second argument and therefor it assumes `q={!parent ..}..`; * `sort=childfield(field,$bjq) asc` refers to a separate parameter `bjq={!parent ..}..`; -* `sort=childfield(field,{!parent of=...}...) desc` allows to inline block join parent query +* `sort=childfield(field,{!parent of=...}...) desc` allows to inline block join parent query + |concat(v,f..)|concatenates the given string fields, literals and other functions |`concat(name," ",$param,def(opt,"-"))` + |"constant" |Specifies a floating point constant. |`1.5` + |def |`def` is short for default. Returns the value of field "field", or if the field does not exist, returns the default value specified. and yields the first value where `exists()==true`.) |`def(rating,5):` This `def()` function returns the rating, or if no rating specified in the doc, returns 5 `def(myfield, 1.0):` equivalent to `if(exists(myfield),myfield,1.0)` -|div |Divides one value or function by another. div(x,y) divides x by y. |`div(1,y)` `div(sum(x,100),max(y,1))` -|dist |Return the distance between two vectors (points) in an n-dimensional space. Takes in the power, plus two or more ValueSource instances and calculates the distances between the two vectors. Each ValueSource must be a number. There must be an even number of ValueSource instances passed in and the method assumes that the first half represent the first vector and the second half represent the second vector. |`dist(2, x, y, 0, 0):` calculates the Euclidean distance between (0,0) and (x,y) for each document `dist(1, x, y, 0, 0)`: calculates the Manhattan (taxicab) distance between (0,0) and (x,y) for each document `dist(2, x,y,z,0,0,0):` Euclidean distance between (0,0,0) and (x,y,z) for each document. `dist(1,x,y,z,e,f,g)`: Manhattan distance between (x,y,z) and (e,f,g) where each letter is a field name -|docfreq(field,val) |Returns the number of documents that contain the term in the field. This is a constant (the same value for all documents in the index). You can quote the term if it's more complex, or do parameter substitution for the term value. |`docfreq(text,'solr')` `...&defType=func` `&q=docfreq(text,$myterm)&myterm=solr` + +|div |Divides one value or function by another. div(x,y) divides x by y. a|`div(1,y)` + +`div(sum(x,100),max(y,1))` + +|dist |Return the distance between two vectors (points) in an n-dimensional space. Takes in the power, plus two or more ValueSource instances and calculates the distances between the two vectors. Each ValueSource must be a number. There must be an even number of ValueSource instances passed in and the method assumes that the first half represent the first vector and the second half represent the second vector. a|`dist(2, x, y, 0, 0):` calculates the Euclidean distance between (0,0) and (x,y) for each document. + +`dist(1, x, y, 0, 0)`: calculates the Manhattan (taxicab) distance between (0,0) and (x,y) for each document. + +`dist(2, x,y,z,0,0,0):` Euclidean distance between (0,0,0) and (x,y,z) for each document. + +`dist(1,x,y,z,e,f,g)`: Manhattan distance between (x,y,z) and (e,f,g) where each letter is a field name. + +|docfreq(field,val) |Returns the number of documents that contain the term in the field. This is a constant (the same value for all documents in the index). + +You can quote the term if it's more complex, or do parameter substitution for the term value. a|`docfreq(text,'solr')` + +`...&defType=func` `&q=docfreq(text,$myterm)&myterm=solr` + |field[[FunctionQueries-field]] a| Returns the numeric docValues or indexed value of the field with the specified name. In its simplest (single argument) form, this function can only be used on single valued fields, and can be called using the name of the field as a string, or for most conventional field names simply use the field name by itself with out using the `field(...)` syntax. @@ -120,7 +140,9 @@ For multivalued docValues fields: * `field(myMultiValuedFloatField,max)` |hsin |The Haversine distance calculates the distance between two points on a sphere when traveling along the sphere. The values must be in radians. `hsin` also take a Boolean argument to specify whether the function should convert its output to radians. |`hsin(2, true, x, y, 0, 0)` -|idf |Inverse document frequency; a measure of whether the term is common or rare across all documents. Obtained by dividing the total number of documents by the number of documents containing the term, and then taking the logarithm of that quotient. See also `tf`. |`idf(fieldName,'solr')`: measures the inverse of the frequency of the occurrence of the term `'solr'` in` fieldName`. + +|idf |Inverse document frequency; a measure of whether the term is common or rare across all documents. Obtained by dividing the total number of documents by the number of documents containing the term, and then taking the logarithm of that quotient. See also `tf`. |`idf(fieldName,'solr')`: measures the inverse of the frequency of the occurrence of the term `'solr'` in `fieldName`. + |if a| Enables conditional function queries. In `if(test,value1,value2)`: @@ -130,8 +152,12 @@ Enables conditional function queries. In `if(test,value1,value2)`: An expression can be any function which outputs boolean values, or even functions returning numeric values, in which case value 0 will be interpreted as false, or strings, in which case empty string is interpreted as false. - |`if(termfreq` `(cat,'electronics'),` `popularity,42)` : This function checks each document for the to see if it contains the term "```electronics```" in the `cat` field. If it does, then the value of the `popularity` field is returned, otherwise the value of `42` is returned. -|linear |Implements `m*x+c` where `m` and `c` are constants and `x` is an arbitrary function. This is equivalent to `sum(product(m,x),c)`, but slightly more efficient as it is implemented as a single function. |`linear(x,m,c)` `linear(x,2,4)` returns `2*x+4` + a|`if(termfreq` `(cat,'electronics'),` `popularity,42)`: This function checks each document for the to see if it contains the term "```electronics```" in the `cat` field. If it does, then the value of the `popularity` field is returned, otherwise the value of `42` is returned. + +|linear |Implements `m*x+c` where `m` and `c` are constants and `x` is an arbitrary function. This is equivalent to `sum(product(m,x),c)`, but slightly more efficient as it is implemented as a single function. a|`linear(x,m,c)` + +`linear(x,2,4)` returns `2*x+4` + |log |Returns the log base 10 of the specified function. a| `log(x)` @@ -150,23 +176,37 @@ Returns the maximum numeric value of multiple nested functions or constants, whi (Use the `field(myfield,max)` syntax for <>) |`max(myfield,myotherfield,0)` + |maxdoc |Returns the number of documents in the index, including those that are marked as deleted but have not yet been purged. This is a constant (the same value for all documents in the index). |`maxdoc()` + |min a| Returns the minimum numeric value of multiple nested functions of constants, which are specified as arguments: `min(x,y,...)`. The min function can also be useful for providing an "upper bound" on a function using a constant. (Use the `field(myfield,min)` <>) |`min(myfield,myotherfield,0)` + |ms a| Returns milliseconds of difference between its arguments. Dates are relative to the Unix or POSIX time epoch, midnight, January 1, 1970 UTC. Arguments may be the name of an indexed `TrieDateField`, or date math based on a <>. * `ms()`: Equivalent to `ms(NOW)`, number of milliseconds since the epoch. * `ms(a):` Returns the number of milliseconds since the epoch that the argument represents. -* `ms(a,b)` : Returns the number of milliseconds that b occurs before a (that is, a - b) +* `ms(a,b)` : Returns the number of milliseconds that b occurs before a (that is, a - b) a|`ms(NOW/DAY)` + +`ms(2000-01-01T00:00:00Z)` + +`ms(mydatefield)` + +`ms(NOW,mydatefield)` + +`ms(mydatefield, 2000-01-01T00:00:00Z)` + +`ms(datefield1, datefield2)` - |`ms(NOW/DAY)` `ms(2000-01-01T00:00:00Z)` `ms(mydatefield)` `ms(NOW,mydatefield)` `ms(mydatefield,` `2000-01-01T00:00:00Z)` `ms(datefield1,` `datefield2)` |norm(_field_) |Returns the "norm" stored in the index for the specified field. This is the product of the index time boost and the length normalization factor, according to the {lucene-javadocs}/core/org/apache/lucene/search/similarities/Similarity.html[Similarity] for the field. |`norm(fieldName)` + |numdocs |Returns the number of documents in the index, not including those that are marked as deleted but have not yet been purged. This is a constant (the same value for all documents in the index). |`numdocs()` + |ord a| Returns the ordinal of the indexed field value within the indexed list of terms for that field in Lucene index order (lexicographically ordered by unicode value), starting at 1. In other words, for a given field, all values are ordered lexicographically; this function then returns the offset of a particular value in that ordering. The field must have a maximum of one value per document (not multi-valued). 0 is returned for documents without a value in the field. @@ -177,7 +217,10 @@ Returns the ordinal of the indexed field value within the indexed list of terms See also `rord` below. - |`ord(myIndexedField)` Example: If there were only three values ("apple","banana","pear") for a particular field X, then: `ord(X) `would be 1 for documents containing "apple", 2 for documnts containing "banana", etc... + a|`ord(myIndexedField)` + +Example: If there were only three values ("apple","banana","pear") for a particular field X, then `ord(X)` would be `1` for documents containing "apple", `2` for documents containing "banana", etc. + |payload a| Returns the float value computed from the decoded payloads of the term specified. The return value is computed using the `min`, `max`, or `average` of the decoded payloads. A special `first` function can be used instead of the others, to short-circuit term enumeration and return only the decoded payload of the first term. The field specified must have float or integer payload encoding capability (via `DelimitedPayloadTokenFilter` or `NumericPayloadTokenFilter`). If no payload is found for the term, the default value is returned. @@ -185,36 +228,83 @@ Returns the float value computed from the decoded payloads of the term specified * `payload(field_name,term,default_value)`: default value can be a constant, field name, or another float returning function. `average` function used. * `payload(field_name,term,default_value,function)`: function values can be `min`, `max`, `average`, or `first`. |`payload(payloaded_field_dpf,term,0.0,first)` -|pow |Raises the specified base to the specified power. `pow(x,y)` raises x to the power of y. |`pow(x,y)` `pow(x,log(y))` `pow(x,0.5):` the same as `sqrt` +|pow |Raises the specified base to the specified power. `pow(x,y)` raises x to the power of y. a|`pow(x,y)` + +`pow(x,log(y))` + +`pow(x,0.5):` the same as `sqrt` + |product |Returns the product of multiple values or functions, which are specified in a comma-separated list. `mul(...)` may also be used as an alias for this function. |`product(x,y,...)` `product(x,2)` `product(x,y)mul(x,y)` -|query |Returns the score for the given subquery, or the default value for documents not matching the query. Any type of subquery is supported through either parameter de-referencing `$otherparam` or direct specification of the query string in the <> through the `v` key. |`query(subquery, default)` `q=product` `(popularity,` ` query({!dismax v='solr rocks'})`: returns the product of the popularity and the score of the DisMax query. `q=product` `(popularity,` ` query($qq))&qq={!dismax}solr rocks`: equivalent to the previous query, using parameter de-referencing. `q=product` `(popularity,` ` query($qq,0.1))` `&qq={!dismax}` `solr rocks`: specifies a default score of 0.1 for documents that don't match the DisMax query. + +|query |Returns the score for the given subquery, or the default value for documents not matching the query. Any type of subquery is supported through either parameter de-referencing `$otherparam` or direct specification of the query string in the <> through the `v` key. a|`query(subquery, default)` + +`q=product` `(popularity,` `query({!dismax v='solr rocks'})`: returns the product of the popularity and the score of the DisMax query. + +`q=product` `(popularity,` `query($qq))&qq={!dismax}solr rocks`: equivalent to the previous query, using parameter de-referencing. + +`q=product` `(popularity,` `query($qq,0.1))` `&qq={!dismax}` `solr rocks`: specifies a default score of 0.1 for documents that don't match the DisMax query. |recip a| Performs a reciprocal function with `recip(x,m,a,b)` implementing `a/(m*x+b)` where `m,a,b` are constants, and `x` is any arbitrarily complex function. When a and b are equal, and x>=0, this function has a maximum value of 1 that drops as x increases. Increasing the value of a and b together results in a movement of the entire function to a flatter part of the curve. These properties can make this an ideal function for boosting more recent documents when x is `rord(datefield)`. - |`recip(myfield,m,a,b)` `recip(rord` `(creationDate),` `1,1000,1000)` +a|`recip(myfield,m,a,b)` + +`recip(rord` `(creationDate), 1,1000,1000)` + |rord |Returns the reverse ordering of that returned by `ord`. |`rord(myDateField)` + |scale a| Scales values of the function x such that they fall between the specified `minTarget` and `maxTarget` inclusive. The current implementation traverses all of the function values to obtain the min and max, so it can pick the correct scale. -The current implementation cannot distinguish when documents have been deleted or documents that have no value. It uses 0.0 values for these cases. This means that if values are normally all greater than 0.0, one can still end up with 0.0 as the min value to map from. In these cases, an appropriate map() function could be used as a workaround to change 0.0 to a value in the real range, as shown here: scale(map(x,0,0,5),1,2) +The current implementation cannot distinguish when documents have been deleted or documents that have no value. It uses 0.0 values for these cases. This means that if values are normally all greater than 0.0, one can still end up with 0.0 as the min value to map from. In these cases, an appropriate map() function could be used as a workaround to change 0.0 to a value in the real range, as shown here: `scale(map(x,0,0,5),1,2)` a|`scale(x, minTarget, maxTarget)` + +`scale(x,1,2)`: scales the values of x such that all values will be between 1 and 2 inclusive. - |`scale(x,` `minTarget,` `maxTarget)` `scale(x,1,2)`: scales the values of x such that all values will be between 1 and 2 inclusive. |sqedist |The Square Euclidean distance calculates the 2-norm (Euclidean distance) but does not take the square root, thus saving a fairly expensive operation. It is often the case that applications that care about Euclidean distance do not need the actual distance, but instead can use the square of the distance. There must be an even number of ValueSource instances passed in and the method assumes that the first half represent the first vector and the second half represent the second vector. |`sqedist(x_td, y_td, 0, 0)` + |sqrt |Returns the square root of the specified value or function. |`sqrt(x)sqrt(100)sqrt(sum(x,100))` -|strdist |Calculate the distance between two strings. Uses the Lucene spell checker `StringDistance` interface and supports all of the implementations available in that package, plus allows applications to plug in their own via Solr's resource loading capabilities. `strdist` takes (string1, string2, distance measure). Possible values for distance measure are: jw: Jaro-Winkler edit: Levenstein or Edit distance ngram: The NGramDistance, if specified, can optionally pass in the ngram size too. Default is 2. FQN: Fully Qualified class Name for an implementation of the StringDistance interface. Must have a no-arg constructor. |`strdist("SOLR",id,edit)` -|sub |Returns x-y from sub(x,y). |`sub(myfield,myfield2)` `sub(100,` `sqrt(myfield))` -|sum |Returns the sum of multiple values or functions, which are specified in a comma-separated list. `add(...)` may be used as an alias for this function |`sum(x,y,...) sum(x,1)` `sum(x,y)` `sum(sqrt(x),log(y),z,0.5)add(x,y)` -|sumtotaltermfreq |Returns the sum of `totaltermfreq` values for all terms in the field in the entire index (i.e., the number of indexed tokens for that field). (Aliases `sumtotaltermfreq` to `sttf`.) |If doc1:(fieldX:A B C) and doc2:(fieldX:A A A A): `docFreq(fieldX:A)` = 2 (A appears in 2 docs) `freq(doc1, fieldX:A)` = 4 (A appears 4 times in doc 2) `totalTermFreq(fieldX:A)` = 5 (A appears 5 times across all docs) `sumTotalTermFreq(fieldX)` = 7 in `fieldX`, there are 5 As, 1 B, 1 C + +|strdist a|Calculate the distance between two strings. Uses the Lucene spell checker `StringDistance` interface and supports all of the implementations available in that package, plus allows applications to plug in their own via Solr's resource loading capabilities. `strdist` takes (string1, string2, distance measure). + +Possible values for distance measure are: + +* jw: Jaro-Winkler +* edit: Levenstein or Edit distance +* ngram: The NGramDistance, if specified, can optionally pass in the ngram size too. Default is 2. +* FQN: Fully Qualified class Name for an implementation of the StringDistance interface. Must have a no-arg constructor. |`strdist("SOLR",id,edit)` + +|sub |Returns `x-y` from `sub(x,y)`. a|`sub(myfield,myfield2)` + +`sub(100, sqrt(myfield))` + +|sum |Returns the sum of multiple values or functions, which are specified in a comma-separated list. `add(...)` may be used as an alias for this function a|`sum(x,y,...) sum(x,1)` + +`sum(x,y)` + +`sum(sqrt(x),log(y),z,0.5)` + +`add(x,y)` + +|sumtotaltermfreq |Returns the sum of `totaltermfreq` values for all terms in the field in the entire index (i.e., the number of indexed tokens for that field). (Aliases `sumtotaltermfreq` to `sttf`.) a|If doc1:(fieldX:A B C) and doc2:(fieldX:A A A A): + +`docFreq(fieldX:A)` = 2 (A appears in 2 docs) + +`freq(doc1, fieldX:A)` = 4 (A appears 4 times in doc 2) + +`totalTermFreq(fieldX:A)` = 5 (A appears 5 times across all docs) + +`sumTotalTermFreq(fieldX)` = 7 in `fieldX`, there are 5 As, 1 B, 1 C + |termfreq |Returns the number of times the term appears in the field for that document. |`termfreq(text,'memory')` + |tf |Term frequency; returns the term frequency factor for the given term, using the {lucene-javadocs}/core/org/apache/lucene/search/similarities/Similarity.html[Similarity] for the field. The `tf-idf` value increases proportionally to the number of times a word appears in the document, but is offset by the frequency of the word in the document, which helps to control for the fact that some words are generally more common than others. See also `idf`. |`tf(text,'solr')` + |top a| Causes the function query argument to derive its values from the top-level IndexReader containing all parts of an index. For example, the ordinal of a value in a single segment will be different from the ordinal of that same value in the complete index. -The `ord()` and `rord()` functions implicitly use `top()`, and hence `ord(foo)` is equivalent to `top(ord(foo))`. +The `ord()` and `rord()` functions implicitly use `top()`, and hence `ord(foo)` is equivalent to `top(ord(foo))`. | - | |totaltermfreq |Returns the number of times the term appears in the field in the entire index. (Aliases `totaltermfreq` to `ttf`.) |`ttf(text,'memory')` |=== diff --git a/solr/solr-ref-guide/src/jvm-settings.adoc b/solr/solr-ref-guide/src/jvm-settings.adoc index 4f2c1e0fe4a..56560da8a66 100644 --- a/solr/solr-ref-guide/src/jvm-settings.adoc +++ b/solr/solr-ref-guide/src/jvm-settings.adoc @@ -51,4 +51,4 @@ If you are using Sun's JVM, add the `-server` command-line option when you start A great way to see what JVM settings your server is using, along with other useful information, is to use the admin RequestHandler, `solr/admin/system`. This request handler will display a wealth of server statistics and settings. -You can also use any of the tools that are compatible with the Java Management Extensions (JMX). See the section _Using JMX with Solr_ in <> for more information. +You can also use any of the tools that are compatible with the Java Management Extensions (JMX). See the section <> for more information. diff --git a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc index 5c95f6c4bd5..e86e0651309 100644 --- a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc @@ -234,7 +234,7 @@ While starting up Solr, the following host-specific parameters need to be passed // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="30,10,60",options="header"] +[cols="35,10,55",options="header"] |=== |Parameter Name |Required |Description |`solr.kerberos.name.rules` |No |Used to map Kerberos principals to short names. Default value is `DEFAULT`. Example of a name rule: `RULE:[1:$1@$0](.\*EXAMPLE.COM)s/@.*//` @@ -281,7 +281,7 @@ To enable delegation tokens, several parameters must be defined. These parameter // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="30,10,60",options="header"] +[cols="40,10,50",options="header"] |=== |Parameter Name |Required |Description |`solr.kerberos.delegation.token.enabled` |Yes, to enable tokens |False by default, set to true to enable delegation tokens. diff --git a/solr/solr-ref-guide/src/language-analysis.adoc b/solr/solr-ref-guide/src/language-analysis.adoc index 11b0b784e41..50180482bd5 100644 --- a/solr/solr-ref-guide/src/language-analysis.adoc +++ b/solr/solr-ref-guide/src/language-analysis.adoc @@ -513,7 +513,7 @@ Solr can stem Catalan using the Snowball Porter Stemmer with an argument of `lan The default configuration of the <> is suitable for Traditional Chinese text. It follows the Word Break rules from the Unicode Text Segmentation algorithm for non-Chinese text, and uses a dictionary to segment Chinese words. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add to your `SOLR_HOME/lib`. <> can also be used to tokenize Traditional Chinese text. Following the Word Break rules from the Unicode Text Segmentation algorithm, it produces one token per Chinese character. When combined with <>, overlapping bigrams of Chinese characters are formed. - + <> folds fullwidth ASCII variants into the equivalent Basic Latin forms. *Examples:* @@ -565,7 +565,7 @@ See the example under < [[LanguageAnalysis-SimplifiedChinese]] === Simplified Chinese -For Simplified Chinese, Solr provides support for Chinese sentence and word segmentation with the <>. This component includes a large dictionary and segments Chinese text into words with the Hidden Markov Model. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add to your `SOLR_HOME/lib`. +For Simplified Chinese, Solr provides support for Chinese sentence and word segmentation with the <>. This component includes a large dictionary and segments Chinese text into words with the Hidden Markov Model. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add to your `SOLR_HOME/lib`. The default configuration of the <> is also suitable for Simplified Chinese text. It follows the Word Break rules from the Unicode Text Segmentation algorithm for non-Chinese text, and uses a dictionary to segment Chinese words. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add to your `SOLR_HOME/lib`. @@ -613,7 +613,7 @@ To use the default setup with fallback to English Porter stemmer for English wor `` -Or to configure your own analysis setup, use the `solr.HMMChineseTokenizerFactory` along with your custom filter setup. See an example of this in the <> section. +Or to configure your own analysis setup, use the `solr.HMMChineseTokenizerFactory` along with your custom filter setup. See an example of this in the <> section. [[LanguageAnalysis-Czech]] === Czech diff --git a/solr/solr-ref-guide/src/making-and-restoring-backups.adoc b/solr/solr-ref-guide/src/making-and-restoring-backups.adoc index 55c08d40947..3d7a17de6b5 100644 --- a/solr/solr-ref-guide/src/making-and-restoring-backups.adoc +++ b/solr/solr-ref-guide/src/making-and-restoring-backups.adoc @@ -22,7 +22,7 @@ If you are worried about data loss, and of course you _should_ be, you need a wa Solr provides two approaches to backing up and restoring Solr cores or collections, depending on how you are running Solr. If you run in SolrCloud mode, you will use the Collections API. If you run Solr in standalone mode, you will use the replication handler. -== [[cloud-backups]]SolrCloud +== SolrCloud Backups Support for backups when running SolrCloud is provided with the <>. This allows the backups to be generated across multiple shards, and restored to the same number of shards and replicas as the original collection. @@ -31,7 +31,7 @@ Two commands are available: * `action=BACKUP`: This command backs up Solr indexes and configurations. More information is available in the section <>. * `action=RESTORE`: This command restores Solr indexes and configurations. More information is available in the section <>. -== Standalone Mode +== Standalone Mode Backups Backups and restoration uses Solr's replication handler. Out of the box, Solr includes implicit support for replication so this API can be used. Configuration of the replication handler can, however, be customized by defining your own replication handler in `solrconfig.xml` . For details on configuring the replication handler, see the section <>. diff --git a/solr/solr-ref-guide/src/morelikethis.adoc b/solr/solr-ref-guide/src/morelikethis.adoc index aa3ccc3224a..ec6129eeb29 100644 --- a/solr/solr-ref-guide/src/morelikethis.adoc +++ b/solr/solr-ref-guide/src/morelikethis.adoc @@ -96,4 +96,4 @@ The table below summarizes parameters accessible through the `MoreLikeThisHandle [[MoreLikeThis-MoreLikeThisQueryParser]] == More Like This Query Parser -The `mlt` query parser provides a mechanism to retrieve documents similar to a given document, like the handler. More information on the usage of the mlt query parser can be found int the section <>. +The `mlt` query parser provides a mechanism to retrieve documents similar to a given document, like the handler. More information on the usage of the mlt query parser can be found in the section <>. diff --git a/solr/solr-ref-guide/src/other-parsers.adoc b/solr/solr-ref-guide/src/other-parsers.adoc index 76607800b10..70329450bf1 100644 --- a/solr/solr-ref-guide/src/other-parsers.adoc +++ b/solr/solr-ref-guide/src/other-parsers.adoc @@ -942,14 +942,33 @@ The {solr-javadocs}/solr-core/org/apache/solr/search/XmlQParserPlugin.html[XmlQP |defType |`xmlparser` |q a| [source,xml] - shirt plain cotton S M L +---- + + + shirt + + + plain + + + cotton + + + + + S M L + + + + +---- |=== The XmlQParser implementation uses the {solr-javadocs}/solr-core/org/apache/solr/search/SolrCoreParser.html[SolrCoreParser] class which extends Lucene's {lucene-javadocs}/queryparser/org/apache/lucene/queryparser/xml/CoreParser.html[CoreParser] class. XML elements are mapped to {lucene-javadocs}/queryparser/org/apache/lucene/queryparser/xml/QueryBuilder.html[QueryBuilder] classes as follows: // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[width="50%",cols="30,70",options="header"] +[width="100%",cols="30,70",options="header"] |=== |XML element |QueryBuilder class | |{lucene-javadocs}/queryparser/org/apache/lucene/queryparser/xml/builders/BooleanQueryBuilder.html[BooleanQueryBuilder] diff --git a/solr/solr-ref-guide/src/replication-screen.adoc b/solr/solr-ref-guide/src/replication-screen.adoc index b4e5407fb03..6dd649f17f6 100644 --- a/solr/solr-ref-guide/src/replication-screen.adoc +++ b/solr/solr-ref-guide/src/replication-screen.adoc @@ -28,10 +28,8 @@ The Replication screen shows you the current replication state for the core you [IMPORTANT] ==== When using <>, do not attempt to disable replication via this screen. + +image::images/replication-screen/replication.png[image,width=412,height=250] ==== -.Sample Replication Screen -image::images/replication-screen/replication.png[image,width=412,height=250] - - More details on how to configure replication is available in the section called <>. diff --git a/solr/solr-ref-guide/src/result-clustering.adoc b/solr/solr-ref-guide/src/result-clustering.adoc index 932e3733ae1..c4fa048366b 100644 --- a/solr/solr-ref-guide/src/result-clustering.adoc +++ b/solr/solr-ref-guide/src/result-clustering.adoc @@ -358,5 +358,4 @@ Some of these techniques are described in _Apache SOLR and Carrot2 integration s The following resources provide additional information about the clustering component in Solr and its potential applications. * Apache Solr and Carrot2 integration strategies: http://carrot2.github.io/solr-integration-strategies -* Apache Solr Wiki (covers previous Solr versions, may be inaccurate): http://carrot2.github.io/solr-integration-strategies * Clustering and Visualization of Solr search results (video from Berlin BuzzWords conference, 2011): http://vimeo.com/26616444 diff --git a/solr/solr-ref-guide/src/schema-api.adoc b/solr/solr-ref-guide/src/schema-api.adoc index 171093688d9..e73da89c3b6 100644 --- a/solr/solr-ref-guide/src/schema-api.adoc +++ b/solr/solr-ref-guide/src/schema-api.adoc @@ -617,7 +617,7 @@ The query parameters can be added to the API request after a '?'. // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="15,15,10,20,40",options="header"] +[cols="20,15,10,15,40",options="header"] |=== |Key |Type |Required |Default |Description |wt |string |No |json |Defines the format of the response. The options are *json* or *xml*. If not specified, JSON will be returned by default. diff --git a/solr/solr-ref-guide/src/schemaless-mode.adoc b/solr/solr-ref-guide/src/schemaless-mode.adoc index 05ad77e2136..fdf97ba538c 100644 --- a/solr/solr-ref-guide/src/schemaless-mode.adoc +++ b/solr/solr-ref-guide/src/schemaless-mode.adoc @@ -69,7 +69,7 @@ You can use the `/schema/fields` <> to co [TIP] ==== -Because the `data_driven_schema_configs` config set includes a `copyField` directive that causes all content to be indexed in a predefined "catch-all" `\_text_` field, to enable single-field search that includes all fields' content, the index will be larger than it would be without the `copyField`. When you nail down your schema, consider removing the `\_text_` field and the corresponding `copyField` directive if you don't need it. +The `data_driven_schema_configs` configset includes a `copyField` directive that causes all content to be indexed in a predefined "catch-all" `\_text_` field, which is used to enable single-field search that includes all fields' content. This will cause the index to be larger than it would be without this "catch-all" `copyField`. When you nail down your schema, consider removing the `\_text_` field and the corresponding `copyField` directive if you don't need it. ==== [[SchemalessMode-ConfiguringSchemalessMode]] diff --git a/solr/solr-ref-guide/src/stream-decorators.adoc b/solr/solr-ref-guide/src/stream-decorators.adoc index abd7c10145c..c4ab4f44491 100644 --- a/solr/solr-ref-guide/src/stream-decorators.adoc +++ b/solr/solr-ref-guide/src/stream-decorators.adoc @@ -416,7 +416,7 @@ hashJoin( == innerJoin -Wraps two streams Left and Right and for every tuple in Left which exists in Right will emit a tuple containing the fields of both tuples. This supports one-one, one-many, many-one, and many-many inner join scenarios. The tuples are emitted in the order in which they appear in the Left stream. Both streams must be sorted by the fields being used to determine equality (the 'on' parameter). If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple. You can wrap the incoming streams with a select(...) to be specific about which field values are included in the emitted tuple. +Wraps two streams, Left and Right. For every tuple in Left which exists in Right a tuple containing the fields of both tuples will be emitted. This supports one-to-one, one-to-many, many-to-one, and many-to-many inner join scenarios. The tuples are emitted in the order in which they appear in the Left stream. Both streams must be sorted by the fields being used to determine equality (the 'on' parameter). If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple. You can wrap the incoming streams with a `select(...)` expression to be specific about which field values are included in the emitted tuple. === innerJoin Parameters @@ -732,7 +732,7 @@ See section in <> start command will refuse to do so. Consequently, you should determine the username of a system user that will own all of the Solr files and the running Solr process. By default, the installation script will create the *solr* user, but you can override this setting using the -u option. If your organization has specific requirements for creating new user accounts, then you should create the user before running the script. The installation script will make the Solr user the owner of the `/opt/solr` and `/var/solr` directories. @@ -103,7 +103,7 @@ We'll cover some additional configuration settings you can make to fine-tune you The Solr home directory (not to be confused with the Solr installation directory) is where Solr manages core directories with index files. By default, the installation script uses `/var/solr/data`. If the `-d` option is used on the install script, then this will change to the `data` subdirectory in the location given to the -d option. Take a moment to inspect the contents of the Solr home directory on your system. If you do not <>, the home directory must contain a `solr.xml` file. When Solr starts up, the Solr Control Script passes the location of the home directory using the `-Dsolr.solr.home=...` system property. [[TakingSolrtoProduction-Environmentoverridesincludefile]] -==== Environment overrides include file +==== Environment Overrides Include File The service installation script creates an environment specific include file that overrides defaults used by the `bin/solr` script. The main advantage of using an include file is that it provides a single location where all of your environment-specific overrides are defined. Take a moment to inspect the contents of the `/etc/default/solr.in.sh` file, which is the default path setup by the installation script. If you used the `-s` option on the install script to change the name of the service, then the first part of the filename will be different. For a service named `solr-demo`, the file will be named `/etc/default/solr-demo.in.sh`. There are many settings that you can override using this file. However, at a minimum, this script needs to define the `SOLR_PID_DIR` and `SOLR_HOME` variables, such as: @@ -116,7 +116,7 @@ SOLR_HOME=/var/solr/data The `SOLR_PID_DIR` variable sets the directory where the <> will write out a file containing the Solr server’s process ID. [[TakingSolrtoProduction-Logsettings]] -==== Log settings +==== Log Settings Solr uses Apache Log4J for logging. The installation script copies `/opt/solr/server/resources/log4j.properties` to `/var/solr/log4j.properties`. Take a moment to verify that the Solr include file is configured to send logs to the correct location by checking the following settings in `/etc/default/solr.in.sh`: @@ -129,7 +129,7 @@ SOLR_LOGS_DIR=/var/solr/logs For more information about Log4J configuration, please see: <> [[TakingSolrtoProduction-init.dscript]] -==== init.d script +==== init.d Script When running a service like Solr on Linux, it’s common to setup an init.d script so that system administrators can control Solr using the service tool, such as: `service solr start`. The installation script creates a very basic init.d script to help you get started. Take a moment to inspect the `/etc/init.d/solr` file, which is the default script name setup by the installation script. If you used the `-s` option on the install script to change the name of the service, then the filename will be different. Notice that the following variables are setup for your environment based on the parameters passed to the installation script: @@ -175,7 +175,7 @@ Solr process PID running on port 8983 If the `status` command is not successful, look for error messages in `/var/solr/logs/solr.log`. [[TakingSolrtoProduction-Finetuneyourproductionsetup]] -== Fine tune your production setup +== Fine-Tune Your Production Setup [[TakingSolrtoProduction-MemoryandGCSettings]] === Memory and GC Settings @@ -243,7 +243,7 @@ SOLR_HOST=solr1.example.com Setting the hostname of the Solr server is recommended, especially when running in SolrCloud mode, as this determines the address of the node when it registers with ZooKeeper. [[TakingSolrtoProduction-Overridesettingsinsolrconfig.xml]] -=== Override settings in solrconfig.xml +=== Override Settings in solrconfig.xml Solr allows configuration properties to be overridden using Java system properties passed at startup using the `-Dproperty=value` syntax. For instance, in `solrconfig.xml`, the default auto soft commit settings are set to: @@ -269,14 +269,13 @@ SOLR_OPTS="$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=10000" ---- [[TakingSolrtoProduction-RunningmultipleSolrnodesperhost]] -== Running multiple Solr nodes per host +== Running Multiple Solr Nodes Per Host The `bin/solr` script is capable of running multiple instances on one machine, but for a *typical* installation, this is not a recommended setup. Extra CPU and memory resources are required for each additional instance. A single instance is easily capable of handling multiple indexes. .When to ignore the recommendation [NOTE] ==== - For every recommendation, there are exceptions. For the recommendation above, that exception is mostly applicable when discussing extreme scalability. The best reason for running multiple Solr nodes on one host is decreasing the need for extremely large heaps. When the Java heap gets very large, it can result in extremely long garbage collection pauses, even with the GC tuning that the startup script provides by default. The exact point at which the heap is considered "very large" will vary depending on how Solr is used. This means that there is no hard number that can be given as a threshold, but if your heap is reaching the neighborhood of 16 to 32 gigabytes, it might be time to consider splitting nodes. Ideally this would mean more machines, but budget constraints might make that impossible. @@ -284,7 +283,6 @@ When the Java heap gets very large, it can result in extremely long garbage coll There is another issue once the heap reaches 32GB. Below 32GB, Java is able to use compressed pointers, but above that point, larger pointers are required, which uses more memory and slows down the JVM. Because of the potential garbage collection issues and the particular issues that happen at 32GB, if a single instance would require a 64GB heap, performance is likely to improve greatly if the machine is set up with two nodes that each have a 31GB heap. - ==== If your use case requires multiple instances, at a minimum you will need unique Solr home directories for each node you want to run; ideally, each home should be on a different physical disk so that multiple Solr nodes don’t have to compete with each other when accessing files on disk. Having different Solr home directories implies that you’ll need a different include file for each node. Moreover, if using the `/etc/init.d/solr` script to control Solr as a service, then you’ll need a separate script for each node. The easiest approach is to use the service installation script to add multiple services on the same host, such as: diff --git a/solr/solr-ref-guide/src/understanding-analyzers-tokenizers-and-filters.adoc b/solr/solr-ref-guide/src/understanding-analyzers-tokenizers-and-filters.adoc index 7dfce492fb2..511a5e9935a 100644 --- a/solr/solr-ref-guide/src/understanding-analyzers-tokenizers-and-filters.adoc +++ b/solr/solr-ref-guide/src/understanding-analyzers-tokenizers-and-filters.adoc @@ -22,8 +22,8 @@ The following sections describe how Solr breaks down and works with textual data. There are three main concepts to understand: analyzers, tokenizers, and filters. * <> are used both during ingestion, when a document is indexed, and at query time. An analyzer examines the text of fields and generates a token stream. Analyzers may be a single class or they may be composed of a series of tokenizer and filter classes. -* <> break field data into lexical units, or __tokens__. -* <> examine a stream of tokens and keep them, transform or discard them, or create new ones. Tokenizers and filters may be combined to form pipelines, or __chains__, where the output of one is input to the next. Such a sequence of tokenizers and filters is called an _analyzer_ and the resulting output of an analyzer is used to match query results or build indices. +* <> break field data into lexical units, or _tokens_. +* <> examine a stream of tokens and keep them, transform or discard them, or create new ones. Tokenizers and filters may be combined to form pipelines, or _chains_, where the output of one is input to the next. Such a sequence of tokenizers and filters is called an _analyzer_ and the resulting output of an analyzer is used to match query results or build indices. [[UnderstandingAnalyzers_Tokenizers_andFilters-UsingAnalyzers_Tokenizers_andFilters]] @@ -31,11 +31,11 @@ The following sections describe how Solr breaks down and works with textual data Although the analysis process is used for both indexing and querying, the same analysis process need not be used for both operations. For indexing, you often want to simplify, or normalize, words. For example, setting all letters to lowercase, eliminating punctuation and accents, mapping words to their stems, and so on. Doing so can increase recall because, for example, "ram", "Ram" and "RAM" would all match a query for "ram". To increase query-time precision, a filter could be employed to narrow the matches by, for example, ignoring all-cap acronyms if you're interested in male sheep, but not Random Access Memory. -The tokens output by the analysis process define the values, or __terms__, of that field and are used either to build an index of those terms when a new document is added, or to identify which documents contain the terms you are querying for. +The tokens output by the analysis process define the values, or _terms_, of that field and are used either to build an index of those terms when a new document is added, or to identify which documents contain the terms you are querying for. [[UnderstandingAnalyzers_Tokenizers_andFilters-ForMoreInformation]] -== For More Information +=== For More Information These sections will show you how to configure field analyzers and also serves as a reference for the details of configuring each of the available tokenizer and filter classes. It also serves as a guide so that you can configure your own analysis classes if you have special needs that cannot be met with the included filters or tokenizers. diff --git a/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc b/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc index d8a00bc248a..7afc2e15b42 100644 --- a/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc @@ -129,7 +129,7 @@ Three additional expert-level configuration settings affect indexing performance // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="20,10,10,60",options="header"] +[cols="25,10,10,55",options="header"] |=== |Setting Name |Type |Default |Description |numRecordsToKeep |int |100 |The number of update records to keep per log diff --git a/solr/solr-ref-guide/src/working-with-external-files-and-processes.adoc b/solr/solr-ref-guide/src/working-with-external-files-and-processes.adoc index 3e92f502464..31abc186672 100644 --- a/solr/solr-ref-guide/src/working-with-external-files-and-processes.adoc +++ b/solr/solr-ref-guide/src/working-with-external-files-and-processes.adoc @@ -178,16 +178,16 @@ value ::= text Special characters in "text" values can be escaped using the escape character `\` . The following escape sequences are recognized: -[width="30%",options="header",] +[width="60%",options="header",] |=== |EscapeSequence |Description -|"`\` " |literal space character -|"`\,`" |literal `,` character -|"`\=`" |literal `=` character -|"`\\`" |literal `\` character -|"`\n`" |newline -|"`\r`" |carriage return -|"`\t`" |horizontal tab +|`\` |literal space character +|`\,` |literal `,` character +|`\=` |literal `=` character +|`\\` |literal `\` character +|`\n` |newline +|`\r` |carriage return +|`\t` |horizontal tab |=== Please note that Unicode sequences (e.g. `\u0001`) are not supported. @@ -249,7 +249,7 @@ Token positions are tracked and implicitly added to the token stream - the start * stored: null * token: (term=`one`,positionIncrement=22,startOffset=123,endOffset=128) * token: (term=`two`,positionIncrement=1,startOffset=5,endOffset=8) -* token: (term=three,positionIncrement=1,startOffset=20,endOffset=22) +* token: (term=`three`,positionIncrement=1,startOffset=20,endOffset=22) [source,text] ---- @@ -260,7 +260,7 @@ Token positions are tracked and implicitly added to the token stream - the start * version: 1 * stored: null -* token: (term=` one ,`,positionIncrement=22,startOffset=0,endOffset=6) +* token: (term=`one ,`,positionIncrement=22,startOffset=0,endOffset=6) * token: (term=`two=` ,positionIncrement=1,startOffset=7,endOffset=15) * token: (term=`\`,positionIncrement=1,startOffset=17,endOffset=18) @@ -284,12 +284,12 @@ Note that unknown attributes and their values are ignored, so in this example, t ---- * version: 1 -* stored: "`This is the stored part with = \t escapes.`" +* stored: `This is the stored part with = \t escapes.` * token: (term=`one`,startOffset=0,endOffset=3) * token: (term=`two`,startOffset=4,endOffset=7) * token: (term=`three`,startOffset=8,endOffset=13) -Note that the "`\t`" in the above stored value is not literal; it's shown that way to visually indicate the actual tab char that is in the stored value. +Note that the `\t` in the above stored value is not literal; it's shown that way to visually indicate the actual tab char that is in the stored value. [source,text] ---- @@ -306,5 +306,5 @@ Note that the "`\t`" in the above stored value is not literal; it's shown that w ---- * version: 1 -* stored: "this is a test." +* stored: `this is a test.` * (no tokens) diff --git a/solr/solr-ref-guide/src/zookeeper-access-control.adoc b/solr/solr-ref-guide/src/zookeeper-access-control.adoc index 0bf3b1babbd..342021fe03e 100644 --- a/solr/solr-ref-guide/src/zookeeper-access-control.adoc +++ b/solr/solr-ref-guide/src/zookeeper-access-control.adoc @@ -60,13 +60,13 @@ Solr nodes, clients and tools (e.g. ZkCLI) always use a java class called {solr- You control which credentials provider will be used by configuring the `zkCredentialsProvider` property in `solr.xml`'s `` section to the name of a class (on the classpath) implementing the {solr-javadocs}/solr-solrj/org/apache/solr/common/cloud/ZkCredentialsProvider[`ZkCredentialsProvider`] interface. `server/solr/solr.xml` in the Solr distribution defines the `zkCredentialsProvider` such that it will take on the value of the same-named `zkCredentialsProvider` system property if it is defined (e.g. by uncommenting the `SOLR_ZK_CREDS_AND_ACLS` environment variable definition in `solr.in.sh/.cmd` - see below), or if not, default to the `DefaultZkCredentialsProvider` implementation. -*Out of the Box Implementations* +==== Out of the Box Credential Implementations You can always make you own implementation, but Solr comes with two implementations: * `org.apache.solr.common.cloud.DefaultZkCredentialsProvider`: Its `getCredentials()` returns a list of length zero, or "no credentials used". This is the default. * `org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider`: This lets you define your credentials using system properties. It supports at most one set of credentials. -** The schema is "digest". The username and password are defined by system properties "```zkDigestUsername```" and "```zkDigestPassword```", respectively. This set of credentials will be added to the list of credentials returned by `getCredentials()` if both username and password are provided. +** The schema is "digest". The username and password are defined by system properties `zkDigestUsername` and `zkDigestPassword`. This set of credentials will be added to the list of credentials returned by `getCredentials()` if both username and password are provided. ** If the one set of credentials above is not added to the list, this implementation will fall back to default behavior and use the (empty) credentials list from `DefaultZkCredentialsProvider`. [[ZooKeeperAccessControl-ControllingACLs]] @@ -75,19 +75,19 @@ You can always make you own implementation, but Solr comes with two implementati You control which ACLs will be added by configuring `zkACLProvider` property in `solr.xml`'s `` section to the name of a class (on the classpath) implementing the {solr-javadocs}//solr-solrj/org/apache/solr/common/cloud/ZkACLProvider[`ZkACLProvider`] interface. `server/solr/solr.xml` in the Solr distribution defines the `zkACLProvider` such that it will take on the value of the same-named `zkACLProvider` system property if it is defined (e.g. by uncommenting the `SOLR_ZK_CREDS_AND_ACLS` environment variable definition in `solr.in.sh/.cmd` - see below), or if not, default to the `DefaultZkACLProvider` implementation. [[ZooKeeperAccessControl-OutoftheBoxImplementations]] -==== Out of the Box Implementations +==== Out of the Box ACL Implementations You can always make you own implementation, but Solr comes with: * `org.apache.solr.common.cloud.DefaultZkACLProvider`: It returns a list of length one for all `zNodePath`-s. The single ACL entry in the list is "open-unsafe". This is the default. * `org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider`: This lets you define your ACLs using system properties. Its `getACLsToAdd()` implementation does not use `zNodePath` for anything, so all znodes will get the same set of ACLs. It supports adding one or both of these options: ** A user that is allowed to do everything. -*** The permission is "```ALL```" (corresponding to all of `CREATE`, `READ`, `WRITE`, `DELETE`, and `ADMIN`), and the schema is "digest". -*** The username and password are defined by system properties "```zkDigestUsername```" and "```zkDigestPassword```", respectively. +*** The permission is `ALL` (corresponding to all of `CREATE`, `READ`, `WRITE`, `DELETE`, and `ADMIN`), and the schema is "digest". +*** The username and password are defined by system properties `zkDigestUsername` and `zkDigestPassword`, respectively. *** This ACL will not be added to the list of ACLs unless both username and password are provided. ** A user that is only allowed to perform read operations. -*** The permission is "```READ```" and the schema is "digest". -*** The username and password are defined by system properties "```zkDigestReadonlyUsername```" and "```zkDigestReadonlyPassword```", respectively. +*** The permission is `READ` and the schema is `digest`. +*** The username and password are defined by system properties `zkDigestReadonlyUsername` and `zkDigestReadonlyPassword`, respectively. *** This ACL will not be added to the list of ACLs unless both username and password are provided. * `org.apache.solr.common.cloud.SaslZkACLProvider`: Requires SASL authentication. Gives all permissions for the user specified in system property `solr.authorization.superuser` (default: `solr`) when using SASL, and gives read permissions for anyone else. Designed for a setup where configurations have already been set up and will not be modified, or where configuration changes are controlled via Solr APIs. This provider will be useful for administration in a kerberos environment. In such an environment, the administrator wants Solr to authenticate to ZooKeeper using SASL, since this is only way to authenticate with ZooKeeper via Kerberos. @@ -102,6 +102,7 @@ You can give the readonly credentials to "clients" of your SolrCloud cluster - e === ZooKeeper ACLs in Solr Scripts There are two scripts that impact ZooKeeper ACLs: + * For *nix systems: `bin/solr` & `server/scripts/cloud-scripts/zkcli.sh` * For Windows systems: `bin/solr.cmd` & `server/scripts/cloud-scripts/zkcli.bat` From 40ed09f8caa40212f829d6916151f954033db953 Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Thu, 8 Jun 2017 13:13:05 -0400 Subject: [PATCH 029/131] SOLR-10853:Allow the analyze Stream Evaluator to operate outside of a stream --- .../org/apache/solr/handler/AnalyzeEvaluator.java | 11 ++++++++--- solr/solr-ref-guide/src/stream-evaluators.adoc | 14 ++++++++++++++ .../solr/client/solrj/io/stream/TupStream.java | 3 +-- .../solrj/io/stream/StreamExpressionTest.java | 15 +++++++++++++++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/AnalyzeEvaluator.java b/solr/core/src/java/org/apache/solr/handler/AnalyzeEvaluator.java index c92f093889f..7c7349893f8 100644 --- a/solr/core/src/java/org/apache/solr/handler/AnalyzeEvaluator.java +++ b/solr/core/src/java/org/apache/solr/handler/AnalyzeEvaluator.java @@ -54,6 +54,7 @@ public class AnalyzeEvaluator extends SimpleEvaluator { } public void setStreamContext(StreamContext context) { + this.streamContext = context; Object solrCoreObj = context.get("solr-core"); if (solrCoreObj == null || !(solrCoreObj instanceof SolrCore) ) { throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "StreamContext must have SolrCore in solr-core key"); @@ -74,9 +75,13 @@ public class AnalyzeEvaluator extends SimpleEvaluator { @Override public Object evaluate(Tuple tuple) throws IOException { - String value = tuple.getString(fieldName); - if(value == null) { - return null; + String value = null; + Object obj = tuple.get(fieldName); + + if(obj == null) { + value = fieldName; + } else { + value = obj.toString(); } List tokens = new ArrayList(); diff --git a/solr/solr-ref-guide/src/stream-evaluators.adoc b/solr/solr-ref-guide/src/stream-evaluators.adoc index 2c2a2ae2664..06c15df4def 100644 --- a/solr/solr-ref-guide/src/stream-evaluators.adoc +++ b/solr/solr-ref-guide/src/stream-evaluators.adoc @@ -20,6 +20,20 @@ // specific language governing permissions and limitations // under the License. + +Stream evaluators are different the stream sources or stream decorators. Both +streaming sources and stream decorators return streams of tuples. + +Stream evaluators are more like traditional functions that evaluates its parameters and +returns an result. That result can be a single value, array, map or other structure. + +Stream evaluators can be nested to so that the output of evaluator becomes the input +for another evaluatore. + +Stream evaluators can be called in different contexts. For example a stream evaluator +can be called on its own or it can be called within the context of a streaming expression. + + == analyze // TODO diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupStream.java index f099e31034a..855f05fe59b 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupStream.java @@ -170,8 +170,7 @@ public class TupStream extends TupleStream implements Expressible { for(Entry param : streamParams.entrySet()){ try{ - List streamTuples = new ArrayList(); - + List streamTuples = new ArrayList(); // open the stream, closed in finally block param.getValue().open(); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java index 3466af29270..75f3637946f 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java @@ -5042,6 +5042,21 @@ public class StreamExpressionTest extends SolrCloudTestCase { assertTrue(t.getString("test_t").equals("c")); assertTrue(t.getString("id").equals("1")); + + expr = "analyze(\"hello world\", test_t)"; + paramsLoc = new ModifiableSolrParams(); + paramsLoc.set("expr", expr); + paramsLoc.set("qt", "/stream"); + + solrStream = new SolrStream(url, paramsLoc); + context = new StreamContext(); + solrStream.setStreamContext(context); + tuples = getTuples(solrStream); + assertEquals(tuples.size(), 1); + List terms = (List)tuples.get(0).get("return-value"); + assertTrue(terms.get(0).equals("hello")); + assertTrue(terms.get(1).equals("world")); + //Try with single param expr = "cartesianProduct(search("+COLLECTIONORALIAS+", q=\"*:*\", fl=\"id, test_t\", sort=\"id desc\"), analyze(test_t) as test_t)"; paramsLoc = new ModifiableSolrParams(); From f631c986772a148c15ea34ad9a30b256a256afaa Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Thu, 8 Jun 2017 15:18:59 -0400 Subject: [PATCH 030/131] SOLR-10351: Add documention --- .../solr-ref-guide/src/stream-evaluators.adoc | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/stream-evaluators.adoc b/solr/solr-ref-guide/src/stream-evaluators.adoc index 06c15df4def..83fa8a85fad 100644 --- a/solr/solr-ref-guide/src/stream-evaluators.adoc +++ b/solr/solr-ref-guide/src/stream-evaluators.adoc @@ -36,7 +36,26 @@ can be called on its own or it can be called within the context of a streaming e == analyze -// TODO +The `analyze` function analyzes text using a Lucene/Solr analyzer and returns a list of tokens +emitted by the analyzer. The `analyze` function can be called on its own or within the +`select` and `cartasianProduct` Streaming Expressions. + +=== analyze Parameters + +* `Field Name` | `Raw Text`: Either the field in a tuple or the raw text to be analyzed. +* `Analyzer field name`: The field name of the analyzer to use to analyze the text. + +=== analyze Syntax + +The expressions below show the various ways in which you can use the `analyze` evaluator. + +[source,text] +---- +analyze("hello world", analyzerField) // Analyze the raw text +select(expr, analyze(textField, analyzerField) as outField) // Analyze a text field within a select expression. This will annotate the tuples with output of the analyzer +cartesianProduct(expr, analyze(textField, analyzer) as outField) // Analyze a text field with a cartesianProduct expression. This will stream each token emitted by the analyzer in its own tuple. +---- + == abs From 0a52191577fe39e824b5f7a52b39685a42ec73f6 Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Thu, 8 Jun 2017 15:28:42 -0400 Subject: [PATCH 031/131] Ref Guide: Edit stream evaluators description --- solr/solr-ref-guide/src/stream-evaluators.adoc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/solr/solr-ref-guide/src/stream-evaluators.adoc b/solr/solr-ref-guide/src/stream-evaluators.adoc index 83fa8a85fad..8571b0e1751 100644 --- a/solr/solr-ref-guide/src/stream-evaluators.adoc +++ b/solr/solr-ref-guide/src/stream-evaluators.adoc @@ -21,14 +21,13 @@ // under the License. -Stream evaluators are different the stream sources or stream decorators. Both -streaming sources and stream decorators return streams of tuples. - -Stream evaluators are more like traditional functions that evaluates its parameters and +Stream evaluators are different then stream sources or stream decorators. Both +stream sources and stream decorators return streams of tuples. Stream evaluators are more like +a traditional function that evaluates its parameters and returns an result. That result can be a single value, array, map or other structure. -Stream evaluators can be nested to so that the output of evaluator becomes the input -for another evaluatore. +Stream evaluators can be nested so that the output of an evaluator becomes the input +for another evaluator. Stream evaluators can be called in different contexts. For example a stream evaluator can be called on its own or it can be called within the context of a streaming expression. @@ -43,7 +42,7 @@ emitted by the analyzer. The `analyze` function can be called on its own or with === analyze Parameters * `Field Name` | `Raw Text`: Either the field in a tuple or the raw text to be analyzed. -* `Analyzer field name`: The field name of the analyzer to use to analyze the text. +* `Analyzer Field Name`: The field name of the analyzer to use to analyze the text. === analyze Syntax From 7ee9d337308e53bd71341683f6e5ccd42726e2a2 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Thu, 8 Jun 2017 15:29:09 -0500 Subject: [PATCH 032/131] SOLR-10854: Add full Solr logo and use it for PDF title page; add "author" and pubdate metadata to title page --- .../src/images/Solr_Logo_on_white.png | Bin 0 -> 47661 bytes .../src/pdf/SolrRefGuide-all.adoc | 4 ++++ .../src/pdf/themes/refguide-theme.yml | 7 +++---- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 solr/solr-ref-guide/src/images/Solr_Logo_on_white.png diff --git a/solr/solr-ref-guide/src/images/Solr_Logo_on_white.png b/solr/solr-ref-guide/src/images/Solr_Logo_on_white.png new file mode 100644 index 0000000000000000000000000000000000000000..83a07ed527f41698b11dcfbdea7dc21467fbc323 GIT binary patch literal 47661 zcmYIw2Q=0H|G!f9`j};=K~yMPMyO<5;=1p36K$Mg9X^H^7dk&c^=f`Wok^U-|+3JR(;3JOYN z8YJTmpBps;vI`g?{wNnI8A;YFVZCO%+yJ0E|jmo0_rGj|)?OPa1w zds_or=(9l2@3x8*6t#7l_f-u8hL^`I-9{cA$++}9>6!6AKOps_qG#Ibh4k~D8T<44 zMl8q1EOzHt3NQVo`z9~j_m|ElvsN&xeO$HvWtgcxdPMU;aO5EWatZyVBwgXlrZ)|A z$B2-_nO>FSo4`RRC>F-O6f#ElddHkb>NX}4*Ny`A!o_(M)0Uj>?yv5O%~Ww79j9y` zkI2{7_;5f!9tnD!B|YXa;H6*{t+6z?T@trP{A7;`u_}C-Us|MI@E}A9SiJYi_+AqY zAPwM8hNC=R31zPSs2i1zctu~)Kz4tfmQ$IxS6Sc9x2SN(lMSDD-$c#mm(e8&wM|U@ z%J>lYe%t5y(hh=MMVE)>ncCzz(oP<6H5tF$HvDJbKS7&P!zun=BcFz9?O{zt*w}z* zk;1Titz5yBLOdp=NMW2lfcFI(=|PF>x061yhevCRS8Ab(3yKN}4RiP^W6PQKC7eyA za{uM})ietWCntvx6kDV_8jBTGbD3QeS@YKw>*Ox>uBmDo+tetIy`T&; zhCF^OPTs%o##v-xB#VRp{GVxU#=*mypup`{O{}5?oJ$~z^SxKd$NbER>KG9}rz0O! z?(KEE1kHS~mg-(!{G4vYlTYvhLrgO*>9rj_z-y^>rhLugo)JRk|JYO`k0eU0tOi3v z0MX#EvoH5)PXzn?PpEO*mR8&rHhhuB%L(Ha^yp)lxXy2`c;juh(NmFK6)A435U5Tu z%Qe;ot{^16o%lH@aBydXtAIi@QZGeJn5?Dek*#kXjXsK&tQ}cuJO*#46fIt>*GpOQ zgQ0ilVCyk2*yPTTPM6Mlhp{I*JuxlJfW^7r8mscHc8LAT zZK472CDA`xoP=c7Gl5$^03Es-G#m`z{OZJvAbQc5jE*;CH3fX0^-%=W;!L}` z;RtA{>6oFQLZSn6{4vVGWHlPe+m*hpsDi^-!P)UoRU;6LOBBt}_ zM5Cu+p=bst{H0+&`H~q|5jXO@;Nrv_J}J0 zuBZPd)U_a}wy=r|zO%C)=kFqux+4eaa-X3(vYUdo}#a7ZUh<9QwL zN}5F){vadIF7(be1i8rbncK%y@sgZ$ z;2?t(^o24qw)mo80MQ-)jehrz z@4plw(zTu+y^3guP9EFttx*l&UX1V>hOTIjt)WeMxXw0AVCKI$ZYwxvsl7QvKG;2G zQe4`;Bi>j= zpXS1p>QH*PdM^BqSm7D_P+|U-q#)AS-=@mZD3(;MP#-xYzQBFBp=$f#d+^_Ysed-1 z!ZJGvD#r)l8>DsjsBC82p=DFUGJx+KJFJ^vxy8jTl2z|wh(m>QVH_`iXoud=Tx;2E zC+d#2o$mdj2C0>A5>z>5E$_Pb_T1SPKTD}{KirGPNuPWbKE&HSE5vglu#po&j^uC>1~!orAcLXf%TD* zqOuDe+@i_x4;|w$zUTXc-}f>zIhaedA9AbZ&6*b0N0yLP`u3xoE#9`WA?(`-O`3yN z>hJwuuO+spzAOfEyhK@czYy6bo#WDfo|wA8$`z&E20vY~bGlMf6^rJZRKzeEQ;6)W zh~bm4;^L|3#Yz_k%!4~w*b^UPElrDOj)g#eSuwi%53f(T)!IZrWVRZq;WEMQad@d< zHqdPyvb$xe`x;4)c?_&W5z5XVl>x)O$z4M&B)uD6$A;+-1FuN^Z{ZbbW9hMepUGUT z<3~6~4emHRuD>F6?{phmMu_y7&+m1SdLItz`4&LOw(7fB=k5`0Oghy=*hz}2aFFL) z~-U9T0~COiowmlp!@3^xG?FQ`0^IRG$r8^KYz{{nS~sV2LxI@Y$a7w zuovmcTp^Yf@SUmZ0Fu7K0eAK8Zf2}r&Ts=gMQ z6-YgAxvgl?ie@KVBwMA_^{|}dYMaUoO&&N?no#j#7O#|#wF}2tR5l$hCZmn;fbi}# zXXjfVP=$X~TeI!@Vl;K%l9XTKd4kqVe=hgG&%I@_e{9tIJ!f${D@KIJF_Z2j3*?&~ z%!Sc}4KpEPNzt|C%f}jQgucw)I0iw3<+g&xDmk}oKPS7r6DyLopek^?>zH%Q&^_%7 zUH_ka^!n#I4Z8=yV4e4kEI-6;nl3#hCyad6gQ%lVCZSK8Zn)~8P^{U<#c_582rbYL zf3&5WpRTTm9-PX061$q&TDRexc<~9Omky|?);j`S zOkB%a)Uf8Rvlqg5WeC|(E{W_gNk32~oA&ynvS0I`A33id)WbqwTO3+_0YJ+~*}Wr4 z(2pw(MH$uh1HPRRTYh;K61a6BQxO6O*qpa12{u z%2cGq&*Qt=?=$O7wtA<@C|-MtjtPunSr4x&1`nUU`8m&$be=`FpvSbl9cWwywkWg+ zwM0dd@x&y*CpU^^ZaFU`jnNoY_-5;sl>&y?Ya6i^o)v@EHyI=h)#(z*VX>L-18yU1 z`*VKPN-OhrecqwV5=Vo#rPKwQw#WA-?~hr*?7b~`Uf(B!i%Cw;qPNGK1(oUs{sU^k z`9ce*7C`;tjE%=P6N6tJ)HC!pn!vQqo}M!OB(o~p`ED9bSwnB5ZpY%Kpq{ zu?vg0X308L`YSaKJrkZ>;_T87riM4onlSKj639rljip)Yb;Z6I_eHa!@Jg8Pgn zuijgTW(Goc3gg;SWUA&pn)DtUjvKd5-z<0Ag*X)L?Fe;}BF@w~gsmX;xBr@f6JM>v zUc&m&d}N29vIVjWp;dY3k57-elVn8Yp+}9k*GAq{Xc9cNl_uTn@`*D%kpL7~AM~$M z1Jys^6J#Eww^IyAs55Gk<_VdF141T-6N)4&+Cd({`K!m3~lxf zVONmWIJxn?wQVUEIzzd#7#yJ8nW(k0K8tNh2M+ZZJ|0e_Ktdt;rne+Cy zR-JP-YzthbtPl|K5jZ@uflEeu%^FTh49^=c-Wy`CxF|=<_LrUNx48L`9vZDVgjiJm z=o~eNKDm09F%>Imo67k|C7r)~EBUJWRD@vn-uuuZ%yLh1#q;qCkAYnvQ&PXmKl_xD z#0NV}g2T_jnthGNDBIzml*qykf2&Cr3jKK7!m{tZJDUosK^P8}*&5N^x zFESe4*S!%rGMKxVU~$6+wA*ElrBZ_Sn^47&#*o4)Sf8Ue`4XSKpGtVVDqiF@8Eekq zPl1*Cfz0tBYB=J48s~O2=!>kfwH3P>>domvI)TEGy@e5R77rSoD}8h6j*%>F+9TEc z1?S5yW+A&HOMTyi{Fuo$`}U38g#g8MJ2~gF2H_w}&SK0;XA&TkjiVxmfM-)LzBUV*E-;3mTs+|q@F1TRbxJ61fGSg5oIV@ZKwfoKoN2wAo+Qqs zAhz!G`c1&<0=0Li3WI(R{P9@;;U}1#RtFK0AZI}z1)qy<9OW*)>P-*!>AuspGHvL+k}f8Jo6I z32G>JvIHL)@OA_jTx%?I2n-iZ`?U=vC84}vPg02RGOJ=j^l&fWHa}&EOY$16&B|Xq zeM*;i@+kjWSf3vZM-R%#m0+>1s%c6ULPaQN1PxV$JrtWZdmM3kRgsO6>2|YeYElhl zZC1~81(Z?PQbkl-rbbrgg68ouPS;)F(tVjQ-P0RCYd!NNF#GeIjH|Ki8>zteYU?Jz z?gojtPkc$Bh?ks=o)~Z72%sVygad%M>*f z?|_;9ouxY|TpnS)6qqKMCUC6&W)<~K?}YSvNETk-cF5=8W#(nLy)qVsMQZ*Whd*D# z9Ih~LUnW1dU~#bX&-|8WN!5+CDycxPPR_OO9*5_3D+B?ni6VM(IiKyB38OxZKd9;E z(d-X3gAlffgYf2JJ3XxnN!ArLO{UOW(!+$oaw-*bPQI;~(x#aU!%VSc9rv`Jsa;?3 zrX1P5yYwUz(1Sr$y$T*AwE~KO;_?@ST+Bv_tN!(g<&&3-0m?L&RLqZSCgj=aecK02 zqN~b}6IJ75{VP`%RHX5tkL#ipq}_mt@agml_jzB9;8=M~D?(fiW%GcD9p3rfpd6Ql zY(cKq>s*rvH|E0~Jr*k@zhI#4OpU*p%~MxnInyQ)K3l|+KV)|<_wm?$NwolC<{22s zc~h>8xRl*zc$_l%30x2D)EA+ip%>j&XYXX2bY`N zG0;}*OclD2i;F5yb(6#`_b=BE`%&`)I$qfi(j{BI*?-i-<4!Re=02spF?MUCs$IMD z`5jODGD0FtMGR#uWrp0leh^(t2EECssZVZ6Mk$UCH6`!#`SKs8JUi6GrBZa+_B^IH z*>wA_Sh*iz`+s{6ONbYaY_S`In1;rm8sq$h3jM)VwJozg^^knGPs$n(?m}_xIwPp+ zCHYga7^eMo(9>4bG{%FlUsri$3+ECmljl=B+6UtY}a+kVh=;Y!YWlyltYItY2*7}cR`w@p2 zx9Qq(!_aCMFKQ(Q#DEoKegF~tn4An&rnz}aDqG7f{2#BdzLp9VW>TopvxhzYI3I}f zc^s5uR#lQ5VCx&-5U@(NGl!SH&okBva}MLBLYGKboRe?4x_?XYr*Gw9(+4cl!@Yjl z(|pIyNr?>fL2|fry;TbiO;+8$8TMEM^bf}#q$`SEEgtaBBki&}ZQbLZOwZOmQKN_7wybEtnT%cauwfH0 zV?d}uX0wHw@NrIvd7tJ}aa8o3OAyLksxDT`G%e$9j@qyPEnn-9m+-CBG243lhE7p+ zX4ltK&A(s&v%$%?8mzY|yBHdccg7$BIuKO@oNO{mor*@+g-#=N;yK;n*U6^M7`x6lePCI2G0v||f7fC&wmq)d$#&$;WQED0yWGtDcwWybbyvdAcnS*7tynx0AliFNdjW7=)V z`^R%aG>x{!KlR0BXgB-!6u9==4H<5?1q!-gTA+pO+V9CaLDB-ge}hA}d*O4PBl6!r zT^fXtYq2nA3vv+N$O&C3xPy;4HR${rp`Vo>h90H7NTfp%`6?A{e_I3B&6nutt&Rj8 zu2(q?UTwNW<`1(NF=_mMd@Z%buPt5%mDvAuy{1NB^-!l#@i%8e*uhHCBGDrJN#rxg z+?4tnQO7h}W%<&K&yVBMH)$(|t#VE*(zGj7VtFkG>HfnC|7{=5_rLY|9zjE&3i#Ry zgZX;-Zi}DKPBmY<#459G>;*ZEufvx-w7$G~2yZf?cGb1?0~fln%KcYQxa0x{bXVnK zyprDEAVxM~Pfw>ZXCP;aQKD{jTO&3xO}%$Vj|+#bY8WDRkt<;Uk>_8K@)R_IxfAt6 z^VC8)A-euwR-GC$9R{!EcYkf()!VJ3LvX@M%U#6{>gG@L2jF+B{k|7aSTsKhz!d4* zn>BttRb3J$*|A$He376WwW0nze{^9B8GvNn5=~pJS_n86O69JIyBq)yC}~z;Ym{ z6j7(+C7)U~)ceuC?f_8+V+@V7@?Pnvu+RYlbxUa`hdC^GFXfaL{Xl;a(Y7sLZ zG;LU)FO94xZN{<>KoyT~+)ff}$>N2Cx(&3&r{FvnoUzEQWJ{$%$Z5{4;;QP6l=*J_ zd$tE4fQY!d6nLFx&?+!!y_y6-;+dg?kbw7k_)2r15C&?omM8UV zaxTOL)@OOkHYl;|Gjda8RWlu!2 zNKw_}sjUD3)_fg$)n@vXPrKtjmAXR!i0h52)qY#_d4=yZ_MEI`F;ndE{kbs=(kB>f zHD(?*#kU|~^Xda;CI34sCOwh<1#UvYH4Z55y=eK13W@6}@g*YwR`(MLx)BLV#ANdM z_cuj?l!B}(YbD8Awv@x4(~NY#Vmzf^jB)n+*z9TY1>H0N1q=5^)yNFhv&d|>Ss|=% zp4#wvh^}L#+m)0leFf&AX?9a_%UCM~@j7M%C((E<$#vyaWdMKeSn;*Pc4)}a7@QW0 z@(~Hr9J&v-s)o#x8KNxuv%)XHZk@%>v;Vts8&JTjry6Kt0Jh&U%_wwBo zbpDvFeBo`_5PVT8vV2^mef#>zMx88~s+VB#>%j@N2fi42bP9k)Uin zdhtNZ?dXSLRAftFXj%Sp%zw?xLY*PaKz=@E#AaqckIj={e7W6b1sRgZDg%>l2BKh$dpA4)d}5XNj=SCVyOGHL zz}NQ&?l_r?&etyu?5WbwzNZPq?0XwD)ca_D_OWuk0%fBeiRVbFB)jX@geB!d&i#1z z$Ijf>*gs#4Jo*Xs4Q;&Ybu+K)T^>+g)vapCVuiA=`KRI!qsZ#ykDGYXc*A5!>yJdV z;j9n*QPbnGt(*dZ9OJG9L@soR6r?ix zLidNSeb=Hv*u`vR)*iK7=^u6{Yw}P!c=A~IKZUCmNPQkdxjeMbO~rSmF=?*GhjvZH zvgG8tHFz`UsS+RdyykTao?4}&zvw!#ICiFI04sILW8&KPZ-__$lk#GP6b?;QPaLXG zofP&PTvXQEcUTqv2pY=gv18thUJI9hv}|HkbJa8;u}|tXK$GQYW7!&Fp}XZ|uupXS z@Oy(v&_j#BtKs&pM0pwL5+OYDnbnUMP7hLKv{y~+M!SAQ)} z%iKyUgBp8?Y~F-@nC5jCx?XoEo9{>bh7ei&N8jBh6|yVez>`|#2c&%sn0bv3SGCpJ z!F5+P>7lr*$@UOmfi)xXb}@SCK+|pXPwxpbPu8chpRiZ{g_dvEF;iSiEql1wJhui@ ztdZLJwbk*bVS3(;wo*s^{8u?vy&1p2JuC?KOFM@MeRkA%|m$FP&YFRO0aGf-|fL+lhchNpxpy zrkq;HM$$Hy0Rm2_Pv8J4DT2Q5oojj+)OSNQMefis+ei>QV9ENMn*T!k3K5R zinn#<8*bhHsCnx?kJE$heZzt2TF;4woguBx!BUYtN(}M4g-W(9`2vE$(>_7YIgz?~ z2`>^Ck*k(m1)}eglEkzPj`W7kS(7b;%I)}KxPMY#AWa%KK#vPkHmEQez%Sn_{$6hV zHdYpL{7Cd|A@PwZ4PxP2TutL>&Z$8`cV0K$)tbFZqKcL3Wkl2Jd?DKffi~^z18zZ6 zzphS!oDQD?hCKi8Zc5Z>7?K=xo~@?h2b`lW496-bZU2ZvSA#Z+u6)sUB(Xp;ds+fdGRKfBx5&bMk-D`K z$(nX{Mc#lu?IC5YxEhCcEFJ{pw)j$7{&`!AD_&#<{(1e%;D!+3w(o}m(|4z)uU^Pt z%&VI#b=nsf;d@-)ZpG=dt(basNLa0H3P{%PiZu>h$hfRyBo{T0k9ORCPuI=A588fh z2^oRDY=;C{_)df0xeZlZk9?cEqHpa)?l(LW)on90ZtJ{ShFemfe5TOgmP8vxKV>zx z+3#d&((Y_MbWU$4U-~lH{QkkW_D0X?ZjW#`D|wQ-wR-C{g=77VEHd4!N>zOtFwirg zv*T`Yn(jLW!``e(YZ6*nE%p~gfGhh^d+1%|T5d%%SO;5FE~;aoV7$Ow+m(WcSxT!)7OrvViWtel-nuts-EwD5H@Lk8Yf&4zpJDY)tDE{%$5QBjhgMeu;^nDz zMfig=Z=_+_ol5+jUnbK zG5qGHeG8T1znBneZ4235_w`)dKgTDMU!{^5HSV7@SvzMU;!_+~h1A)|va#)|>cTB3 zRUrZZl^fRl6ld0*y>XJrI(l=ncloXb6w6cT+EXq{S}K5i$1_(}`mL^!_moank3>gr1XeWME#Xv5a7R4kv zuzbds?`HvrpZ0MI<_A{-ncyZq;>&+BXqTKdq1{70vS(Wfff@sO|Vd?q)xT!5m_a8RBQtVD``VpPlj?=2T!Ev+-uzMTGXTlsfB z^`Oe^J$n+yvA_y-M3Z~`cLoQBwSr?ZnwklOhpD@VIjs?|e$5a})odr-sG#5gDtbiL zb*onKuLt)|66xFMF~Q6b3=SPCj6btk^jbVLKSTtqRAaL7d1YhknV<_@GAKnv;2DtT z#JEO}CCOt~mzWc){Mf0D!9K}<&uhs6&0WtX6&=ObrG7YEQ43OBMBvYx((e^9>#}6+ zej8?ZVNmo+GqAJhYD+2Lao)aQ2V9FGbFtHi<~#q7gLyfZ6G|NoX%Lga*?+eNf_JTl zNn?SRTs`~ByU+9*q7vv{;0o@GgE3H(9`NlCG1RPiC|qKsbgkV*MSpdZU<(?=lNC|r z-T1X28Tu0v{nwzM!&@iFk59wn*MeN^>l}-ENGuha-y+S9=eLoKDK|i;0e03vPm6Z;(z20G#ezgBiLoWTI_T_$54JgVe}4Sa=U=}p zIlAYTthkT33y*RNdITj;-qufG?!CC|VwUFSI$=Bf=t6S_WN=e1I)X#y_T;{l;WppE zMq5m{lOCzh=L4ihO^toNYr|LL&LxP)X9C>^zCW_d3u28WW$M6}wr(EhH$e|XIrA$Vtd$*&| zTZXb@YxN!CKqdb9Hie)AQP0<17j6LtolJb(8}_?9>W;xdVPBUG2U6>_YDDdC zmr1c&7-(wz;}@5$e&6MZXxH-~XL87{XaM+0k!X%Mr{I*1vRq!dS%WD_CRC$ zzK3=fmB2tG2F3$Qb+Qc%$u3x|AH6@WU8#?itp@0g3s1D6vE`D7^QK<;yrN36Zpr3D z?4n0=K;d#E#~7-RC!$Ht+k|2KIe$7;y}=ha$TaTN8`h?%wGk`pKV>!#eq@ZSPzzw3 zQ@W)hZo8PVi{cleS)`IhjG~Wlm88kUM@A^;^;UGNf$!V}{)+VVvZ6t5&Gbq{*w(|L zbLn!Js&M1oiK}?@qf@nO!*FAq_?_aJr;lVjVk4GCZm^XwnU`em3>WRZkre?=RAzhB zV}-nY`S#bLu^icWZ!b=(!QS&^_phsCVial<9n#dNgZ5sYYE1Mpxrn^{BfAd0MMKoS zu%+B_%LOR3rUI;ML!Jfo!sh=#W2Zt2pELD#NX#zmZ?B#DrtQkdyOC#VW_>~+K6hO$ zI>C#17E?(FJ}zZyE)(fHkPCZE-~eDAqVKkgLH0v8?7Ug3Qw`4zm9DkbqVGhm6Wvm* zz9%MyK`H7BnhByCB2xwpekjNnKKs61_3Jy>E0;!q{!0`MqT{<`Wj7lWylUEeV;t>5 zmN+eNNRe|?=n_hHs;}0*HSYZ4DT_$Tltt&fwy-EDt}rNY>F>K37AaUP@&R_&Fo6F< z+nC^iHGYU}R#JH^JGoX<71#Oqs(d311R^u92Iy#{Msh(#8?udM)42*d0aNDlVL&Jo z_byRtw23;@>FkBkWnS5w{x7-pbdxu%a#8)%V_z7=IqMd3BGJD)b>b^|_bORsRK0t@ z23GP3@k_zP5s{7}yZoE}ST4FF0yk;Y?y6%FR<-p!Hj$mWwM1b!bTL^WIEGK}7iFO= zaldfE&9ao8)QP0-5PTr$5G0^HSP;B)32zVGt1ltV74_=vG5j-2E;{M;Ujm^EGo>(9 zU6CUq=Tuy&&l%S;cKF|9sG4J?VhCJ8O3Y5(td58WIy(}KmkG4iuV`>{-E2+K14{#Z ziZjaO8vp9eW(-VJ&ocbI=-7}=j)d~C##ynDDsN{ zbBx%2Wk5@7FF)$A!E;-De2E+VdbA)FFMZW?aR5w@IepboH(oS3@_WFF+j3^Q zyM_qPEU||k_mffs#V>a1s^U83csUNePR?7aH=#$^_L3!{LY3lrKm1*VSz{c%Jkzr2SYJCLdLL39} zhhO`CuN?+``Hka^kxMQw+i6w7sXrBbmI$gXKH(*G3Q^$)|K>C6n1BuLVW(x2cNG*P zk8_n5j4JhMEo&vX!i6v= z{ivCIlk=dIi(^-&OgXWe{(J(R;-X|YAbX>Z@jsyl#AKyxuN=bTRfUMobl`^Kl656m z!yt8l8@mWe3soPwudQ}|vfcz@Z)N9~z}q_SM%Vk9+@-rWjin%wx)XIWdHrLY4>n(b znQ-rXYsUv1#hlab+xD)}LfJ;e&w-$HbuFW-iHk==3GI#PzdTRt6PCudQOJXDS0i>D zuP~TB>tp)DxBBJBtX(Yho8M1s!7SwtITyM^6M$STj>T65ZtASPVUb$N3kV@QK11Y- z8Nk}8QpX+Nn5S<)6X?Z5U^rC&;6GxIYQh>UXVL73m(reGR^#=oFd+5zN;zjwkZxbS*6lT+XIpGIcO1DN3Oax30M??hl() z+&BlawaeZkW(>vl4o2{<#)U6BlT0P>74B&=-?Bo=*9jrY>?Iw}#I;XYg1V4vxe@2# zRyZ)~1vED9H(>kU9~ZC2X(FNEn#`8+ZA-CT7`e2$;whSitJ>wc!E8|Nu-?#nhj7`| zcr_C9`A*8d@XkyFzcTl@VQ*B61+%h|C&`x=1BF7b1MYaUTp}b?MF(>$WG>{#N`}PA zeC@rkPN)5%*-$szJ~2>~8vWSnPUD3eYhFLwD-q*=YJ|Fho+0*d``1dVWMq~|WN~z= zQli^9ea_;w(XQW7(`4cPh(x~tnpvpDloGSimFp2m`L3-7=l=$UoNy3bh_t7~u5SYW zdFXQ`Ll55hptHb{0+=`oCWABng?bD{%;Sp5@cVDSU*5Ot+f*@NVN2Pq9s4?q`A`ps z_TbFEUSauYkp6{dTrwN%P^aII$7DMdT}0#X3ZPJ$mTJS3R>rgIkv9NPR0!Uv%Cn5W z;|OPAz$@|n^c7lqu46Vj=%@?TD7^G>`_GOw0JNgB@Y1PVhZwn>JbS$|jDy*;@#B`w z{KBdgWlQph9QPVKZKPj~fBQ6@qci+Wzh6KO?N-;YHv1WIO{y94EyqFhz4}|6$M}CFH-zgMZXz>X;sSK>W zlw%fCIf?;n0*Z8M1ai;LB?r39Y$)A#C_=f3BQw=K+GFYny+-=t zgc;Q@qDD)S-TOWUo|qmF-4U^k1hSDSeFlaENt2h&gr}75nc%jLPf;^UT?-)^j16o0 zZXC2_bp+ps(Y#n>=?9SR;Yx88myD;G>n1-HXsY&(I920AocifbY*a*`2s35a=Na;4 z7#F{zFi}J`zUhnyMGY#Wp00w1*n!zM++$c4`iCpAJGzxc+K{QqFGhe|i!GpltYx%| z5JSz3=}@)zvIZtpN!{)UzaTp2wYcT!5%A&rRFp{j&OWOQ#8m|t{0ZD*>H)s4fr|l` z+A#Hc{PEL{{^?gKMB%WmgNhf`sxX4m98z)?s0rxpC43?)s0ll{KM$YquI{kM)Ji))0smgK%a z<~pg=HHtyFZT;&9x>@fX_m7^)^C8?^c#5MoWK-P> z0AeP@FY+>scjnV_w&AFmdZ|Wzbz2mk)bjZhYCUS;7Bu8r-34kt*P{U-IIFciO`Y+E zwuC7HI0!Bxn5jwL$~}AfX;Z%Yr7+nX?{Hm*ijDe7ss{s`#X#QTp3-V)$| z61gXqF5#4wwfBaIoH8q%+fhST`;Fu9{edZ=N?$R@tL;Z$R=ftuqaA(T*_|R`h%fQ6 zk)u=uHIJ>YKr{3P(S`f;(z0M5USNo$P7!@>%6*BOHxULnr z8gAppjih0{1y1z4Q|`b7geMu8L|xHI*)eBalN2>_w*E6IGWGLv`I4Cig~x@#{Z1_g z6)^ApRiFYKa3k?@p)4zggkS*_p;n2fV#~n-l!r8OJWN zH?^=fZXAv_i$wC8GMGg+5cqyYImWA7iznv<6I+rh4ia<3)hBV&dq%7#rL`>(5icP>}7$nMLg`>iOH zSL$S#Px-(-+tjsGP1mr<>jwb&)rr$DgDb3kUK8fNp~Ot@Eakg$V#j>~A_rH3r|#V2 z>~unK=w-S&$d+#^1-DAxscwE!5Y_lL-6T*_`;SLY9As!KPmehU^JX4s)yoR*F?{U` zxul9SQk)nR3uSF2(}7e%46nseGZK$vliK=wxryZ>7nEk!_-z%tjq=^JprId>qd`;~ zo1TzDx737B!N5(SM2^iDQ%b zSqT(@j+=Ap@>6@Y5Y7QdE%Ln0lNxN+VfcR7d#EK)r`8h$1hgi%lm|tsr=NQdLKvZ; zX%YNUplnwdh(N#ei_fn9Xc$bK%QXRI+QZv@RGi5*?DH3N5y=2hV6afGvk_r9vX$W* zc{qO&i~b=t18$scvYNkUYzu&Q@P9qT9mQ8roQB@F+)z?IV{={DV!KN}#HzvTACAmg zT&Vgy9LCce7a!ToaU}S+IW5+AWv6+BG{SmO)nehw zr=c$DMi}^Lfi~Rt{cm&ze#_qA^L}KLck;g=@p7i!!*C@=gqjMgOrEge<;Ie#@aQnv z(o%Gx74{z;;_rV0736}UdC>jcZs^ztWhz}gvPDl_yv;zCXRH2@%Yimklm&hz0#mmy``>JLCCO3_u<}VC5 z)Op{Nj5JVc`bJ&LzXwn{94pKZqeZ%RB$)}T&b<$S8JO8xc1&t~wBK}@z=sFNH(a78 zm%I%o?~5NE;KZk((!|wj;oXDe?`;0Q7ptk=tJcyEkJ{(Oi`juFhNc0_0xTTMBbT}l zU2ajRi)Zt`sKwzJdhP$dQVC89%rdH~Zvt9b#0zRhyMaJ6vPRHlaR6n*A$2PXcpr0A zFT%f075G}X1M7e07cm-Lv=&yg^68IPVrb)+nKZW}dlN%fzI&k}M`BY=|CYKYj`u>) zvcQY`k|dJB7KNdS+mc?qh04Lx$6b~*dR`@NL1SLQ+wv#5TCcu`m!kl^DHa{@cO#C` zR&afp$64-Od}cWD+BuCWN_W<9=fk?YZ=-v61^$+-I-s`2_1q66c-`i@wuO=apf;Im zhu;|y3YR!{CY2GsbIW6RtE)OY_VRqH(J(_7Hsz%~R>+A{?T3UbA*5LqcQq}%>^B#8 zNAI9LvN>j;^|7O2p<3?GH`dzanMahfqjQduyZp zpH7V6_=tLw_g8F6E0~gnHXvSO#oNrx!4sBYU|9(p4GH6wbLPs^&lU><9uRbOD1E+z2iYDJ{FKgU|bzsEe6bYZ58{w<|eaXOQ& zi6lcxOJ^>NQ11V}r_{^Z1+UwePNep;$CI5BSGDaoCwwQ_o#4 z{qh2jSls!rS{Mt*ry#`J61Tf;A zRO6E8JgF&Z5ZH$6H2Lk0^_j12Il@6W6V7j`|K8D|8i~Lq2F~CbnYmc^1?7^48souh zFlMJIz6dRR86S^a$?~ zrl%=(m$)bHyp~>Oz622A{)J`VBCt;aFC@U};f(|^lpF7F7Qq;@8NwlF`)58g?Drel zFK=U%>2Rxu+*O#8=wGueirv!mfM5(dZ6E|6Zws6EgxWv6YOhyoWNPo`Z!`dC!xop) zIQ|-ow5Nsssb6S^+xzaIBld3I-uZzPb3cl17V%U$-qPMXW1|u_Y4-L_Hceh#3Fb(k z5&C!ATnLi^D6@`>%8r!6{LKU!|7CbjR&0-j3(Pi->oDlqNjF%owICe;{*Ir-$gJX@ zGmeHs59Ad`-l%Mvq{tqtPbTHd*;?(rmDUha-?@BrWzMoNNa5fxp_jfJh?#xqz|^Cc zJYR2Y6Sa`Pq*8JkC2DP?epz);;rb#PLj%6Hx~l|`d0zveqw8gdmjTS~9P3Wh5BLz7 z<*4Vpem@Rz49p940DAqLA+R~e?AqK}w#P3ZFp2F{)?TCno;!1n`K9OfGeKt)^R zN_aVz5UMwjfd)d^V}kV3em55|P<$S-M?60_dV_*zK|N=WcQ7HK*j+jNXk$Jz-06M% zmCXz(I4`u zW@!$yq_^nvu|K9=x#I9i3^j2a)r_|F9IijyQ*|0v3~1a|(0fTjzML=-3{aWmU#5|) zdR_78E=DG^)!GU&I9W{i( zo&5eC0-sEnc|8pknvLya_)7B0{Z6&X{^Zn678ynh8hEGR)(hbs#JUBO>r=m5aIcdG zxP=^E4vaBmbdvIYz9yg+_N6IzJhT z6c)Y*CIOWWG(Cg#>4WKnxEE;(s;sS24`^7JImDXp$89(a@3eTEOd3k6vRxJaL3vnK zCu$!Rp z7|5EcPeu@k5G_T9wLr@9;b)o(QrYCEn=u1|51;Ai4&p4zo>bKcZJfzB{O|8)s|iP= zhMutFnR54O;4t@s`OB&BEG@_s%4Ie|2RfkNF~-ll^X@b7oe!ge-%;2Up zn{(RzzRY+QH6Obq1w)vK)UQjoU4oR};uuwJ0tcT+Pi)0$mWhT*baA3H+I=bHZnRp@HXzaukLkIWBta`h{ECTweXOmUp4~V5b+$ICJto@Klg)Ow}Xc5 zlF(PVguz9#s6emXk!2O&ZGE!0Eo$8hS4h7ICFIGGb5Wn$xPlkd%UUnfOaj%Q?3wt0Kblrc$ zhj-5D;e9O~NZpKp_rH~n^@jSHp^7E-?h#5_FH2<@anEI?W$rLk$G zq*J<)j-f$Gm2Qv@2?5E05rk0ziJ@DN7`lh%+vEKl@Arov!|s)Pt#j?mlnRW*!be#$ zbMSQhzEq8LUce`t$opaFx=Kuu_UGQ3z6p^Hl3Y_Ex^?o!zl+l+`m-9o%t$_RTOlEH z$@Qt*Eruo6yfT^AJ$x06D5a6nb98W?w*lWv1+-+dVN#9sov`}p^s0;NH z8IDV(c~m6Ue2Hqf#K7{Q;wC9rw?Rb0jhR>k#Y4Maeu`}RO;?g{XpQvN&n%8eM?Hpp z@2S7>P;U||O~NGF!5L~Mav0}F&0DN1KN7v4M<;UZe`vxtNzk-Cr13G9dfe8;0LN;c zsz+1);r#hyNDvVf=8kAa-D>B4cFPKwbQZre%WbJjW$K0|@8xsY>)F?r5bO`5VNv7+ zYe$i{22VRBufn%dCPxWXsq`vV_X=!!8y3Nk0gfFj1!?Lyc=&@n0tf zxR(k`^x^J$ThVD3S}ZQ3@n*AET6|f83PK%Dmsh25eiPSc*}@2<=AfFb=4<`?xCq`jFQHI__Sqyjq*onO)$zEGw#vdbh(`{JZgR?wrj z+@aFA_>VpJ27`M;0X*uTr{P3i{Dm{xF_h}$N*m7{jjdCu_5$74oz7#GTfRF=(XEJ> zZ^?s+ZemMUdZA*j;JM)smt z=Qgd;bi^HA5lpMnJ&uY9c3)jG=@+~u?pMWt61?wfvgP;Q1PxRH{Y<$*Ldkut>p@lR zw=}bcd_QB&m{K((Tg0dTQtgX5;e1dRuX+(4-^aHU6FkZ|iqJj9Skptn$cgubQ`G-P zvjSL2t+Gc^E&)qYd;_r9W=^eJKlS_e{;o=JXrY(Zttp62tA4^(%x7sxS{@Tz_UaSo zUZaB$ACOqNuwHo-SufuZjTRo zM&okrYJZV^+yJ(Bak$$n?c+8~04_1%&km8vdU5YUHh%8M`#e6qNtus{t0$&|WSFFV z^0%m(u|vJm`K5Ts_uos(W8bkD9-4ZSrqY!U##K08e?uAgDPQmUC-lk@|JYxugOax5 zW7LpCRjPuM6)i1z`RK>%UlJdwPZpf@RqVAcm#%Pd(=?R3Uc+r(hKluuaHzTPVo-r@ zxnuIsQs}U8wlF+wkIgB<1Z^rKa_;?7dPwp0tpoG}yP-2isiO9}M`5b(?%_5XkQrfn zlm}2M*&G^z291w;TfXj^CU-qWDlySF?t!w(Kw$8ebkBh1eRG!1GTHPexQRQg3i%1M z4+RLrTONSKS`7f77Yed9oVy%@I*QQ!yexl%$9fyq6!*$YEq@Ic_=}!gDg*_#V4TnA zf4=siFp;BK>_dJMoX4%RbBzPm8X0x4Q@~<3&B#D5qYGnXDSJ&&@+8%vTMJBh|YT)&B}jNS6Gs!QqEaq zupJK1@%tZUHDoS8KmiDzErA%L3PAcjCCx0c*E-vo89FE~EMyq|d?^P!`vcVQ=>_gx zIHu<LE6+jI=pj9$^ICf!huO5K-?YxcIN_wpU47Us#5B*D;3 zz?yABl$_!YdYx>%MqfXp&PcM9Sg-SpE!0_lT>u<9NaJ?6()?)?d}vx2ZNA&7YkJFm z%(0+rNm89@%V2Y(QNz+LP(8ApO_o|;pbmwGvk*$y_oE{5l$vk#jSPSRRKE&^DP0xp z81SP0T>0L8O$g0k(>p|b+ILuf@kldJ-7tt#>UCOMLFv9$FQ3Y@a-({kWB8N%Z{o3@ zHfZn0UbH3Sf;k~zvjCUq)S*zNtTaqyUARYv0kX%%SX2>JVME}sz6{0^@)51QZx_LY zT6J9m=sfoU4_0DggvGJDie7V6pWG^7HklOOgRk=*Cy}(LDjkuS*8DpMmW-enZW!L~ znkv}FRv$@)P0`WmxLnv!SNNR32_S@zWPorJZVRkto=|Wa{Fq-%nt{a zckF_~4?VLM_8#>Mq<6?$c-^UOtZ|RH*~_9-d4p=DDMC-p(bh~vs;^p}X@37`K@Nb<&3{y1 z_UREME$^@J;u8lW7|YSo2zOM6XNko2dbbti*SzFoep$gDT}hURS-y4Q@)Dd&@K6%1 za1iTWw(3J@<5lzbX%o(qV#F-2_cqv&nQd%%SEIlb9wJYJ|WsmONKUK6x%15&t)6-lPUC1xDS(Ta2G|IfJAFP_B zoyyx=)V!MQGG&Tf;&67Ssuo$M| z+WfE2yM;^F87OLc9vwH-5-WSDe9ZmDd7h-5*tJjM)}>%VsU|(BA)Nj0EZsYC2V9T> znz~HTv6%$lg&80F2z<_y48zczI85Y{4KKY}*`)i3F7pXP7k&BtWR#GHTEWc0eoZ+Q zLmV&kR!YDD^7~8zmQm*sS&6Y1Fc5Yu`o(%#+}R!NmFof=i(LO3Fz;icPN;M-4Yojy!Kw6=;9={#jUCi zA%H!R3Mv4Tj=kt9D?7hq#VEY5C=pQpE+zAMpk z+=1ww0dHy^aOyH63+5oRIXVLlt@J~}kWYwjmBh%KT0Bir{XL`lFaOp(qBye3JvFi0 z6_;a6@xt$$3E@LbEFU?aO*4<6{BxsHaQ=O6N6}{lLRF0PJ`)IwMppn*#Tz<&fBXym z^di4hUyV;JW1vvAMY>gjp?)|ZWs<%5b9PbhE;o4F6$;3YOnHsDULf81*w%!4aKVYe zA0vC~MzRO&}cp@lkFU4o8(xlnr z#F%LK4Rbx+DAU*FGu}+U;|gg;%+9nr=mC*~Q=U?J-fiWLCQZ%O$~8ahP?%}w=yl-q z%dIC(Q#P*t46%EJ$&1$DyiW|$sna=w*{;&d}VCeVB9N|lPbfS;&+EZaRa z%A&D(VSgSC$oz|aj9bX5#Mm8u^6y-))qftej8dOD&^w;r45fFA|20;vaSsgO^y;CN z*pKl1U1PuLrCrgiJ5M5({0h@hmK~B~p zBB2$4!dj@1kHPBG3(k4l_C;(Pi=$Jm`vm)O$B#7l>z5jaIn=aA&iGoD_sFL*+h=6d zkang15#&A;(37XqP9DsP+PU_rbi9|t!Tsuk=s_va&j|)taT|ZVQs<(Cr;@UIOwPo2 z-M7)pmrc69FqA|YtocLKCjGJ3Mde?bN<#h@t0G-fio5m#Q)zNATt&fTSI4xwL|yd4 zZe|IEF1Oahv`HslUl?@jGDf^G};(Cyk?3{b!cUWYkPe>;uyZS6Aae!8R` zC%fxrji73fW-O)yos6&UNGWZbt}4`i2iE47CH-hJZBvb39oi0@W5)ug*7O52#8Ii`fv&6iq!!2?=CkuJa_ zz9=*?%PJ^O=}}Idk;beLE#aKRl^mML)IVTwTr=b~9EYc2c7I#it5nL>%c;Tqf6)D z?~lSIC!TPx2g7B!v1KU!Nxth*^OB9(pa0Lh*SM!2TZ-`&ta*p;d0KI zb}U&+8dS_{;_d_1=bo9Z?BekGE(G+gU|IiM|`VLZn_#7d-P2sXfzGYI`xVGc~Z2H2LG zfP-n=c{{XLy8nlUfjWLreKdQ9-lg*^8(Yh}-{LMch4Xb6{=iN`PC^t!7~&6(SRS+& z8}%)D`oq0G-N6MPp<&)lG>t5}>r&)J)sef_K+;*9aBQz~E$?I_Y*_@YT09~lq2IU> zd(Dzu>aNo`$MT!x|2UyclQNZZ=jDo&(&nyLdau!e+5>U2jkBpuk(uB6x{K~l_`966 zO(p+Boj@$hrj9;+iZJ|&H(3Q>d3@C$cN#eCE9i?3o2DJL=yCfWbUM5hsLsJO=_k=D zky79NkL;RVR9R+6&6*Of=Y6M=iV?ec-xq4LS!Xug!W8^h2?jj;G;I8)xSYnvN7w21 z7&>`202K1U2-?uq@hNb}?e~=Bu}Pl@ij!&WWNe6-hbo9Xyw^avDc=6m#T(aN zX6Zl8ydQJ10_?-zV-xplWv4!qj=!jq%6;^_4}E8l#jpA-ub=#D4t2Q$p|UFwxXfa7 zQejaY*LFC5R5N&{m3?Ps0qW86}-<6KGXAAs1{dZ(dpjFjXkNzv%fco&;@>{(O&|fmJDS!6z z;@h%}GNb&6aOVnT^S0%~j==?9xU3k7B}NTQl zk_OU>-NMk8BmW*1h9)zeimirt^F`EcXJ22i{Tt+v@{gKkiJ`pXqVhkHFlzd@WSNH{ z3YsT<8ZGhDik}72JIiJNwQgKAbrf&QGrXg36S)FL_sZk>??5ELTBS2f6STY#zynR? z39+F_mujH=_fISGKWaoicvB2$l6D}K zr}w;E+Wo;(tJv>$+nWgF8*jLmsqjOfW`jtR>Sfm^1g$5+zl2jmGP2+0ILXlAM$j@FbGGfNTApEU7+J8c1f$_~AFOFt&0zC|m2H#>#>j z9e2Xk4$EvjIMo zKLG@$=WJ_Pgex_thB$T!u8l}OIdwiB^`*!ur!)g=%@zjcyL!g8jdf$yW~Knpewf$H zQBD#HAQ~MXjn==_=O3jg^bTGkcLHP| zUNOhucoO^Gl5an);?GCe%{q~^tNXs?Qf4}VI;29vq8qD6rI;Z*7X@3Gcp(E&C55nBlv`0GK*Nq;MJV5h&Q#CYc`Htu6?%D=`KEUzHR5& z6V5o%b?w((eN!bVppYpVw`6e>&wQ}tUtelJo-CyeoUS-}`_*$sk)Y^Kbrz&sm=*Ji zDBi4yLstNvCcPM(Z&=-!6xOLrFsxAj+?q<~x%vGI`fWfVehS%0#DvVU#sNJwlfRkKFt z%!epwtW5!XRA~#-oh5`^pH1KJeorN>G0xsrI7;wWb9&c$P!a2UoCo^-sM4uO2~((U zSCHv@iKF4!@Nv(GlU$a}XDLDS5@iOirbib9D(bRGeE5c*S#}@ziYczc6gHT3U8ta8 zDYi(GUuV9_=9J8=`kxj6f8%KQlODFoD5`XcA$yNI4j(!-ZoZ$^<2s8{DC}UNx+>U` z-s3Gl|Ko!VUO+V<1SH|fJ!~7I*vHVudYD!RCJHyGAOr+ctpx-F)>gPTm}0~1>trU# zcW+2hMw0wK=``(g0IhSdpf$mes^je56EvPo-9onYayTXV1DHgIl!}=qJ~YD|zMqCs zEJ(5;7=Kz%=y8yf(&8(tzR9B8-_L9<1ue74QY%<=QbUsx#F62B{FLJNDTCeXM<+If zj2`F1r6x$|9X8)gGJ!zP8b$gPhf@S>(mEh#7z#+B7OXM_sdKBAwH_L?N>87OV5_2d zCo($Twt{o8<>HdVUmv|8nSMg0vN3MYT+6|hw&gv#5*OXDI@iUhgL zj&b-hQv+wdttnE=ThDle($+U=sxzLI5CpWLjKsZ8&5umAgMmM#Qzp&a zgs;8qF!%p-1Z69G9W{S|y{dW99C&Qu;LpzY3{Q3L&~{Y2<+|3hdcx0FOw|2%54$7i z>YQd6lv&>@dzX>nW%Giif-;YMHml;X2R;?(qrL||5RKsZ7hitv4^Cvda-*)#Yd8MT zc&Ac9D<80A*M$;$bD3^yM4KNFFD0>?4dDfB)7OjuU$X4`U>6EV_R7ISI2wE%q2Pc4 zsDeL0+s~s$DpNMz5Exve!k#P61WM0|8-JySxNi9Gr(&d(zy*`r2!7KqK9&oJ?zUul zsLGkIaMzU;u;OmJFB-?pQx5?j{j#1_do%{ri>j#OP+J129=69F-}vc#g|?XEsAnaG zZ7neEa)qKHs;cxRXab4GrB-@ddMUj(V=xpGdYXnPU?+Kq(BBNq+{J>MIa2B%I`9!$ zPPS)T0ZF6dti~^@zsmur)T_`h6D{h~+%dbtGvLWen?l-yhh1fCMe$cIMF8wTzzMPP zd_k0m#WC)y7ZFvpRZKoaVCW0(w$ZpfKQBLEa5R`93FEaP+ishn;iK>?oH6iSp{SjY zFz&EQYuODZpj0?o{nDXq2pqq&3X9OHkxO0AbZ>OKg;`ay8{ALk6eBEP>5?YdU}7h9 zVp$EV`S!SM!85^zEjKL1B2-zX3P9ALqM7YvR zsxP%w;x7u5$Nrk2+I`bc@E($Xl;W`+tKB?B%)@Hjq%Zh?t1Q9?ZB9-i?{5mWJTF#? z(IA29uKP&-G^+qApwWK!a7~@1-JbzPo-?Z)qD1aqf#f*5&Qge{Ot#?9h(;l7eCC@*s9Ll&sTU7HE z^{7trODrq|BG(>$`wWl_v;73;Q?a40iv2HR1X-c7m~dtf7J(l4RtB$a=YUv#?_M3O zcH~i!D0dgutaVZ6^@(8-{%qLxtLFHvrc`J{o|MS{j-Bqua6;uc+#_BYW@HFhDphi) z0R?3X^`nm1(X(gJzKW?uGV1}94D-=>M5Qfyi-sAzW%~*AnSaDWvnO;1;?4?ZzPOk8 zBhtxuvXenE^5g2F>qctCmNlKs(XEpqyLX6@;@@E<1lj2M3eQn-8*WX#kG;F|DQoCj ze_<`9!55@YNqk?RWJYTwsDO`YO>-iOhy2)q_?kk@$>Q$2&pYz-WoA!1^8$m!^CQMc zSb4Aoqs$SW_`Z$9JIRsjGT2TdXO$a5772V4z5#+bkEo5EkEY+ejx@(RT2jCGF?fQ} zSE>50J!$%*%WafG>=3shCH@>LG@SWSLHjT%ONr5oV)3Mq8JaWew)N>xNJ} zGdi>Cp_;wzDz>~FJ2=*&+?p@8{fSma9A)TS!6d@cIo*NMx8yghPGX~ZLE4duP#T$) z5i0xrn4oZ(+mL2Q=Tq)ZCp#pe3LoX?TxMyN-|oB3vY6JRkvz%bY!I{a(dVj!8~>Fv zNh$vlD(O^cT zTf!Y_4o}wH8hOKYgT?jhoDt2s9uEiHPAY_^TBqwTweY8PxW0}e)@KiyqXQ3k?^)sb zKbI)tF!`C`DJty4MGDmi<1z;4CNl{j8ZQ>cGFbQArt3OEj2N9nN9-#4@O-=)A2?aB zCMyh!-*HvHxBi<+d>RMq4p~{)eIqm^$A2q7tNbeVgOKi%SsJ4)wF6J^f)0||ZLLRF zR=vyZzGuXHPi|NIB8O92Y-tj6MVRM(Pq!I&Dgu@{ae%tuzIRLGq+^8&yh?t|!|BA? z5h-Z3>h+coszc{gavWGO>L%ukgS9+ecKs-|%6IgJK=t5|>v)AXT&YU!_mAlmGrVS z^*HjIGpOlDRnhoJtlMnQxF^FRGT6gnPWQ!#Uz85jPXF+%+%lo+;wV>JfioJ>vu>7n z^uv$rUc!|Uo7VgTaSaM{YqQxs!D(ZmWW0bhx0`mrYvIiTVfvXLBj4})lZ5e{vo!zg zQlzRCleubE!5gMj<+zxGNC{vh%ZZ@pOYVd#C0g+GeZntvHp$|e)@8Ofgbua3+6z2` z-HlV|;CGGg1UJkUM@!QgE%aGuci;_+9jvI0(ojM3)81AR#5K`ZO0HKKp>Nia5b3^{ zrE!cbkkzm|*8U~-y;sfa_`ntlsL;J3Y=D@0J#W3i9~NC@+S+jUJ6$ zkflj2oh;`tEXpKJJ_s4N9!JEkaXvl2ViM=0^o@mQYcnETg4N#lVH62z>UBt$Sy)FZ zmLzRr22&Y2L!Cp7^af*)rV%PZBfjh*D?&!?h44EejjZynQ04z z&}WJ?Q^vEv6N$)=u=t+DvB-JrNU6*|GUHmaim%~HL2LoyqY_eSdFnTev=**Bn!;q7 zwH8Ehb31W5=^*agb-Unax9c(rBb4W3a)!+rT;7f;%w*Je8rCl7l4U?z>?{$rLU!I0 zik3+#f&$1hvK$9iL&2$Dx2LQuJcpZH_BxW)>Ckc+U6|6VwBUl1#vWOnMpB{O2R_@9 zHeHX~Ni&qumg0kLkOH%Rtuy)%>vco8{}<*m2f;_y{g*JCYXITd7mvmN;pg>3mSd(ggz=egn0Jvg^fAKl}P{l=zCpG{ep zPDXptFi5}5oilQcoGN{e;;-Q7vG!;OleNQX^7P1Gs77;5Z?py|yVOUUvZDDv|123h zI2Mj4eHQKZ5%UVZGJ8H>A&V|gS>PuYANT_8pD2^==N-AzY1~-#je)gZm}$lNs3@G) zl));9;KTIRH}5&&XD2Kso|G6U=4_Iu-P3_zsf()^NquT8{QHaq!ZP{Svf3Tr*|2Y8 z3viEQc!IfJXqFsv=DIf^ectMNnjSOKdGjx-}|*Pgk|VQ*X8xRO`~S3ecKM0Ih`289fC>w)7m(cQEjymn+$5f ze64dATT9y=;?10sH13#{5Bzg8A&Qo{oR`+N*Y$L%0<`3?6W0k@gxtcF68my>F^!Th zX4;+AvDbX6`-=Ho@t3;~xje^60+@LCLn7#zO)V145qc~A*Df~HL^N7-b$bP!A>G>> zmc5+37emHqmIs5}oe5r4wJ$qnP?pj3)sTbjWPB6m*j|4<=}!YGVjMvDDmHgZrRrr=7%#-a63 z>Bjj_$EwSg{+S>FBco|*J?eUln2X(ttb8ZjMYRjJE>NHwT2{*ZOoSZSe$_e$j8N?P z9Xos|alMfC)bR%0>5jrzOL zy_eG-TTeru#)hx?QHd{q@jctj+uSZHcC0?4FP>gXMD`S%cbWbKaO@lmb7WOwq znSZzp({qGskY2Hs%_exJ)f?nTCL8TYq~Kr|FgE8ndUXC3%^u=cf5rR zf!`N@MYJP$+_KXU`S`yytX<(X?AmFvzwPgEwO758Js*<0cM%EwXDv|UKnN+ zT?jOH6|Gg|+0SJRFrTKLKR2C$`skxlAB>!~{tO?aU$d(HR1W*!4oTCa5ZU{{xzj4n$}vkLZGi&k*{ z0U2OJR@ZC1@owv+>GvsZU}k()DL{q{#X3AU8^-HjHt$KdhHG3$U5w#y7cBQi08mGw zzRbQg!nns9qH}{qjp4B;(1=Vwi5zQ5lnq$GQsfq7_F0c`LLK@d`0O^#z?xqbW$=+* z!VwqEk;x~e&V*fowXVU2x_jb5nmVnB1)$(rX>JFl2_h#QSLFBoV@W~^Y6vQ!+z8@5 z7Ek2LcLklRFGjW?)A+7S0af%)3SM7-|L&MR0Fkd+tj6cbal5F~x}~4RQuBLNKfCsX zMl2LkSc$sSzh5q_1bOSb9{9EP>6Q>F^6`fMcJ8Mz5$|;~(p9EAv1wwD*sFDa)mwBs zwRsjXo37FFH7IKkxjvqr!Q6z!4*igC?p->PSy=(Fl|>!C$;bSF4_|k37hY(IU71uN z_*SKGDL{M@YojaQ>!MRR-EzA#MM%Q4?^W3a*!4fD0pE$?;yacJUzp`c9P=p_9(E8` zuyOY#%Ln29y?&=);1EmnMN#gL$giSXl%dImOnQPrMhl+Pj~kA6c!+J+`E1CFr1Q?( z2E11hTHsqGo>QIsx1Ekxq%q>Id5qZ;IPxpf_A+FZMt=f=7+sOk^YK$$1UZnw>O>N! znbp?^JXIt}stva)#kNganxV&!?-kB~Z@HvhidbPS+c#wYKQdt+2QvMaD2AsTc$BEy zp|175V;`Zc&a=u_BPcW!$jH=Q_0H!%^P7#n3vssgA{4M+{CJ3;?!$!2h>d4H>>J+s zzrlohCibQniil^~aLg`!L-@eMCat%p&gaGduoWOhE&LQfwvaljaOIiHbZBQ>q-_iB+6Vh~(e|cL zjRuVMq=v3iL$9$GGPg}s$ErLf)E z)Vo9;!Ey%`+fDjLir#RreR30TYrZOpT)>Upj&efJT>VOl$|!0sdfCUqU!R1wz}-t0CrCh%fafSE6acvg$S_e zV3Ecw7fPQOa`+QV)KT*#oPCJ#e?C!QmM32%SLGwx)8+U~IrYFUf?gybxY_O|#Qff} z;k;tYv_%D5&@voUa-5hfbY%Od%yBvD!2Oqe~$9g8tygB&7*>v(;m|41ay+`mRPfdr45Up^9 z?WqF!#2<69b9PUBtIq>78+9I#1~=a;kv=MXjrLJ;?M705Faq1zWW4m97V_JtkxS!O zA5fTH@Mu0_I@8b#L}%L#)(E&>3StjPlsh}RnUP7W{~dcU7I3_Pke#8^|8ChJO}N}h z1xyg<)5;Gq85YM9K>zC4XJxIPU4ZdecTI^?ZE||xB~^V7UA?E~+*7DBk9pMt#PX$1 z<$c@%Q!_6j74n`=mL1-J6?4YCFl>&KvZ2jVo@`S&YU! zJLk&ndniNn`BF)#pdPOy-2>+Fq<7{2E?AoTByV+OdIXa(L7rwp)A6t|eVXjU6ZPf} z7(q7>Tn~)loH?RYO|cX3w;*mfTH)2Yi1>rfn6%`ai*KUz&0tm_Jj*{{>;P zBxUd#_$ww47If#w;d#-tZ-ho?8Yd(#2)@dqGoGAVi@{Y;7hPTJ3}yK7T!vTf3E6## zXQ|PK_Zh8p_Qj@sNn#|@rZeNq7Ud^rXFl6QWig$IpS_5xxOm$r?{;vPnV@Uo+sxTu z7*&5&pEa9>TZSD#Sq?ZGOZo=5-m zTZfz6(!a2{D1236f%%C+G2>xN*7%OB%krKc$%GSp82wU=nAbL-mWfXPtG4{+3u@V> z1(LLi{W!=L+kY)ihR3v1Szv00qw|Bo<;>}&-j+06@=!;S`W4Bf^1hF#K@vfd2(#nJ znAoT1ES&gdr&2V*yBD;+{*@k=TuW6VW-&TB9VZ?uv8SF!zG)Nf|5^3%#&C2m=H&kH zFByA4seb8JyKPid0LSriRfJ{^KSk0;y?&5Zy0!;( zS#U@$XUd8xU)ta!NhP(as%vA**7DNoodvjxc$9rF*TswND8b{up0^Z`gbD+V49Wr^ zm@eKx?Nyf2x?O(6D5a!9f_$_oi71#HQ>u!eZqW)-K2QqB?AhjY2y|I*uiJnfE~YI<&05xOv9hVz|pq9GN%42z3F@Y zbarp%QKV%e+H_+4@_*igHTrU4EJnN!{Y6F(Xe`w3@N&~DNf1sk=&X09<5db#P)&Sp ze-UsL{V78*fLUmL`me)PkWU_wSiVRW-*gtB$nGp5ooxyWOl+ zub(aF66=y#UG-z^vjHyLcJn?saPh>^$oUTZi(~Dac}_)}mN11N)}rsZ<-UD;cYNN% zUq46fHCeRQ79Q0%TIU@{qQe`M{cl)0=HVM8S&AeatF_CJ^Vr|!Z`=s5izD3Fa;0DhjjySYlkG$rXl#yu78tIi-|G^2gx8J>Mp;b|*=Bwg2) zwPnXZrsJ~fv64KMy(NNxhuJQTFEXvwzs{DS}p+4)_#?wFZ zpPK;Oy@+Fzr4ui5+S=b-wrv>-d(j+^| zLk+4WD^eF%TVBna>c|k4#wF!)IzJo^9L96NWFz@Es|Q;TqOAiWzOS(BpFy-i7FX7yk!3mGbE5Z0(d()3~XDR2W@npNDf6jIGx_u@2~w$w;<_)Ark~5gm*MZ>TQNH~cN%nao?7G#V)2WyN|f2D zH{??u;u*!)d&B{RqF-kdYp)BV&8=)F3lTi1`>N>FM!2Vh)8^8d*j#vp`N49K&Xs+0 z$M73ZI`n)Cdq!@qtn|XI`e`vfTX!=K$hpT=S0SwH`;%G}QG#{05dr6YH2~W*I?KD{ zuz9+3mpHB}YGBpkxR1H<9?P?**J%}$&NE}A(c2dw2P@t2ARZ$cX-LD zrfvs(mIs~DNzdfT;^5!Y*yA3 zz+uqNxcn;r$vYvAb)J)0jFi9tH!=uS8RzQ#O1p|!m-zlUK6Eb&X?-L;u7bWQ{qbT- zF}SeThKS{_^{(52x}B4hJPJ9j!mVzn3oOfY!e7-ch#6zl{2D*rG!MXoT2y9hZf}3J zqRjNh9P%*3hAv5r&RaAV{!X)K#G*fJy<_FDcUNz2z7OX8*!g+Bpe#QJhB`Cof{1Wa zAClk~4qXIX#K*LQkg5Nviu&|NR@G|yp1MDABA;~P2g>tbPO1o!S9ZgnNnhTIuJamu z2_}6!Tl37i(mOupaQjBrI>MpbJLoCMf5OKQEu4R(c4~KPDou5isk9G7-84o=_d*%E zk*U!dco0&WS~OuhSSoRXDRrMvfnU#kvlnEe}e%bF~BEv@SsKYbROfZ}F1 zHjr9ec!u|Hk!{^Q-p8#LjU#B1T6}WqY7N8udH=-8dCCYqWtBLHOKsdv9&0b-PYS8t z4^+=tJn$X|(Zmh1EaNH*g^iT`?;Z=PJI=CCBkF8pYc5xoY+H#Tcreo-;%}^kqn(57 ztOhFf?A;bwq6K_74v|^^dyOr7Rese2Bo(xJTslGh%3Y_DZjZJ?tPL@p>DLjowHR?p zFvtU5_lZGca^Vci_viVb0KIFXLuZgNofN>xWU90ZqRmyLN|-(0TVgx~>LxQ6e8h^7 z3wzB5y=0)e!e7a(XPE1Gxk=~!4gwixd&=ubL{H)bwS=2` zYvqpTC|+4tQRPnZ0^xOkJc#DS?4TOJR}^|3py6WY3FrO27|$tfIMv1-i-zu?!qKJT zLV2;AJd)IH_w^URk7IlVw<)MzR_Gt$L`K{w!G{7SeDW?e-l}Mo&OBQJbjUb0lZ-js zRj(zKrunA6w!8PhFMRIXMBg>b_4tz?ACI3pj^MNIJ?10lh-6Ska{5NmKhz`ur8y~o z249X?$N;8rgL;`{~WM%=XP>ZjdovAXjwuPOp|~H>;5kj zAGtq`b+&3kj6)x!=*lL0{R~?->hc+2?GVUIVr*zr#|`j;eHX@014>?ipOq z+m$thzdersnm!GVE;I^H1ueN@rrsI(OhsMC`#_B_@w+GsG|NJ$%&iC~SlcW3b~Gt= zGSb^Sbc2V6YVJI({A84B9Xat~2t%(y2hJ`3owFfFb^51P7VB%qWP1RGj=kw*%DQOr zG}`fab9b#syR#%Iwy$?-Ah9lJB0^^WuEC6d-frKy^3d}CMqm&!=P4B#xQh((`a=TE zW)dpX{P{|7tSy#*#Q5Urm0AI;QSXI^fQ9PIAkX_EZ}Shht6zP+0Z*!7pL$UWsdVgR4R^quF!>;OkFz>?|9&|nTmPf zL#?Tn2bVS9cIQzoQA-^1xxsg7nJbWW&lyxu1S8)mr|QjBB*X5ESU84`JqT9XwYM^V z_pK4};<)UTvrko@+<7}w|XTMJ?oBx=JLxs|yCTe9cpJKgrejsR!Kj2Druy>t;C=dImiQfq1 zTcEhJcdl3-S@?(}>F3etBYA4u#oYJe%l>zBRcv-^;$?rvxyx zm2iCaSwkFeSYB|4t8VwTWc4QgZORgY9L{+(xvQSfPDObk)$%Vi&WIPa`j0spkmLy?q%`( z21ynDV==0Ms;rnoo26e}8hp-DZ!4AExSzEQ%kvN4mNA#d)MZ)&YZ$Qqpq=cV+8X(X z7*Z{7jP$c#riDYHxh>cQds*=Ahl`&fMDm2?iQzn5w_;dKYm_%`%xjfEit%Mv-Q2p zXe(LKQ2l$CdUo8Af)9chsjZpBgMH5ZQR8kH>CjQBsvFW?-ZRc0oGC7VIL*Uw#U=Rc z*>JC)2vG4dtd7HrOp<17@bD=d*1w;NswGv$_|?asuci!xmf9e0-N*Q*I@7zywW6?` z`z4!eW9!G2?#t}i7EdSNOpybQ83teSu2Gv_EuQf;aHv=hQO2sv`}rx4tUdvIsV)OJ zaTyd@dVp4%p2Oyh(n&CT4ti?;5=Z!$lFYh12Ic*afg}l)W!S&3eTX4cxhlrIfOvOU zrPkI^1(y6ues*vKR}->&%L!jB|IRlw+CmI1x4dVDc=XF8j9T^NtTrCrjQ}8ZVhe=mmOG0Ayyq?DonK$(gw&!vta3}#d1*S%p zVnLM$X$G+_*6}zrzTL%JMlXIKYhMUBz)0N3zseuBN*u+<1n;4y3Zl;L(fv`9Q~0qR z4Mf1|o!qU3`|Y4u)|djI^k^!2t|zn*&rVgqgZf@Tm$E=4+3f#o>dgb8{JzKW zv1hN4C{!X#cBw3bO0tc8-$`~xVaPUkNyt)2wlQU$$U63DQd0Jv?38V+S<5th@8k7) ze}2DzpSzrM&wlT7&$(wTWXG4y$O+DYKzc5kPRL%PsVxI=%B~u4Udul;@HFexfsWGafmuN0@wBY%1qFZr z>?(Nn%Z>(Q_F@WtVCQuVS~52Zo4KGCJSb&vrv_SBz#b~d!V?F_0no801}EDkovxVgGw+}p6Ju&2uKp8l6+C^7F}G@!fpw97lCn+`0NCTqG@ zfly1i84$7EbHHwWoDZ_;Ex7<|-`C&(4Zev0eis-r_i-Xr9?ogiNhL-@5ZiDsbOQvz z1_3l^fOx0N4)o%TAmt0P07e!Ds{@Y+U{3=1`QiK68hI+(0?IP~_v;1l72FlJb@{ai z*<2R^sAWbzoJ}QAbFydIN1iqVN(Y41A!excUIB*c>eQ2N^0-|5zkcB$<1vig3||O1 z6e5N{Nrdv?(Zj$Tq_~f!)Z?dA24I^?Hrn9pk|3D(HAr=iq>QWi0RC9rflMI`1_}?O zBS4bK`TPM=OYCl)Fe(u@{;IALYO|{VtC~N@ZdTw1o{Z>7*0`v)+=z#UxiEs+W*@3- z)s#Clfn1I?SEwWa>;g#EB1&qvEw;4d%gmuKXkf#NR50_tW@Z%!77MBvNYwnn_6DZ1 z4|UcDcQ87LN_|&hR)(LOX8&&8FM&Lu2{6%mF~>^|47xegP%_nan!%nA%6Ep1-2!cC z_V)9*Ysd_Cg9MBgx68#UTQH?Oczajx`TMki#)#O#5fRq8*H?S*?txSV*wO`nZ;0kt z$MP>AskX@znbDQ7j$f(}XCeK?N(Zv-B+_u;Z0)ZhtTbR+1EN@_99f{31UFy}iAMcb z3qzxJuj`{=TH8kdlLst9ohEf2R6@c^vKfP!$}Ipf82~Z0`6yD_&hJztJ>0SWq7uhmAu$PTC2+L=0cqW$sy2v79;1X1 zU#fo*4i3s7F>U~5%=JF_fN!QI-pRMxe+>#S;Z;?&iknBEyksa+Et&>yeMh9YRzsKu zTzOe*d=`FY2?t@f4q@kQb?+ty*lRehXiAVKH6OwbyU`4+Qm-|jTKm6LdvviC=L)E% z>h#@5fJ&^_5K`Ak@UpQ>^uVq5zT%>8Fcv(Gj8_%jM+0XI&{hJpQ2>FEdf7{c4w+Ay zza{``D#9V(uN+5*kjwBFa(ZIuDdRVlW{~tQ-nuFa?Up(i4IpN@0fj*ifdCDETbOkO zV`Snj&_pAFq;xaU4I%pUycuQ;YXEh%)+RRW7mZZQtkJJ9#LA33Xxc_#h z8}M$8`0304LIc>El<@15kXHKnOY~T`sI+P>e-ixvwd&KL49C7~rAGO|s+x=QxH8tG zz+(QL&!8pQ#%b52pLRPWO-k>z!$B!fgo*`dv8-URK`E8`kIO0_S0#dlAC$5l&l&NKZHaKn+PNSD?QEO3@n~4TFLt2G} zQS8od)WFjf9qK#5D>ZCuh$^VTSpl}~9t7KQKm5F1m% z`#EaDzyTiU;11pqaN%T~7}NI-(V;ZL<~6sIp|Ge6tj2c=^5W%}!1fnQDX?q&FB&%B z8uhEkU>AV_hfMIk%PO-VeE9)V)nfjQgTaJooR9syR0q9~$&~xL&&AeLP#QQp*j@CB zvl%13_k1YplscBdw@qe1%|>laq0hi3l(jUR<22!w$wI%_-JJ%wbpoj0pz~|HP#G5{_%BpM5%Yc^6!Q8g99!bz2{#nfUF;P`EBCbA;T<7ub1@V z!SxXM2gRF1mjOafz^oH$H8bb$n=7g07^#Ob1Kn{2Q@)%vY<1!T$afGa)V_8sH}ygR z1(FR~-HxWvoi45O^Oa#&OU$U! zozOMKag51|ub!SceE7ZT1z_7O0hz|~%NWiS;A(4yx7Bzy;(ars#QEhl3OY^ouLJ|ItsO~vOeIv+DK1z4wE z@gA_0^m%Gu3!C=sick=4p!vYya+l;YA!op*z%PIIX7usOw1FtUgq#+qL5hYY5#Q^y ziz%p?LbXzf)|IX14Qxlm?(`k&_dHk-KAN2Q^3&BT*kYC)C`g%r+=Y@qgRuJe6kOsL z#bO*d4Z>*gn!Oxs`>e&3-&Jtl6$1VcT5_%_u)ozf14e5TiSu*-i!K^R#fOzSD3tA3v-$ZjkXBEmWEe^fV+^)Aju(V zbkdOG2ZLl<{kQ9-Gtmsc_Z{5ff?v+^EnIN@01jiEv;H~T1;1x$(SN%9ly$bWMpVfw zYezpe0Fj0oK$H~DJKyECgt=aPWM-^5*kwzyjPB7R`cR3&m(n z;1A7hp2Lt=ZYr&8x5QpUIe{+33xK;%4Ep)RxRraJ4=+CopU()SM&Dg|;g|-g3=1U! zmMZGu*HJnDh`g`N=Rji`6BMJ^A<72x8h?3*=OG9^5RH#f%S15Qpeo>AC~&;eN#P$2 zoX2|<8B`qoKtWT`#&HVrVyYMCZ}7ByJ0#_>I#0H>QO5bZq_QE;MU{HVZxF}9FI)hU z7l7ml+Syztum1oR=H|IxTx~nOi$;uJ+k_L)|DrfFo5~~$&?h@eMA*@`{ZvS>c)5$o zO^WLVOT>Z;*%14k0Ck9R!bMk|sTB4>lbBFNcK``20A4Q(u97&AD5?Y;&=d%c5MkpD5OtSQ~Z5dx4%pFYF0`7s=3`1E2R9(sk@)L9;OEakr*?}mzqGWRL0S(1u~3Vc4eZU@r!*nF6Dul3l}%jHT#XfBLbmYJEFqfVo^s;GLHy72ky{ z4vW$@>v4tMB3Rn{T=mIKEa6R;HHZz-k3Gd^QmC}!)&sH*lsxKEIGdP&P+y&e82p32M`98Rykf|yRiHj<)yD*U^pxrVN#n>w8vqI^KwtSvIIUr z9UMn)1wGbg2r{?pD}kzvxe)GBA*`>+J<$K4VcL_y?n_x0qND)wJb{yAI*qB9w(VjN zGsS;v6KB;!T1!51g~(o8fgZ?>)?J)zD|SEhQN4m57Zy_*YY|xHXl`E)M6{k@GU~py zSweVY4%Is7eX2m0b;)Irry{IP$9aNLo?@cBS!X_fpek(OLl=Yi_{4m;YAv4$_=^QA zH%2ksnD1VBqkx&-GCd(rVP#w5CE}X9-vi`GmEfu~Tb#w~ySCP89HE-QF&tWAb_^lO zk}hkj;8VvSUHtl-d+)b=SY2gzJR#V1d51bv{JLnJhk78Nq}`3Rwmeb*t{EU@H}1uO zZ08wK8=KJn))vSr1QWlmHQ$BcL^j+3Fy?ssr&?5geq+doZyzHjyYPjg@cry23I8X& z=hHe$tBh_lq)7n*hNK}j+^Qlv_O8~>6C{am=%QY(eVqquap{PsL?rUUD>F6{S#E&&rQEHU#)w(cPuA^OR?QVc%+VKF?L`ypYloGCdH?EQD3D% zfzk1ZIH77@{E&YNMsvf@R6^VuY+}%I@u9uu*#+=X-!ioH-QMM^bOcz2DN>|qb8jOI z9{e!X>+*x`F!8$Jo`#_ASSpuK@00fTJMJjmcoYLAj~S?BwSL(?PLxFf$Eg- z3NO2roYw`qB~4}TGJJ(z?O&Fo_cNEewavf)m+_@r>V8up??vz>0L4KkDvO~ZAG*ml z``4`sUBYv9uh-vqH6(7g#NDj*zGO7+e}@oYf;^?{zl)`Zw$1V9<`G2l`qSWz=%}>o z5eRQU$}{t1;sOn4pWWF20G|y8QAxIE52}eM(6p(Y@|6~% z#S2m%<+Gf{9FcpX+0vHww*DxHy?+%g=%?a0m;Kx4<9uAx(HU{bIglKp1}ZlZ#KiZ{ z*wOt*FtM+~I$EW1&ZYf#wMB(~RFJc<88!SSEU8(qXC75H&;yB;yqk)m%#Cb1=}YPM z(ljMS(e7+<=0fSm5ixd?j`b2AdP47V6~ptc!g$+b>%>5&TC#Ey=RlmW>dIN z$Sd6=YZg+au@|YCI>#ISp?YF+-O~)SrVNU9Mc=gM)Wf(ataI4h-Gw3`!ZQ2S+QSJpFrqi#Pg3hLbEL#s=#}X@lGqFh*NU z%{9`X)R0}>UF8GTca>yqjF{!N-sL+-FCUd=?M&|EWhXTMv>V9Oj15W|XsV#%$RB2m{$oSy zKj1%j+Jk{^3>2x_8>~_^nwXa_%Z`v%_0({?E$39PjHauFMvG16 z^p^wsA=%K)=O(mtxs|uvFqJ0r(!23NwF$UOYA1XdrU7WEDg=eg*zJ>LYLdu)SUo;TM7)8?#gpbr1FS28AX6YgrjF( zThSvCh@5l3*QFd~X*&`oXbA>4C)&EIQh0iPLxj$QqBMc^oJ5Z5+%dgTDAi-ZbQ)TVX0DC{zeHe-m;0sVK&Vs@oJ{mSr=VOsgfOgSKVxT-(J9#rdIq5Q!`iRu=2yvpOe|WBk|d}W@|eOeV+=Y z0_6NVVD@4({Y?yU-qQuKMs+!cW;EIOOl07>(|$;ukAlSxKoWlA`U=AAM^43)RcvGf zN-=(IVcQpEB=suX_$hmFrR?`s;0xa`$H8T%a9Z^5o5OcvqdOJ-2T!EU zzJutIN_K3|_M}W@qvH$Wc+i}n!-a!R6>fZfP=TWfis>}G_;2yO=)t7-5p}~&r>935 zpNCiT|MEK9O^3VDAZe>|=b;8D86a`HijU}8nK$I-S2w5UH~V~BRR$BeKZ*gY6fkr+*V+$>c%7a`e%e=4EHmfZ(2)c- zc9EVtIeyBa_dHA77#1g`D)#S^wyt3c9-FeLBT@FOceq`31#z|)Np<~Y7iioS&Vb)A z{{Y1~`>GFqv_-MHf)}R*SSrN^ZvB)Z{uSIB~um8pO9l#x4EgP;YXM4!1~$I zG&y!W*?j*Oe5mo334=)u*xv|)G3V~j1i z_y6QD%{s z)`A}eX^R@KydejS6HLwXM-?6#3u8TFNSjyb<@BLL%3mrOz~-J!(8tj(~DJF`wP4^7qYv{K2EYB73$2XDSP81O%n7&cQvf z)_wj=ZI#%{SI~BlN=l>&Z5@4M5BVw&{;vC^n#ISnGiD1R7?1aioK~Z`!u?Mt-T#W7 zllxg=YC%9o+%k)=z4Vxxzx12#DQAVy`3#m%7KFZ4kfaQ;$oH_wqKav zxiLiOe5w*EQItsoHhXLz@Wb}42$D&%FtlnRNPK$aW8@9aHrkVCUz|tAtM0T>k%+~1 z16NQK*MlEi;+T9>;Vd{tFt?#+&J4DBP#<4E(pzF8TWYkj0B?j87_QDn7CQAojb)}!(fCrODOu1}OvT!%Y2P02EQgM#R1Z-9dh zs|m~K_|T?%Rh%RRF)P{_>K6CZR7m<;U0F6yi; zE#eLHHD8_x-IT4)y9dRQ!K2A*=OUw4iCoX*w##SB&Ms5&BO86IzwQ6Myvvqi5n}pG zNvUv*RitP!Ohy>ZxbHUD1M{kX5Pzbs z(N-^hL+|M*gvPDkShm6IX0gcqR!m6t4IKH^5VV(&MZ358>`T?3f?s7^qI8bZ?G+T= zAFGwC2K}+ZC**vF9_cw)G;P*2Iv19n`~#0~u=sb?v{}P8R*p#uw6A#UCa#~sJw1bC zRT0Z(@%pu@W5DOK_1C&9R3~M#6G)i}9it-ik5Jx1d+Lk`)u7Nl#ZJc1Gd2N77pDvL zof$!i7^BiZXv5IJ_#wnEN>abThiYSDliGRO<+`6YgI+Y#yjsom57`^X;umia|6I(= zv!=l;2pY@!u_g{6DRn-!o0V=`eI|V;MZ~!%UjfHMFm9=*=z*wi6DDzuWj~bEy~As; z{Ky-lh*=Q=UVb&*|IuFR>p|xrczKO9rZt^lxVsT>DuByC#7Xpa-8Y%jyOuc7ZbwSO z=bxD$r*o?=&rrkknpA1PiXS+tT(WUtPV(+{B|5jNtM@S!?P1c*1nPM0DkGM?9PFIj z%48YB7IG1MW7P339SKUA#*@VZz-&G7E#uBwlnK29|Dw^zil+dMNetzQB&Saw+GZYmr%7ecuoXr!o zL}8{+t=gNdD_Bvd7q|u_N9Wnki*KNgOar9olrAoEf2Yc@84?26_dfXu@yp|f6WnHk zxyXC^K%__}GC*zO=*PNa7Im8c{Qi2gvOjlG3Fc`y*^~-SvtjU|C)rq+)WqsouDEyN z{Fqxo-2U94$iwx`^J+PTPc*2*{P+KOTp;%jDFg%`UPXsaWvBUkTI4UShwQ^VEg!OB zGa@J@@B4W5iCbzgxAcPxgDt|}+sQoH13v4$rSSaLaUPj|ToX%rUg&i5#oFgN+c$6Q zUESE3in}l%YyKSmlp?@?3;x@70}-UKzrscBMdF=W6}bWtOYX!CGD+b2gPt$M$lK12 zWtUR>q~FoJnHz{La-aNAjP8GH7^F;pb&yyx}=SgJ4U#IKx zw7**1vCWE7<5KcjaesS?d#Z*FJ4@afSkSyQSpO`z@AiCk=uX^jQViF$<-6e6y`?)1 zRhG5hboDw_wF@7n14|N|&XKeBQ-`anUQ88lVYGhiY|z^_23$FM*ZK`QkUfTUkdMkZ zAS5e!5-TitIEED*R+Kr}vf8r`Q;pSk=XxSE{9;s}=f1q|_hbTocSeJIY(p(+#vRs9 zEaaO~E;iQoIUmr|VB}XUUqs|8@k||Pm6w~UK=_{uZb*9h*cjEi2sb5R{HH_uXF|Bh zUJyOo>2Hw#wqBQKSO0d15VK@8co(ON8SfOa@w+E6hFVNnlTrDaG}czM;?QeQKI2v~ z=JR|!x%K2qw4$AY#&o}yD`Qw^9*zI5_F-*!*+b0Ah|}R;-vN&BdNIqUT3zwrmx{~9 z_5U9!4;jm4ZxIqdqqjQUK=lx-+e}HAd4A%vf<2*$tto+b$+r@%;l&ydrQwNw z#4F-v7duKZv!wcFeeaPKS1k&+|Ac4qFe>210Xdtgl(2aRD@;3*$js7mG}h`vG+fn4 zs|hUp^C-Q!YxIaa#ey8JLtQAoDq6=N80Ax0HX}ElY&PUOXH*{jSi#`Qhlhf1cWa6j z3YaGo`)bcXJ|+u2Flqbj3U0b?nD>liQg;Z^`L#qgLml2aE1#0~8J2RxfX_qPCp={M z`>FSG`dFI}HCqUgVYOW|ccIWqN zv!R_gK_)EJnL>m?;i9ol5Nl#DQzFit4yrq$sG5AKcgNDB(}N#X=bw4DM>EecXw8Fw z@@!`dI?^`_As5Q%zx$Nc4|Ytk7iRE(5SauvyuDH{9w|TbWo@=_{V7>Bbbz;I#8~IU zqMXyRubx?m(h3|d&o(6g5Nv6LqR<0-@?Mm>dRM{)M?Heh_UM=ROGjc}$2MhrnXE#t z8tIis4$|Glfdtx5h5OmGpd=-L;{*mUg8?>{zg<1OuT(8@#RhDU5}PuL@-Q}nqE2pp zJ=Ih&EITElapg0;qG~#Ih8@at)*SWOt#CRyeFrbLClS|_fyiQTBbHqb+882!6}j^7 z)X@nO7Bk&5;1MSX)%SU*jjwRhFTWoUZ(Bz3A%eZst<{nJtNK;Sk4Vxe z5tlrhgrPt>;!mPt0N9BRx`N?3p?|jK6)zm9IQunlk7eBaG&4v^q$ z=Ck9F;rZxp+Pp-ZEsc-EH8$DFJ)MTcK7+8PF@|R_q-TE)xwOwCYDGmC{|#m zMC|$>ION2gP;3p=P0T(;dp}ir<8w2YWL=GS#sP;erS9t6rc*8v-hRj(rMbYmMGy83 zd~u^->ZEO7^Kp29AmHM=r*DKX!nyv*Rg+pmy*Nk-bVkrIHBV*=hF_@}UpOKAaej9r zdEN0jb+(({(EquK>hJVf1#p>Riz{~@c zffgLva->mz5m@lprduoZr3slP4<)+(4X>%Qo_v>j#ubwcZgC56oN8p3`NoN3mDEV3 z^T^FUR|p5mspq((k-D_Es@2;17d{TJSQclupJr=%aINGgJ1}1^k!ZHGrd8ekT38B& zyw^a5_0J#L8H(7X8Zkjr?QMHW-?JvJmWl6*qOymopKI6;^GIaJXDcM4Hg-JUjF>y^ zBPOmNN3v|lIeCer5yh$HL#it0hbYhK&s3+28N;xL}py z(ti&GyL1q}a~E)|vLN;pdOLlRw9F%a4z+dj-8&mi<_D)v{%!$}wx(`cWMb$4STXvb(4HeWmSy2` z7##T^wu0BpqSVi2t)7~xi5$B}yijZ1ubL`AYwu_)s47@g721^3OtZhnK_cHNt9o5f z1bGFBtfVDv2!&WjjwI>K;2;O&y5YeYGYi3R0rX0Zxzr8rwP6vM)7)pTRyz;sS0J30 zc5TFFANXZGsjA#j{17QIJk`;Z7#61dY~PwC9sC?zze|XDAW-6gv&(r8J?I2Fiv0LM`B~AOf{U7$9 zPNGf{cUpq#HEh70iEuQV7|LiQ(m*(bQ9D29Z=>IJdWUxgPJh2{y*O$n7KJ)tnXxf7 z&g63SZ~LC}mukZpWsbrvo2Sd*BUkb!TMrcexcT*u%!4*)af>w0cpD+qq*y`@JJMUaYp|b&)h8p5d=`Fj6{|Dh4cZ&c3 literal 0 HcmV?d00001 diff --git a/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc b/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc index 38fa227863e..aa379f3e849 100644 --- a/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc +++ b/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc @@ -1,6 +1,10 @@ = Apache Solr Reference Guide: For Solr {solr-docs-version} :toc: :toc-title: Table of Contents +:toclevels: 2 +:author: Written by the Apache Lucene/Solr Project +:email: https://lucene.apache.org/solr +:revdate: Published {build-date} // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information diff --git a/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml b/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml index 843dfc3bc90..0c10425caa2 100644 --- a/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml +++ b/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml @@ -92,8 +92,8 @@ title_page: align: right logo: top: 10% - image: image:../../images/solr-sunOnly-small.png[pdfwidth=10%] - align: right + image: image:../../images/Solr_Logo_on_white.png[pdfwidth=50%] + align: left title: top: 55% font_size: $heading_h1_font_size @@ -105,8 +105,7 @@ title_page: line_height: 1 authors: margin_top: $base_font_size * 1.25 - font_size: $base_font_size_large - font_color: '#181818' + font_size: $base_font_size revision: margin_top: $base_font_size * 1.25 block: From d7ff0e2b70b2f46cc8fd845b3b732d967fcfe830 Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Fri, 9 Jun 2017 04:47:59 +0530 Subject: [PATCH 033/131] Add 6.6.0 back compat test indexes --- .../index/TestBackwardsCompatibility.java | 4 +++- .../org/apache/lucene/index/index.6.6.0-cfs.zip | Bin 0 -> 15875 bytes .../apache/lucene/index/index.6.6.0-nocfs.zip | Bin 0 -> 15883 bytes 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.0-cfs.zip create mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.0-nocfs.zip diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index a876b7de81f..71e82aeb8d4 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -297,7 +297,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase { "6.5.0-cfs", "6.5.0-nocfs", "6.5.1-cfs", - "6.5.1-nocfs" + "6.5.1-nocfs", + "6.6.0-cfs", + "6.6.0-nocfs" }; final String[] unsupportedNames = { diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.0-cfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.0-cfs.zip new file mode 100644 index 0000000000000000000000000000000000000000..36e0b3da7e093740f2b8008901e60e782a5a29ff GIT binary patch literal 15875 zcmdUWWmH`2wq@Z?a3{D1ch}%nxLa^{4-h<9aCdiicXxLP?j9gmXmakk-O1^Gr_USX z{&^I&M-^i*_g-JI7ju4dE}#?`I2r%|fCfa94M`qU^<(qAz7ziACwKr|23 zB^Ush<}#=H&!>|s8~_CT01N>5<51I)PD1xpF*!9k!QA|7jC%Z0bcWU_06e6ftGr~L z(O9Saq3t|W)s8ZU2n>s1fnYN6$rl86-jy-~3;_J?>}0x|(%)Xsya56LKmvZAozB46 z=>L1}vwxpEHA6E}H#uP|Sydr6N#p-@34Q=?9%;8$A00_ACNS^=LCSt_$}6&@CL}v@ zeCV<6Fk;788uB^yqdJO)V8+LRwL0wkk+!se!b^8n!(D=a+M}hRpdejZuAp|)mF|o9;bM*cKuLzc)o=qJ|I1(Ypc?wa`hhP9i7jA-2iQ81C$vl;mIV}E z+{Gx&%Eu0@h#c$+K&l_y6Wf$4Z)r$+L%Z%!A$4ZP|F8v%@XO=>;g#M|@l6#?skYJ- z;d|`x^`YaIW^e|tP}*YWYW3kJ3&Qp;N_)%={Y#^BQgyxh+qD!DeQDe*3wmypi9J!p zWIo}syU#==A(r`2LwI8`H&$dl5 z2U6W7+n?#l-nf%wK!tUaW^4Q8Rl=_cZpoWqg=SO2fXKfa6j_%4I-~bZC%H9y$TF6^ zf^x)E9>*b@g)mE6wFWtm*c9d@6+MsYgk${+&dPo>&{ZHXTy;{&BdncmO7)HB&Gdrt z!%srKfS{m3hx<3o6>|H6=vrHgi4f1sBDX4-+DQxfvSL zzc=uXRe=XDN`zKmM3USQ-I%Pr-*92E@0n4ecDX~Kifajaj2&JR?g{a3_;kgDPl|1~ zKBy-o!A&1qYm}8MqZVRi*5la{pxqMCHG<2v!F&I}cU6JVCKVY?A5Tc8OCu=RuI3wS zdGeJ2i76e^*cj6L@kkx^FjL8{pw;uN1)l`wLDGfAkv-A@ZGqM~EJm-3O|oa@7a$Yt+1Nqbsu zkU)oQ%|OM>*`;GNGDmInc%IW>h~?P~Zm=8aa%HEepEi?}Fn+i#@~kr|?o@a`^Sj1V zft#Qxr46BU^;wmTaQ_E_Y%&EeE@N>AsPYNK5wX;Pg17bt>jW98^EE7^rbbMaaTo4y zkIThegVD*Wb(Luk!PsRu2AB7`;yVfb^dU*$%k>$C^$K9hW=7eBbhE8e%0`KC%D_@S z@0XoTDf5(G*?Q0P@M<0p@U9%_KL-chNDm4-_6s|fLoR$b<51n!|4|ckFT5oDEi33? zJ9B`H08?9NL}1?Iv(FE+RaRl5Z{Arnsb)@&p7W=tsvu>{mm2v=Tmg1tz+eUur z09H3=&8C`igMzMRe5j0UKf_3LclewH)GtXykz%9~{-ZP{ALSoj;?8C7OROf+rn&L& zTHZ9?jl3NkURP8a7>iK4j%3e6Z-l5(5$FdgWH42t{|phB&OR;_;9DgEw}9-P`YpK@ zb8tOHcn`+Y^p1fQw4{|NiLdq%^pGOqG*OXPjH{{DjK3`abi!vaMvAnMcvc)pAq?~Z za#rog527k4e?-)>ALAq=_5n*@^%jX7tWyFhARQcOb*fSQ{JFrk2%5Yf?R^LEsuPaz zGod75lx8`o`c zIQ<#Dh@7JL&G5m!xI#^|dU3whwlm=ZM=5oqu?WB2_iar-+G(bd#Ca1tJ5Oc`=-fm!%o7OVz?t9QxChI;1v!DRZ<&lRtO4> zPbO%YZ&fqRy84|K8)aMehTK!UlNeEy2SZ%?A-q^v&t5nY%4+z?XUZ&%)K}T^P2fe} znIvCc`PhLCs(hgli%)w?AU7_+-YpS6dbF1CuD8HpTk~_rS-WYv?fV6)G$Uv|Fe)G5~J8C!I;Id1gR2$R{PLKsF?kC`OA=La!(;sKj-*koTincnQ= zqXQUCcl#vQtFQ0JD~sxfYX%&VOOA?jB<2wj3JTsxAKlEUFGM?=8ZzNeK$MW^Od>{^ zP6P2b4F_gSQHsAfGF=;7_M$e=tJNiKa0}kPZ{*qnd*52MeS5@dpS53tS}cwJMo#W9 z$?yYuQ;D)*xFnr@zm;`PJq{-yGQj4pxAt=l)HUn2qdarDF?&J|eSTfzJ!p)fAY;7> zW{TEHh?#|!mVKGArF{Rw-kyVVe{CIEDpYilt8u(teGAmlNtl`Zop_m|s14M`9Cwf9 z8TiM$JnlEus4vTxO~=Ii4mS%g5Wkb^sRt<@npZ;Y`uc?U7pZnI`#XNf3mlW|5kQiS zeUDS-8}fE81*+v_&txSXoE0K38;Ms^EhaqN&}GD0A*5nc@j$rn{A-u$#m-O)W(@gh zC&TF7*xeYzfbP=SeC21#fwhOPw#}*;uDS8)C2M~ zrG&71Vlijt#F&WmgxqL$_Tot8MepPS@Zcn_lrHK5@H)Q3ApWqYMLZ)bpB?mhC`< zIu93-d`YPrWIZ@~txN0`IQ6&L>>nE=@KCRp|FsyojxBM6$<7+Ybvdp_GG_n(u=Ip z#=h`rToObaKdhsfB!oW+<{Kv^=;SP*)GnK_sj*08J_@uL@{vMdMy9QQa;_r=^+Ckm z6X@s4YWP|AV=$Z;U4c0!XbN|2sJ{r8vD_eXR#*VQ?F_=IY@v(Dkd$+a^_#S}6;HVD&ZJNBqh!$v6^qJ*ar=;E zgmp+s`C-eFRf;983bGZvGp93EfDo***AIda$jJP~@#81sXQ%7s;Svt~Tpl$WCwHky zHhU{cOCDbgZbSoeA#mN#MW_oqJJoSCFM5t;J{rn-MI} zGXTKrlO4lzf(_RgAq}dNWwNuxd)X0z5GtKtw2hueYM95-D`zm4EGnv7v}SPU)3FRY4(mxomv*_Tjz5z`0ihj< z&Cw{b1*?!H5TsRLF2`{Q@mudlmrQ1^4tm4)BYv=yt-AhLEGn($@Q{qob)?f&s_sK3 zt1yH$XllK3&F4UY3TL0R6~&ILnNExz@lvPGWll;X{LWzubu5t%8s6nh2%Z7yBJS=c zJ_nh_iB!6BzifpA-A#k!$dft}fgTSpU_3SM634GN)kxkP0G=yduZBeko4XZ_!w^g( znha&TE@J#l1y}~*kP0LNH~`>51Og9006!2rIRNYcwXageV`i-OqW~mOJ&aErKw;vA zhL#K@H-9${x!EyP&ujlyfdj0b*d=l?v?+LoFt<%=g0M$@Pe3_hcRrG|zrF={d3g6t zoOv)fIj5+7?9L5V&dg-CYMRI-oaf{lFa~I026z$%*s)|ii5;R{5>82_r-6LmqAs0- z8^H{M2QUUC|1Ko>0DTTn5_zn$Qkg2qu@wJ1IVK0eU8wDy-tln z<$iOJIT4%%Y)Q-Y_Abjf*fY2zgihy9 z?H>xsGv_09qJzZw1gZ|^}d$5Xf}&TO}z5!&6;NB{hwU`bWq-Fsncj8YF48%5-+ z+o~g&Oo?bCQVFZPS7cm{lPoenL)A{bJap`IxmqD=l!ddr*7s2qdlW-^?jm12jz4zf zt-Y#nixl4~yRy>Y2o_FrieuIfE%y!T(X zQEVU)`{r(+9PtZv*HM^7!$(&=77BFXIeGfQAPE*2RWZ(_OP1vK#W`u;o+4wtq@!|_ zkMG+^3}t(ZbJnG1OPH-I6iSKUQjYwo$9j8E4J7$ubRa$fA(#_SHfO`N6w1k@aMwUT zwF*iM*)c<0w(D~hCMl&U55Jchx>4agr^E(Imv#}8$4gMT)rm-c8#zKVvJY3`WM(8| zR~COeYMA$g7GqhjGAk`T1Y;p{qb#Zlw^Py(xHJtrg;}xI8Yxqk+n=ynGZR<0ZI7|= zLnutwM@Dru-21C)2OSHIZLntH*+BUNN#(tfAsfSk*-XqxszS8>zG0W6WH1V?DFS`t zYBrRI=MMn}v@h=sSWC>eoo*0Ha8r_r3JT)l0YR1vFAeeNn#}bEcRCtH2sL+o25@qP zkGbtrS`Ss8ZX#Lx!vQ~}lTZ8s4Ph%qN`BXg^2)15v$2^4P@wYsLU3vmlZ|~#io}r1 zWL`N?CBfIW(WFH1joZM>yGba|mHrt&wpgl*Z-wCnCRTiSGw6C>GkluVP>m1MUsbD5 zhaYd%+fNq59UjwPf53#u*QC=4@;+`KZe~!F5WEzDiW*!JKLE_;N)H8xoMWZ44PnW+ zfz_yFEkJuXN#~p$mDtt5k@SlDw@b{zH<{^Bptd0FNInIdG6!4&a-~&}tvDBEofSTx zAboM5%?L|px6xJi0Np2;^|S#-i0=b2x$Pz15&E2$XRvvNtJW6W>K!LrE>c9fot0JM$tvZW zPY1OEQ;vy~qb-xl`UhSJS?;ZU0d$Qpfy@kLbdFQP43nMEJnlnMb=5w?c1SrQ1pPpK z+!ZaaZ~B=C$6opCc8^mZ{4+;AE9x`0r|uhzzL;-0C-!u~;fr~j@pT@usNhTCzd2TH zzMgV!G~UlDfDT3Uaqd0Llg%H`lg%3kKhkG_RxFS=@znY8qI{;aTe6n!vD<)m%J25P zW50ou7SSesxwuLD0k0>jEG24~1Ekx?OIBl;ZTP{J1V$SVEsN68E1i4c>)z2j`f5?| zdhj;rtzozTH?-o`@AQ{X{lg`tg{$567Ma2t?fhf1A0Hbooo0i>n>BAShTue07D56* zihy{d!?4fa_b>7=r}dl(hQ5-F>}MmGY#g$@3A+l@Ia8ZwDe>F&-7XZI)_;{>TW81*4CHpw*4+Zn9rJH&P#dd^U&B?=Tou_dK0e){Orn)L+X7>a* zc13h3HmNFPzKf)+F~^$UD_r!hEM#P(8r~?xSPyxt#!4DqW0D*Pj$Fuwi?0t5G1-lk z1b~Z2-dPIiO7*7r+^JAjeJRI4O%n2bJzByfWmepRd!GA!)J%J*kMQH3R3jxnNVdZ{ z1x)QFu^~NDGbXgc?N%3R7rcbIcAb|&FWnj!t3tZBDlcTF6(YxAzLI@Meqz^*?ka3L7HzIrW#jk0 zhzOW^0sWn!_r%qb(ETcg80r3A4Ei-mq662tNo_ZY$ zdliZz#dLk`nmH}fCxBRxl^&2T*qRkZ#Q&ZHlf1{suowz$UWSQfZv=!Jmb}M2-z49p zGhMedK`l15D>*UcgE2AaytY!ZYIG8u>L46Do*}HJotK%sJ+*~~teG7oCXihzE-^Sg zD=WRKJj%yMM}ui{9u}^~l^InwQlgJ6xCx>)kUZWv-nid*FMrS2c+~PT(0$0P4~aSs zT^!wnX;`>>Y?iQ*w1bkhkCvI0B5#qlgOa`KJv}==S=`7FD+ehbN9WvJz~-m^9n48R zgGBbV3$nj3e0ZTLs*7Lc5YvBH4sraHLrP5&Km8na*njysX*BL;RAj($hr%7XpV|l( zI^_^VX_85DT=1k1_-E~5Bq3MuiS79SFX ze3T%cRVa6SwS*$bbpgox@BEG$jHgEY?vuPpW)&^I+G3>(!FG8wsZr%UGt^+>bYbG|Ak2J|Nna4mP5S?|a?q?V_d))$UhYoW zO4P54$ok)k=;wO&ZyDa%Q-wg-`e!S;Jx{* zJds|xr&%faZDkr=(f1x4?K+|?rvP&iBT9VDIcGuzmT<(}XviWoq&aG8N|EGh_8!hLa{Ofn^=g+NT-u&}BOMFqW%=TiAiq20%C-2G8 z<DTV>9|^Z=bd-jr24Q_Z?i5yqx}qbhwP+Pv$vNP zU@0q$kh@!@{9vX|nqJWF#ToF5J{BV->AvLmb?4w^#7 z5y%QH!@>h!_V^dtMG)>R0^f`@U25B-j842UR=CP7n*s;Js1p`_vB&Ad=V8Ij-2#mV zsT8IoRd68yulu~J9~^uVmD}!Ix46)yN8zUp@65c|^VJ=aK;IV3uZ?;q=^z}5-e{Rk zvh?fJXGn`m+1_i7o6Lv-KDawonpMR)=T*gUjCoXy`56R^>=}&g8s*$!*`6h8xNCx1 zuC*U7@8PDnF%Sr5)wQmCa)DN<#-7e0UD%Klj2t6h$sjjAT;TX!!>?1a9>yD*jM35u zwe?yk27s;G<6iXJ3twiAd~vE=?>4t1Ot(jEz`u@N>HgTAeaBK(#)=pTrd;tD4o1oq zJBB%()OJ7MaOj)r$1Yj72ng<`&Z3Z&6PJ50N0$2O62+UFiVUC3tu@^r5iwK^ZJB>O z29C2y@BWScTXhs?RGPRKabCHjVeh8`S1+gg3erMD{UIF7nD{ff98?$Yt9$38-ir?< zeQUfax_}O@ zB2q)0yYWb3G#xKCR3r2GVts81@g+S6o&zUO^o4i04+GUho;{rE`Ny_=>9QE6;Pji$ z!|2r!*do2!+4-Nf+pvO{&vK1NYh5<=_K79MpM!Oe;$nN7I2Of_VC;e_wbrC`Q!Sx- zW~>k-_T8GxHV>WiC9h0DNtHaYOnBds#-9bxU2{J%uO1(YL^pivpnw@aM|z`te1sHR zhA!u#NayL0$8gXu3AZ!x-YUNV)7hzVhPVnVxQwLA8ReL;N*3jqxXK#kI7%}8&MPga zCxs9(9st6rr%6eydcb<(DwG=<0^3S`QIshG;e`&agrp=W4lzaz4VUAXzLViUl$O$4 z*qn=%eAqA-PD)=#2y|Cv{wrgRr`1+4~E1K zE<6`k60D0Dr2r<(> zZ90xhLN%R$!p}2;%IP>O7SY&JBF<+AP)l@6T7Kgw9`azdirzCh!Z%@6HwDC2oufZ; zrl;QFE*H`%q=e0M*%5j9BA$p|J7&_pb4hvA=#$nXKu(xwxlUf|F;dwrp?TuOU3#63 zKku}JcZ<8U^ScaA2H~b- z^@LIn8YznDKG|c@dH(v@r?_girPKfn-j}Ek;2$PE6|~;rg^0I2M7;H4y_vLj015Aq z-}zF&Z|8XtPO@E*2KgO*d{4Tv-LPPDMT8f4QB<&bW2syv-vs)nIeo6wmp z&*LPR806B+QhmWqb&s9l$V`145Nxtsp{%N8)`BK(64Xznu(bx(4DOTLk03MDJ>&Y5 z8D;*o0t>#hglEc+F_beb+Us?Fp3KFbu$&ftkp;j;wh;ei%-HRfyFfIvJ*gM|hDEsWxdjzR6crNBc#K5^Hss*=3&qgd`7y3^|{Q&sf>`!B)9wT zYb96UrX+;9)1xmUQx&o5o;LxDF_b(!*#P;JGYpauxv;QY3OlqYM_AO4FT1e8!QJBQ zUFfDva%2v1ku=rmUoO$35Q$oCZwCyhvQVbe0<<*Vj-uhky-N51n+fAvu6IOhQOg_r z%8~j}oy{lcI5A!;rtexk<+-%o+o*X7f;f!~33;gsvY40LrB4a4M#y2E;*jZcZ+k2ibg?%Xwhk?)f!K$W*TTU&e4KM?Rp}d?um;_V*q7Ew z6=98O_FZsLC&SXp#UTUa{kvLRhkk3G5VPWk()yHe5p`4#hq5!D7%CQe6}$`(HcLcJ zfz0Kfl16}-QV?Sw&lsU8`!SfRg7b>iikEuHR0?8=hQD_k^BEY?)MbY`$Z^FC$}o;& zI0%QwRVnqZDp(~MA`>AC8zApL4ssxBiXlKClmM_4JfH;#Lu-RwxcTpa@c2w30CL3J z;0`x1pyGI|fJwZP%i)btP;NL=@Jcm)LMlXG3M}VSe(3H&4tZnk9Q9(C!zQ6HEAq@&YaV2s%q{!IB_k6`hWNkQNHXLf-Ky zxi7M#!HM`Ut3~Biorm_CXZ!f}&pg}D)%sfzRHyD~i6@4hk+jyd(o*C3gJWRB#RC(i zi6|iSl!cb`vd^xRBjbzZDkSOXp0ibj`9?sC(}$dp)MTK@&Km_)S_!F@!eB6J?^=;; zB&6VDegHapog_49aIt-W1`K)>gnL3BZ{WXXWEdWDu-Pvh#}PK!&$6^;(w%R`Z>G9p z4HilnJ%cyAPEN)i;ZIg%B@;_uRRWhej3Uhq$O$+UuC2rfocy7UtPf2-Fyq@`5Bvmr zX5ZmJ>s7A)0vJqL3nT*fdw`m|DAahl(A~M0c?gnVOc_O~m6TUKZExgFv($K{?xQ&E zmG2&(i=ZmxjS$E^#D`=M-E#N7{fzCw+IWo7)op zJk~kfZ>Q+a8T|O?QZt%tIx!qvE0pKx?r8fv>FCL*P5QK2IWa)2y=p)m&Ec$cQG`MLepq1Iux~C7{mH}&Jv^3dMeAQEeu-y zx_Akg6%YIGC2w?dd5h*eu7J@Z4L?Q;Jy<}Fp994P;~_>w1+Bm&zkrNcEsFJ%nCape z#p#`u*nhY$nv!?IdHp*x5*OO{TnNphvihbE&U?Tl$LgSCiUBX@a*=LIaqMy#t}8vb z2zTm5TR{DcJ+|Lx39EGPV`)5k9^jnuG228`QGC`v{;Uw@Xe>V{FPN%N+f7xSQ_3T} zoOeSCbl5b5t85VDYUZm?R^{4cWh|V#LH;}gbK;oUH`Di!XvA4vU{Z64Qiku**B65m zooiEbzl2*@O2@tRn5YyHRo)FJ6XcspN#kP6@^4dL@VbA$@m^)RxJ^D{7nL-5)-@;f z#1%)zylqOBho3NyKy`;}f`^|&_V#@*?hR3P`e~z}ND0)ZJ94E%azq&!YEIU+FL=7V z45&)X%L4JM^@9jIrUn{>71S67!q&A8NV<%Ym~}qp_Su%Lh)qkua59k)Mx7dX4Eh0c zL=9X(p}f)hLLUoqiP7QyaeB<2JFrP3^ZgUgo!R(rXJt_l>)YfWsosJEY#2^%; zHSg0~zY~MyiS<0FCDCibkw6Q9gQ4Xc4qQu)eT$C}qWg9$7Fx&Njqx3Xmu|D)5I1;G z9#gglf8^NcuB)kdFWtG6b_jzt0@j730C-Se={S?YD)m~?qx~MSZ#7&oDz(5CYB~~Z zUH#p@V}Q^g^@8ftBk}cD_5cTwDcc0ltRuS8h^71VEMsI6 zp|F~VbojvYq(X6L$)2+v#a_zhhMqGu^=a(89EII$yUWzl^qb?J676Tc;U3`AbLsjtgDNJ8FEpTcza2S4 ze=SDy^5oZ0);OmkC3pvsR^)B)*xz(f(phlpvPSn#AYp~oI62>f6lS5|^CvH;2lwnQ znOw@F(>jj3etP2FCZ2)2XYU1_#pXSVf|QgFM^>Tt{Wo4Tgac~O_XZ3}aXWHmK^ ztVo`RPSEVrMw{C@v&jE|V1ucr2LBz@s)qUJ!$TqTeAF?nPd?tlnf6fIz^D5zGdu0K&i)DFBdTeNL$n0$0oGz-e7*S z=6dgpVd&D08$q#|2l@c>;(i33n!s}-%0RA|LI=2`6M?j}!Wy`mL-;lQVB6%!9SgLL z22(nwVpYaAlpE@DRTv&wMfE2;l-WkJE!+HngNkY@V!NU}Civh zlEdZW#(E~a@a*5M9_#;x791Kv18p>CUr4Ehkuw&Ou~}R23*_jFHXjJBiYNn%mW<~C z=A{v#IOd^xE$(dJ zc!S@d{s-$3S_XpS%m!Ka<7NmiPC|L~@%JZ97Ty^(X-9ZPFLgiG4zpj*K6b9mDUYpw zr&U(nhnLIGuiHXhVCQ(>%Rp#%?pY5RTKK*A-Az62ocCHdj+$*wV7fz&O5;(PmSc{} zM6>*{qs0nhzKIuL|7*pj#|6oET?6cHu9jn+m5W}{$Mel$M*X_l#?wM1ouL#gYJx5f zqf}dkHPJMtV_zLyinq+(>-be|c-W-OId545Zk5V%@!i%9&(#NdoX8Gx$PS_;*}sWv z!?5UrrWMg5CsQGFDf>{b(xetsg76zWtLypF!;%SMO3}1xX>T1RFvcv`T|$4j!c7L&h;8e)UP|TTrNmS!Cq%ge`GZsmKCp6; z?wexyc1B!q+Q#?xZHTLVIq!S#-|0RD{KBW-=sxp*knW2C{zdoIu>Xg-9vFc?>3;v8 zbiWKsDDS`RdVIGx@uB<+fel!q8jpLW`*8mrVqy6Y(|t5BsGoEn<`25B_LJ^wQ5N7z ziUE}Oy1)s94>O)F8CJFszDY8{5q5{UWM0(~#wZY4=8G3*vJQ^J7Jfx5)ME&dpt>Id z-tPnNd00|1>GTl;h6%Vc8Sf!t$832yRk8n4)O2YV!NhAp@4)|L z$((4l=)wPcG6>C&7gz@`?PM$AC2t<0X>2YlS*7{G4n_%+B(OXXQj{E$oID+jIWZ9x z6&3T7glPZISQ&#rR|{&Am|$S25CZ${2>meatqiS%bk{D1QQ|rdI>#|T(gjJ%N~_)) zYGjiTlEWB$=lIR2)ChgQIVGx?y_b%JxlV8u6BtR4H>F-Qf;cVcG5jDs-O#8My)zgW z{j1f5AZ^zv1fEd6HC z8aoS{$->+`(WF=3i~FgTu>~Rs5Ld;-z{I>^04hoimP{#ysW!WcX|Bsuy<4*tzwl`m zF1wd}Tq&Sb`CQ)uUc7{+TnX|T#F)w819;OQ>}TxOsPK<-4|{}z;u48bl^1aI!!#1K z_Cs?0d(^oAeFdPQg0a4l!T#5st-a0NZA>II6bxi^)P%^`==i9(7_s`XPZW=^9DP}k zty0GJFvy#9;cMY~B+!l3K2b@i(N&3CSdod+!(q~QiP+fKg2aTS-Gj4zzP&+Qt(Z%g z1M7@0qI-AgPuG7P)cWEJ3-(*4l;!`^Q2w1MomG0xly->S>~+yCJ(V6x02P7Mu$Mbz zr76E;N@s(RHoE*Oz7EB()04|A`{*a0r<|wgkGEGpp9%_677QXDqUP{)Z2gfbZRUho z_j=8g?w&v2Y4FA-&YM@mYO^Q!GeJ_(lUq!ZthSd-)0Tc~Pqgau=2f=6d z30MarvsF1WjFR?9`2A18iOKtdt^@%tg0@ATcpYBsHZv;uGi!h8-7wzSMb5A77q5DN0jH*lQ2>M=sm7N9_?c?` z6S9mIWssbiy}3?t60#C63G|B?m>zb%tFYsmzU|r|wcP=m&;q_}h_!*^8glLdo8SVz zNUL?=&s(w6cc#wh2-zN-zMf3Ut?O(0>pJDwzdG{+;_3LN#$L!*3 zhx}()zlXGc#hQWp1M6o>;h#bM9zy;ViVxux>d#2>Kg0Sxqx&nC1>&Dr|G328qZhwo zeSLlNdR_m##NUGjze362yh8mMGx%pE{%*Seisgy>C)PhM@%L@jU$OSd{=oX@CH}r? z_$!nW#VgdGTZsRx#NW5ne#P>o{1fXRm-suC{1t1S@eizjUgGaW=~pOW=2xgcY12O| z@ppdmD;5^ZpIHC6#NYq9{fb4w_6OEKFY)*0{VP;7JK(og{&!{mS1t8dJZTQVZw>ap X!h?YP*)~yLAKDNAK;-Lh0RaCCrftb~ literal 0 HcmV?d00001 diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.0-nocfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.0-nocfs.zip new file mode 100644 index 0000000000000000000000000000000000000000..90ced65808643b501556b5dea20ba2fe56b14c34 GIT binary patch literal 15883 zcmdUWWmKHowq@b&PH=aZ;4Z=4-Q6KraEIXT?!n#N-GaLZNpPphx#xB#r~93EdW`$$ zQPdt)H3oBkYt=UvbM7^loFoV+DgXe01SFCUOSoBLe&v39C-lcBH~?J+T}C=9a~EYL zC;*7jGKc!lr?V>z00{ID1OWJBsqM@prrT6ZO;1g-u-J@KPdtvz(i#JR^3kQ!KMhyu zDSx=Fr~Z+%QichK)#cOt;xn*|0V`+cZ0myxko#?S7f6mTS8qFS0098tfSVW|K}xu-tFj*?7&?qT*P4!@FNH^z6iz8503?gFdy-o zp_`!I`#l+iyiem*dX-0SZ>vFt41R{0v$I5RuZxFafQgFm zwYCANuvRg?MKOJ{IHXO4fDvw2KwE7U>f^^@C7zKH{7-jzk(cg0c? znvRQ@ss%U$)pk%Z)U8VLWj*$-L6#@2yJu1-rF=3_BKc_0V0(EIv+pSde8uI9m5}8r zc@>~UbS_vo8bqV&`qT2~=^cyu&oN0dq0t_9Yp zSiyGAY8awsFu%*c1S=kS+Wx$2DLYu-Qj(%MiTfm?)p@-dwI9d%@ifIvU1VSOW!+?+ zBE6^IfU%tD4Dql&&0Hv~C{~>(W$*TzIW9TVlM~gK%P5kXN6WcvQht$oF+#KxIrV1Z zg-ud5pYiT-aciolb8){bN1S|%1TTH299eTuCblos!%7#D)l%Fs4477i^OkqSR`9jv zv{yauAzk2y{$7iaAXQM{^M0LaUSouyki_sm#QBM0qa)%QRQXqlXOGx{EcH?B$Tp=3 zlKEMDjewUx!>Y21;t}+FSYe7AvXf!yc1zxsSgh18Vt`wIA{4{LaVCd5D87tr*(?r^ zY}qXyjcg&h4Z?DstIEPB7NWl%RM_L>$esM!;=&)&9VrTX!S9yYB0ZoCC= z{(|ZHvr_mZ7~}}YWzaQFR2DRcsF}=NHZ!PG-vYa;I3T~iPvr)ZKb;3U;8dXt3V`;a zqX5kbec{>)`HV9}0P`v9V7&f=mgNxwP?Q#!C+6G?zRhNJ=hdKh%)vS3Ga4uw4UHNI zUcrSYXGYWGTx6#rIU7TxRTG~`uWNI_xe*W1>%m5R{o2K$YhnPSbwEY?m^e)wuC&Di z4eJJJP&rjiN{0mA71Dd-^~EXs^bso>8S2FRf(vMNCSHLgC?aM%FhyKawXJ;en;Hz2 zmd!xr^~;_IovWS&t;?Q;_M zN7`%Bxsnyv)`#{0c?6+m82XG9LX}yrb65jC6jFlxi_i`^GP{pOd~HEwm3BlZg~QZi zOd81p=?rk<6l=2v0Xc{opj@Vb=FnUbjX47oc{~(NB0<*}(}Jb}9c)vo_@Az4zluI! zXIDyJvAQC)Got6^=9u;4JU~k*)(RyqIdz}R?DXA@DZo@k(RIm^6g5dkUCX}jvnCOs zjTti$9jXX1{&;NM5V-slgwq0qtD1lF@jjsv+WOpKOJwz;ZqXf-Q$&d<+h!TenT~k%}S?@7h&Mc@BS|NhsN7i~lB=*j7^!r;i`8iVHVGBoK5Sr1 zufj0xU`Q9N-BL3K8hfP+m?0M$%zqbLDp~dh&tu{7=yOC z&3zDyR@#urP1nrCX3JUwEhN4<4@ zdiF)kH3p5$K~9@Of` zAqQX1rqQ^|?d4yQXwOw@!4x}UR$)!Q&WplDNU4UX;<9f5ez{dLRn#67RB2Gc69g!2XX50OZIaT>^ z{mMv&GcFgL%3ABWJ*POhdd?t&JI7Ws7OvOZQGoK&$V*AwpPi190@LVk)cWp0#7NC{0^Ehqs0kE#)d0#!$4XK^zk&!1UZ`CoixJq6=6P=hYMp zIcTkb6Q@UxCz*kBUkxWlae~!HZ}UfaS7g9TO!@YCPPDVr-C6{4!Drj5l`3PEgQWen zBMXs_m`IQ-VfGq)lrKhlz8HW~rAk-^EotpPi^QdvGPkF)Vksk7)}-hTrbd(`u`UUV zNmb2xs_(jq%9cOYtF`ndHljEbsa?KwTOU(I9%c2D)axEj$YG7W^X#p%X?n;$Wp!*skuzNSIED+Zzj#6D*$m#Bv zm}$kCxY)>Nxu~SMw`AB7qU=0BO=1ZVq{!@1boFptZuxYUtzNDcP~FVXs<{f>L0*5Une0F`3@GxMxFCz7tf znT4`kUTk=}eVS$^EAb#X@aAb|2)e_KUWZ96oyEYs!r0znxY?I}&ko9(Y)nl}q%^I= zFf;1#rd^Hk#oUO>_TCp37)wamGJ;bImUOb@L-mFrk0hU}ZI z^lNU>Jg+M|Tp6e5-Us(VufV?(>vTwl4VpJn?fUiv`xmixH2*t}_y?yy`4_J?K}}E1 zOiNYWPDx6f?CR>e+v&OQ$%2JZNpR9_MaFMwho%_l0`Dj2>L2Ql*Bv{KOHtM6&PY&8 zELKfSO^Vee+0~6tPLC_yOxIP4){R$-P01PrNuZ+wcX$tXO>+gi4fh%Lp0KBx0gH}U zkte&#CN_sr7k*Km0ZWCMhnWZD=EGwH83n28EO(|Yc1){DP^W2&Kyfkb zQZhVNQGM1yBAmI&rzNMfPwEO5?mEKxGc4@YOssE%idu4iT8f>Y6cycvG1^U5mTHz- z7G}0u21X_ZW0P?9DM=YgV=#sO6hs-uJZ_RzDL6mLi*No;yl4A*gy@cMl3;QYCo0){`X;WNlZ_mfy8qp1+{Vl-Yf?g`~eBZ*$bNZ zQ^`-jrHT3*2P5}?Nfw54h&Tr{0ATuW@$xfSe=lCJE#Jh8n~BSy2R4cbwLBCnMm-Pm>nG2=$C<>)uNLpxnM$|sF(2tvf`e16Qb1g(?FM;F^$7i+21$LXcCXOqr@F;C`=iLrI<&Uarv zuQ6w;)JT}FG^{Rmuez$7C50gq9A(+Q+;n^UqR^vvUf%5S=z8tx+hJ_!lsF>nLHA3! zcyD!h78dUY=YJqaDtA0LPqThr5B4d*=f1h^lryg9bWL4#tcmFJeU9Tx!^uj~;C%Wo(V$4$%(cn|LNDTsIp5XeW$K?A2;?DV(4q!JUS6 zX$H6-0}SZt?f!*fpa@8`y3>ZGY=?x%PU@j3ax~MDfxZt#bXd4W7fB7KIeZrfM#?og zzzk%Q1}hOT2Fw5oz{>Rix$Y~^Xj1OQxMM#0DlJ1UIyU2sdA`9VLS!i*gWBWxHRZOaF3j?5TCpb5xV#h z!G#!5gSH?eg!SucETpnZTmNuV6|sB^y*_2xyX;d^HPLa#48~!MmHjY-F)=v1V>*BxOtEKiBTPM;&AVYyo#f z=oYfuPJ&O;EPR1FJWDdkuW7g#oi6|G2SoRBTw%vamCKQ+<)lrG%Xf53KHGYiTa5Dd zb;Hvqoic=n@`^>1TgX#}Uc&O!7DE#(KJyh>?SxA!x67RTdtdmvL19Oq`<3Bayn3;DIes^;}W{EZ(Yz$bfT3<^>2^J0TaaIlp47?oH(8C-XAJMh$6Y+{o zwa@c5=v7An9mB{Lm@R*VfjcSGVPzzsn0F@%0u?K%S3^pRGA zRnW@&B_!Inq986{7-HYWN`@;rd|U z_A^!YafD@~r-=yo{@-=y)&wm^|fg_u3Fc=^@Hu z?McO|rmJ)XYC`C2^H9pEp&o;S@t$PuzS=IK&`r=pB2%G8S5-#GT zYKArBnHfpR%US1Du`(4q4S?QhhRDY99SBTX=FzR-a2t^(@W2J9e&4uvz^uo`8 z8YyhZN^P-2{q0zlm4!_}Ms}1I2!5&J)s}X&!Nh9Nrn64#oyqyD>$fb~lE?gxX{`s} zt+w1L$2~!BGHDkL&&+~n5 z??1#Mg0IRK-?bk3cUbQn3A`IYb#cx2jOOtnq!CZNfqTJ+*tNY!-@bRE`sn4_;chtI&2CHR@d<`6F97T%YuuJ?2|FBH_ zYDla$MKcsB`%cY)B_5%NNip8sB|m^}6P)P#Tqmfu$5ANjf{!W)FWem~;uOCe2q9bm zE8;Z2(tCdP?I!{EGd>{SPOD@k<}* zv0TzQDPt~|5-LTpw@#6=ciVv%_6M$iM}%cz0FKJMR`_DIDTmW%wjRu=G&o?Agx`*D z*58s<@^RIR1)q?@aii??z}1P+rZg|L$$H-|`u(}OHE-7ug9>gqF2Xm(7DL&0TAyxT zY*Em0flotbB{QTlBr_y3pyqn@5DR7UCLZdZUdvWbM^syR8b@tiR_k?buyl?!^De8C zDDk-3L?R5|*ElmPV3G4T^-X?|ToP5sw#PCdBg8yVttBnR5~ztoqQ* zy3owf#pH>%-}~sv;U11G>ANr)=4Ef`I!5hU^*ah9$6+zGy<_PrUc{nK??MNq$*A-zsOJYAmWhG#++ROOV#whsQYg8oSntlAP}#rZnx(3j&i2d9dYF zl5R_PJ$|6DhtQ@aV+sU)TNja;1awcK9@Zi57ZQ$Fc;RgX4P%nCEe9y;;Dpl(H3d#2 z`?M~YlC34CHM!cIr!>jit@9h{+*gEc)9C$mVLvD}UJG5A{}||$=8_!cldwTE<21{@ z4_+y^Iq1QcRMMVniJM|wv!L#VJHaNi0w(xzfh%mSq!*|dxmYHncKcRj4&cbkZ`H}m zZP%gRxo$wcv)}!!D_5<}H5(Vzi?La`sren`4!?oKU*7R6Zg|^E@%{bI_-c5QhQbK8yQo=#m zZ*wow97^imd{0@TB~#H&_@f)BqJj<WbdMse>*?&XE>I@FGaX2hCSZm4oUofZJ8g35!=z7TGw7{Mx$Sz9ar4 zDw5~XWTP$0?Q-4otc5R%t+g?Nd@o`ZX)PMpgO@AwLqAF&sNdBjHU8NPMk=TJ-Lx3< zB=`LpZykIwPecn(ru3P3MC9mw^H2fu9H(})Ph2X}B4L9r8%GisHuR5tCLrAD8pPZ>;7u{}$O0vimJJ5HZa(OR? zLek@ik35ZFuY_Zlqy#2Vi9BWlW5kJKHsveE`J@sar+CtbO*&$R$Y+nl$r#D-N#4mR zj^?oKzK|7ba`+c&8eCO}@h%_b`SWrf<%RRM9OapUXW&Z6En$9x6Q54B_ zeOZ48JUAEDi9uCbHy*eub5l@vJ?u?sv}x{Fp*AHmP5NrTR$&&ka8{yBdSwK6w-Z+T zb2fX~mhvX@G^z5+G&Nb4)DW5E5Sg#h*wk=EmQrI`@y9HWwqazsX=HF&U(>WMn)Btp zWEp&NN$*yK%`~=4?Usbq(f@7;Zp+}>x|-h_o51>!2j>PcN(Al3gUY(=8#P?NKRoq9 zH#kb!u&?XPhjf*(I2n}7Z)EM{X4V;AKHO9hNY8*!c;m}JCll3|m1ZY_iSBm=sg^#% z`Uz9WMWp$pxdluzU9Op(3Gp?NHQYLqf)K$XG|;EId&6L-TG{nraQ;n)&ySisyaIXYl0C2CfzJ+g{8E7MzajOj3Onskf&4P_W5j z=O~yYGZKm4{l@7yNlZVNHB@xuR1cs2V0(P>)uU)DF0k4Zlnfy*+<0w_2eX{b-R_oz zna{a>p>D(pF1da8C{{as&~RXsQqg#2vLlsXE1Jf{Jt-no8tyWk8bXLSCO2 z)D8o|g&{9^Ec;|rnNKp&-OcMK9>V2gtV;i>x)v8!wvm-2ddo3oaKE9{i%oqli`h!Whq(u zlViOi5d%hKO%a1rk_7>S43Y&AgEf)`;sl=adqS~bb^#n3AmEzr207u#&6UNQa&Zha zG*_ifWr12~Ppa=_1lfMs(1KKuFvL!oyIH}*8EIWZ$||tqSL(;2v8db<=}1~(ER7W> z(+TY@4z(30vkA{F4*aU{$0LN=95kZD7Q+i-aQsh*EbX~Vg9m0eeme>+L(j>Egd(mE z8J3oDdZ`60ka32txtLDj-PLUO^@gWw9Q+B{ zj+2~*${9Pp9mq-pNQ#eMO|y*It%_ctVdA2tirS+Hb@@&W+QNMhT7$^~wB}<9TEleg z!bh2I7i(#m7tqeYjb7Csv`_d8l}#TuoX>7$o&p`%tLW6Pg%f4+9WxfL%SFzq&lML; z!fU6}5BL-!_+dGtLa75mkj%wiV}xuA5_yejva6Xz&deUZeFdMy!Sk4vJIqV_Or)@l zKfM?_<>%+)7v|&VXLrYYdK^{udiwY&D^&;gQuX6C*M4Q>M~?lD&~8{H18+V6KJIp@b0{TS=q^|&t;128^yng&4TjeWo~Hlbko z+Pyd3-NM6_GE!Hc>@KXK&GSfB$n1M_Ff<`zuVSpN)t=_`buHpB>CeuQ4A1oUenbTg zhp7XjJF40tH}Q?_YcCpd6VsZ;nc{CT3Z)CPsNtJA~rRQ|ZnN z`acnuu7saB0}aJdXkmy5uE{3stW*Z0nh!|E6umpQTF~7oaUAkw#}t=*tc&{^;1ZcU z70JHi7i2ka7hWHG)O5G~qDg10TilS(T$;`Z&fL3jAY7MUhmjj6dgLn6Bjg~XPjiX@ z(UmEz-(LmXH{-;Pq3-4B`ku!(FCS0FwTuCr7XxO=GrNuGRz5b zYiag8vhb~?(1_t+?#7^4rQDlFgw{ctwq#rxGi}WiQAP6vrO_-nYu&;$oRfGch7^Wie@D#ah}l>M0m#h9M|LAvBixcPzdl ztn<@5bVroC$Vb}5VEtsO2_8{Lz-1O?c^{-sLrn)iMvgeohgy1AOm>;E$ro2fx)Rny zn8XDgLk1(sn#K(riw9@oOORy6DN4WTcWYt>vekK{bI>0nAuD7U+;&|R6G{c8t_Ge` z69sWPi)*R^pW`P`gKEC3rQ~LvNnj)H*QctD6jc&>jx)en4D7Kx932~!_{2jH=lisH_EHL z=BDwg!;Bp{AFt5F_^m!Eqj`bANBdlin|6fErI@<`dyF0v4-***<|{&IV7+%*tE?D;)<*QvlNR?nJ#TZ;mXT@zuW0KAY6 z?T6jQMFI>;CZ5Icv}%M$djNzzWN@_(GF=#Vds2?>b}zv;sRE^6sJ%zkV0im@(^Bv! zuM>n#enbeN_oxF~BZZ7^VNztbu>uwCLpY5-`=QB5tyuZnh41|@3+1_5wT|_w;=1X; zTd_{`?O6RSG2c$r==}>trD|=LXHU{tShmm86Qns;(9dMzVysCGvGUKJ{n$mzktF8A9-xoE%6GoZ+$D zJnRfiY*<}&qvugKr1A)INjZwI9wwWMA~Za&t8h%9v*Cr6;+tjTB3)mO)zBwB^S{2x z+#Hj?g8i<7sK_^)ep5kA|49`D{y$a)vFn?BW}l&+Fan7kHZMUeP5%E?LH^S#M;?D+ z^$JsEG-to6Am;zH3c>^lhYwLjo?5MkoGPC^1x^x)S zWah_114EGdY>nUUlpfy>Y{?ryk0|Y_B}FXk=40#PMM*>DsIs5^(E}(dNCQBjtxKSQ zDV2AjNqwlWAQcb^n3*59)>?VEM3$CTGL~-F&)ZYj@!2mMZf{Yf;ev~1EL7H}oB7z? zh}aX96>7*pV$V7rCw5+eF)A%&b#<|yd(w0v+J#%z;_>Dj+?0<-i%}$UWa-SF)kQMM zDAd4(qUE|xhvB?HFgP>Y;Uems4~|qynXL{_cw z^FacjBF}>Qc=N{?uaub^g}PEH0!@Z?!sAJyITgmGkihS|eb!&78ms$W0f4@=UU z0IUUL>U_~b6%x&IFwI`%2xp_@G@9a9E^7`GXE{!pp~#aW_A&YA$mf*5Tjt;%I94 z$^2oczvoBHy~$p7G}Xizatrpr;?DV2%NGn`-4jZ|b_o0FLBO{Va?g%t2wuhw+oyfb z2{q%lp4}#vD%n|QjSpqi^gM2iWpr7c6r&EDN%M2% zY!&(xtaG~%aBoO1)jI_a~|azcC>Q$HC|EyZx4Oj_MDx2R@< z?E)MxPoU0u-l{_*Yi@`@d?=9MR+@4tSgYsS5`HVph1EG)*~yWkM_>0??yl_^cN&^xo8?VJhf71H6q&5ivbybbFtCUl${-HOU3%Z zb>_D=#=NWH2RWHS2gO0MxoGyC<;wJvw4q^*FYL$F4zcBA`nK_LShA{doVIl+lL$jc zgGvSK%##R_aU9c|pC1V1CrwtfOId-tJZhDW$ReerOYf!JQIV0wCtW;Bn z76$70BdO4fB+VNf5p`KbA?kfC>~pQ!=nCdTVPv9#jJuX`81z5S7g)Ju2UrA3%Y%G9 z>sw0EmdFQnqlrtnWCmwCE-%hlDiEJYO>K?9WppsAZS(59WkBdBYCm%y@l(oZ|#l5%M*!yY!c0zSd z_s3J@H8YD(_@;Foa2FJjP{Tfe2D?bG#y-n$hfvZozcs!xvzUa^2GL_A=%br_fYBebCtdMfAQ5{76Fs38OVshk?B zt9bKnB+3~5DJBR;XiqU{0cJ$Me%oAyud%DpV5c;XvIot`Qvq5e?`oY$Lbg+Jx zQ1F7EmcZfCXytLxNc>H>DkN^fQTo7)Jl4PrChC(HApv+;Y?*MPd~bHFhEnL`hy&6qThcDyrhfGuT@W-`Em@UC(erL%H zU;@h&jd#UXDDBA&9n3zStHSINeNY>Ez}P(EF+kniTs}9X{Qe3XHfNy+VFE;#DcS_A z&OFc$P?79Uvx7RWhZU9oNPfH?+0rWak^T6ryPV7aqZ&c_0PlU(@gvn|5z#A}Qq$PX z{wtcuL34_5J^^%Ao7DH|=7bNHR$)Ues4sM|5>q&6NSYiL)RGMjV=9p_8L>JSq-sv_ z_yji#Rv98LnzySzwz*rZn}me9AABs$x*&n6KnPj{k3w9)+THM5BDsh-ecf31g>3Jk z0&6Mk@ikx(q3bA;#U>$2RPr1t;0&DF!>r(vUp+j5H7abnrtDa+b9DJH>jR8FVp=i` z=JmU4;;IWKom`%-ej#7t+t`=L;-?xI3>T@{LGi_>>7Ez$RjO!suIhD0jt-F@cCvCr z1wEgW&^-Dc&jc0vaobkpGXv>er_0Tl(6mtEY;7=j0ryXx0M9+A~T zT&Bg?KV8xI;v)uf(dT4_UQAu8=0zOTcZ6;^ty9T6pp4v-hZ-HzD(;{g}znZ@?fp_x}`oND&I8I zH)%_D%i}xxQzav;e*2oU>HePT$!oCXO2Bo`cEcAzGg=#i;?+R=p2aN^_s`t=`yxE- zF5D(hZH?V7@0r9+gn100af{VqxGvg;d8S9gKxB8~*m#jCqEH;m2G8SmRGo|i=Q8Jy>q#vc&DEr( zXLqPVVr{ARno(0}v^0iu<=X`R#HR(`uDnnl9f3#^94;EnDt{7Luw5vdaXf9Zbj{c_{M zUkGedEi7Kb8{LO_>oWZ@%fj*>ru*Np-v6Zg5`WPB!J0R^zbDACCdUa}%!EL8hbi&8 z8x_2SZkw$ds}4XiX#+{91WSnr)@K$G1p6Z3qQVbW#1G~#FC{T78#KZfJaUk;1nLas zNTKwy2=sZFKHCa>NwuNzFB%5hl;L&qEg)*pUxNBuR{n>PaQm+VIn*dTwOn7h3GCv?_*VvULUs#Ml zY@3rL-FVHDG>zC~zfsx|hIUpey9IB8O|dt=KXrx}-7JXcI1bx2aXltENS><~TO@R%gM3kWBDwJ<%Llzo>VXj0AgOkLd=v?vI8 z@(~$T9Ru#t4#umlI^0E5m7(ESE-GPu@|t_|x!EA{T~;s?QuuZXel%QeZi&=CmiDco|KF68I&M z36r4#X!8(s2v%Fn=epLm5=f(bk}+|$yFd)%#C-JTU2;Q5GzA|9N(v^*+NzprM>_}m z%kLLIU?8EP!b8GBCB-E^j1G+q6R)_OtQol2g$cpACXMYP6ZL3gx4!EZMYJ;pA!a5g zF=Xs#Bw$R5$4WV4WM*RW6XsV7j425R3q^2trLUpa$<$8yJhpMxMu78|+kv~B8zr%O}Tjg6X$H^3VoUgae;(8xEa`_XP_#x37v8O{h(kVazs=WBt&tJM` zYu)Z1A}VywsbN`;(}RpTs1;WHrBL@CXfx#A>l4LTcodtZ*Y+xw3@FIQeW;e7l~9#N z=?1hQT}&gn`8u>QW-3ODnZ^&i##%zC{9~w)o6z-Rp-LbFmM(ll;=W>1 zP-0+MO2&R59!Ek;=5jA0;b9}s*FbYEYg!Owq}6mzbD{gDK)WQTP+8y;;=1J!h6qJA z`sIqSiE{t@4Ne=ge1i{shHkl!3icKRNl=b5?pwyFI#=$do2Q3AioD)ocGQh{|6aUB zYfIZ*-s*|ZTfU*+EWD1!rq;$bPL8_F|DBI_LjGU=U4^kX^X|=G&ATzv$)7(1LErf% zqXqa1O8o$jFtaFo?a=`j);7k}I(Z~e=+pPw^@f4ul}`FZ6Z9n77K>8xxGE&uMy z^X31UQw00hoZ??*6#t6$`=sHoXkLGy{i|00Gpyey@P5Ur zdHb&Zm|=V?k^cNSYKiO!20P@_-9bR8?>Hdmk3;!q9 zKMwJC>&35Fhi`A*uIrzN_`9*-S14udH>f{t2LCL?-<$5gVg5X_&Y)Q6$_u`Ppp3&;_v_6e#N3=`vdErhxmK({uL^f9q?Nz|GO~%tCIRF fp3-~3Zw2 Date: Fri, 9 Jun 2017 14:29:26 +0930 Subject: [PATCH 034/131] SOLR-10647: move JsonSchemaValidator to SolrJ --- solr/core/src/java/org/apache/solr/api/Api.java | 2 +- solr/core/src/java/org/apache/solr/api/ApiBag.java | 2 +- solr/core/src/java/org/apache/solr/api/V2HttpCall.java | 2 +- .../apache/solr/handler/admin/SecurityConfHandler.java | 2 +- .../org/apache/solr/request/SolrQueryRequestBase.java | 2 +- .../src/java/org/apache/solr/servlet/HttpSolrCall.java | 2 +- .../apache/solr/common}/util/JsonSchemaValidator.java | 5 +---- .../org/apache/solr/common}/util/JsonValidatorTest.java | 9 +++------ 8 files changed, 10 insertions(+), 16 deletions(-) rename solr/{core/src/java/org/apache/solr => solrj/src/java/org/apache/solr/common}/util/JsonSchemaValidator.java (98%) rename solr/{core/src/test/org/apache/solr => solrj/src/test/org/apache/solr/common}/util/JsonValidatorTest.java (96%) diff --git a/solr/core/src/java/org/apache/solr/api/Api.java b/solr/core/src/java/org/apache/solr/api/Api.java index edc2206b4a6..d2c468cbfbe 100644 --- a/solr/core/src/java/org/apache/solr/api/Api.java +++ b/solr/core/src/java/org/apache/solr/api/Api.java @@ -24,7 +24,7 @@ import org.apache.solr.common.SpecProvider; import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.JsonSchemaValidator; +import org.apache.solr.common.util.JsonSchemaValidator; /** Every version 2 API must extend the this class. It's mostly like a request handler * but it has extra methods to provide the json schema of the end point diff --git a/solr/core/src/java/org/apache/solr/api/ApiBag.java b/solr/core/src/java/org/apache/solr/api/ApiBag.java index 7f30602045a..805c31ed3f7 100644 --- a/solr/core/src/java/org/apache/solr/api/ApiBag.java +++ b/solr/core/src/java/org/apache/solr/api/ApiBag.java @@ -44,7 +44,7 @@ import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.PermissionNameProvider; -import org.apache.solr.util.JsonSchemaValidator; +import org.apache.solr.common.util.JsonSchemaValidator; import org.apache.solr.common.util.PathTrie; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java index 36ca4867eb1..baf4abc9972 100644 --- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java +++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java @@ -48,7 +48,7 @@ import org.apache.solr.security.AuthorizationContext; import org.apache.solr.servlet.HttpSolrCall; import org.apache.solr.servlet.SolrDispatchFilter; import org.apache.solr.servlet.SolrRequestParsers; -import org.apache.solr.util.JsonSchemaValidator; +import org.apache.solr.common.util.JsonSchemaValidator; import org.apache.solr.common.util.PathTrie; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java index 6e81ccfb3bf..510ef482e71 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java @@ -46,7 +46,7 @@ import org.apache.solr.common.util.CommandOperation; import org.apache.solr.api.Api; import org.apache.solr.api.ApiBag.ReqHandlerToApi; import org.apache.solr.common.SpecProvider; -import org.apache.solr.util.JsonSchemaValidator; +import org.apache.solr.common.util.JsonSchemaValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java index 8275991f31f..83e44dd4c38 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java +++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java @@ -22,7 +22,7 @@ import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.common.util.CommandOperation; -import org.apache.solr.util.JsonSchemaValidator; +import org.apache.solr.common.util.JsonSchemaValidator; import org.apache.solr.util.RTimerTree; import org.apache.solr.util.RefCounted; import org.apache.solr.schema.IndexSchema; diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index f529bd4b689..11b483ad057 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -102,7 +102,7 @@ import org.apache.solr.servlet.cache.HttpCacheHeaderUtil; import org.apache.solr.servlet.cache.Method; import org.apache.solr.update.processor.DistributingUpdateProcessorFactory; import org.apache.solr.common.util.CommandOperation; -import org.apache.solr.util.JsonSchemaValidator; +import org.apache.solr.common.util.JsonSchemaValidator; import org.apache.solr.util.RTimerTree; import org.apache.solr.util.TimeOut; import org.apache.zookeeper.KeeperException; diff --git a/solr/core/src/java/org/apache/solr/util/JsonSchemaValidator.java b/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java similarity index 98% rename from solr/core/src/java/org/apache/solr/util/JsonSchemaValidator.java rename to solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java index 8a0a09ff083..e7b725f42a0 100644 --- a/solr/core/src/java/org/apache/solr/util/JsonSchemaValidator.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.solr.util; +package org.apache.solr.common.util; import java.util.Arrays; import java.util.HashMap; @@ -27,9 +27,6 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.Utils; - /** * A very basic and lightweight json schema parsing and data validation tool. This custom tool is created * because a) we need to support non json inputs b) to avoiding double parsing (this accepts an already parsed json as a map) diff --git a/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java b/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java similarity index 96% rename from solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java rename to solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java index a5e50dd1a2b..fa6d080288c 100644 --- a/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java @@ -14,21 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.util; +package org.apache.solr.common.util; import java.util.List; import java.util.Map; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.util.ValidatingJsonMap; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.Utils; -import static org.apache.solr.common.util.ValidatingJsonMap.NOT_NULL; import static org.apache.solr.common.util.Utils.toJSONString; +import static org.apache.solr.common.util.ValidatingJsonMap.NOT_NULL; -public class JsonValidatorTest extends SolrTestCaseJ4 { +public class JsonValidatorTest extends SolrTestCaseJ4 { public void testSchema() { checkSchema("collections.Commands"); From df74c65503f0e0ee942b79a6ed539cd785eef343 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Fri, 9 Jun 2017 07:41:52 -0500 Subject: [PATCH 035/131] SOLR-10854: change PDF subtitle to use "solr-guide-version" to add "-DRAFT" to front page when applicable --- solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc b/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc index aa379f3e849..4ac125e177c 100644 --- a/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc +++ b/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc @@ -1,4 +1,4 @@ -= Apache Solr Reference Guide: For Solr {solr-docs-version} += Apache Solr Reference Guide: For Solr {solr-guide-version} :toc: :toc-title: Table of Contents :toclevels: 2 From 463907a13c461e35df4a260f2b7edc2a0adf122c Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Thu, 8 Jun 2017 20:55:20 -0400 Subject: [PATCH 036/131] SOLR-10855: Null pointer exceptions in CartesianProductStream toExpression and explain methods --- .../solrj/io/stream/CartesianProductStream.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CartesianProductStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CartesianProductStream.java index 6514ae496cc..c96b9fe8a9f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CartesianProductStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CartesianProductStream.java @@ -150,8 +150,10 @@ public class CartesianProductStream extends TupleStream implements Expressible { for(NamedEvaluator evaluator : evaluators) { expression.addParameter(String.format(Locale.ROOT, "%s as %s", evaluator.getEvaluator().toExpression(factory), evaluator.getName())); } - - expression.addParameter(new StreamExpressionNamedParameter("productSort", orderBy.toExpression(factory))); + + if(orderBy != null) { + expression.addParameter(new StreamExpressionNamedParameter("productSort", orderBy.toExpression(factory))); + } return expression; } @@ -171,8 +173,10 @@ public class CartesianProductStream extends TupleStream implements Expressible { for(NamedEvaluator evaluator : evaluators){ explanation.addHelper(evaluator.getEvaluator().toExplanation(factory)); } - - explanation.addHelper(orderBy.toExplanation(factory)); + + if(orderBy != null) { + explanation.addHelper(orderBy.toExplanation(factory)); + } return explanation; } From 1dab10e817868e98b7f5120bbcac4188ebb6f49a Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Fri, 9 Jun 2017 11:55:11 -0400 Subject: [PATCH 037/131] Ref Guide: Add knn documentations --- solr/solr-ref-guide/src/stream-sources.adoc | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/solr/solr-ref-guide/src/stream-sources.adoc b/solr/solr-ref-guide/src/stream-sources.adoc index 973d135983f..e77fc143da8 100644 --- a/solr/solr-ref-guide/src/stream-sources.adoc +++ b/solr/solr-ref-guide/src/stream-sources.adoc @@ -217,6 +217,36 @@ features(collection1, The `gatherNodes` function provides breadth-first graph traversal. For details, see the section <>. +== knn + +The `knn` function returns the K nearest neighbors for a document based on text similarity. Under the covers the `knn` function +use the More Like This query parser plugin. + +=== knn Parameters + +* `collection`: (Mandatory) The collection to perform the search in. +* `id`: (Mandatory) The id of the source document to begin the knn search from. +* `qf`: (Mandatory) The query field used to compare documents. +* `fl`: (Mandatory) The field list returned. +* `mintf`: (Mandatory) The minimum numer of occurrences of the term in the source document to be inlcued in the search. +* `maxtf`: (Mandatory) The maximum numer of occurrences of the term in the source document to be inlcued in the search. +* `mindf`: (Mandatory) The minimum numer of occurrences in the corpus to be inlcued in the search. +* `maxdf`: (Mandatory) The maximum numer of occurrences in the corpus to be inlcued in the search. +* `minwl`: (Mandatory) The minimum world length of to be inlcued in the search. +* `maxwl`: (Mandatory) The maximum world length of to be inlcued in the search. + +=== knn Syntax + +[source,text] +---- +knn(collection1, + id="doc1", + qf="text_field", + fl="id, title", + mintf="3", + maxdf="10000000") +---- + == model The `model` function retrieves and caches logistic regression text classification models that are stored in a SolrCloud collection. The `model` function is designed to work with models that are created by the <>, but can also be used to retrieve text classification models trained outside of Solr, as long as they conform to the specified format. After the model is retrieved it can be used by the <> to classify documents. @@ -404,6 +434,7 @@ JSON Facet API as its high performance aggregation engine. * `start`: (Mandatory) The start of the time series expressed in Solr date or date math syntax. * `end`: (Mandatory) The end of the time series expressed in Solr date or date math syntax. * `gap`: (Mandatory) The time gap between time series aggregation points expressed in Solr date math syntax. +* `format`: (Optional) Date template to format the date field in the output tuples. Formatting is performed by Java's SimpleDateFormat class. * `metrics`: (Mandatory) The metrics to include in the result tuple. Current supported metrics are `sum(col)`, `avg(col)`, `min(col)`, `max(col)` and `count(*)` === timeseries Syntax @@ -416,6 +447,7 @@ timeseries(collection1, start="NOW-30DAYS", end="NOW", gap="+1DAY", + format="YYYY-MM-dd", sum(a_i), max(a_i), max(a_f), From 489d2a4aabedb2d313fe60c6a35336b5bf09bd1b Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Fri, 9 Jun 2017 12:18:42 -0400 Subject: [PATCH 038/131] Ref Guide: knn documentation --- solr/solr-ref-guide/src/stream-sources.adoc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/solr/solr-ref-guide/src/stream-sources.adoc b/solr/solr-ref-guide/src/stream-sources.adoc index e77fc143da8..1f1748e0ed7 100644 --- a/solr/solr-ref-guide/src/stream-sources.adoc +++ b/solr/solr-ref-guide/src/stream-sources.adoc @@ -227,13 +227,14 @@ use the More Like This query parser plugin. * `collection`: (Mandatory) The collection to perform the search in. * `id`: (Mandatory) The id of the source document to begin the knn search from. * `qf`: (Mandatory) The query field used to compare documents. +* `k`: (Mandatory) The number of nearest neighbors to return. * `fl`: (Mandatory) The field list returned. -* `mintf`: (Mandatory) The minimum numer of occurrences of the term in the source document to be inlcued in the search. -* `maxtf`: (Mandatory) The maximum numer of occurrences of the term in the source document to be inlcued in the search. -* `mindf`: (Mandatory) The minimum numer of occurrences in the corpus to be inlcued in the search. -* `maxdf`: (Mandatory) The maximum numer of occurrences in the corpus to be inlcued in the search. -* `minwl`: (Mandatory) The minimum world length of to be inlcued in the search. -* `maxwl`: (Mandatory) The maximum world length of to be inlcued in the search. +* `mintf`: (Optional) The minimum numer of occurrences of the term in the source document to be inlcued in the search. +* `maxtf`: (Optional) The maximum numer of occurrences of the term in the source document to be inlcued in the search. +* `mindf`: (Optional) The minimum numer of occurrences in the corpus to be inlcued in the search. +* `maxdf`: (Optional) The maximum numer of occurrences in the corpus to be inlcued in the search. +* `minwl`: (Optional) The minimum world length of to be inlcued in the search. +* `maxwl`: (Optional) The maximum world length of to be inlcued in the search. === knn Syntax From 70ecf9ef3e4ac4fd81ebc80c76551c6cf00840e4 Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Fri, 9 Jun 2017 12:23:58 -0400 Subject: [PATCH 039/131] Ref Guide: knn documentation --- solr/solr-ref-guide/src/stream-sources.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/stream-sources.adoc b/solr/solr-ref-guide/src/stream-sources.adoc index 1f1748e0ed7..97db6f2fbd5 100644 --- a/solr/solr-ref-guide/src/stream-sources.adoc +++ b/solr/solr-ref-guide/src/stream-sources.adoc @@ -228,7 +228,7 @@ use the More Like This query parser plugin. * `id`: (Mandatory) The id of the source document to begin the knn search from. * `qf`: (Mandatory) The query field used to compare documents. * `k`: (Mandatory) The number of nearest neighbors to return. -* `fl`: (Mandatory) The field list returned. +* `fl`: (Mandatory) The field list to return. * `mintf`: (Optional) The minimum numer of occurrences of the term in the source document to be inlcued in the search. * `maxtf`: (Optional) The maximum numer of occurrences of the term in the source document to be inlcued in the search. * `mindf`: (Optional) The minimum numer of occurrences in the corpus to be inlcued in the search. @@ -243,6 +243,7 @@ use the More Like This query parser plugin. knn(collection1, id="doc1", qf="text_field", + k="10", fl="id, title", mintf="3", maxdf="10000000") From a3bb51720c613a1ab2e51ef45541bec8566e4a96 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Fri, 9 Jun 2017 11:44:55 -0500 Subject: [PATCH 040/131] Ref Guide: escape pipes in the table --- solr/solr-ref-guide/src/the-standard-query-parser.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/the-standard-query-parser.adoc b/solr/solr-ref-guide/src/the-standard-query-parser.adoc index f4455d2b97d..94baedf6f25 100644 --- a/solr/solr-ref-guide/src/the-standard-query-parser.adoc +++ b/solr/solr-ref-guide/src/the-standard-query-parser.adoc @@ -246,7 +246,7 @@ Boolean operators allow you to apply Boolean logic to queries, requiring the pre |Boolean Operator |Alternative Symbol |Description |AND |`&&` |Requires both terms on either side of the Boolean operator to be present for a match. |NOT |`!` |Requires that the following term not be present. -|OR |`||` |Requires that either term (or both terms) be present for a match. +|OR |`\|\|` |Requires that either term (or both terms) be present for a match. | |`+` |Requires that the following term be present. | |`-` |Prohibits the following term (that is, matches on fields or documents that do not include that term). The `-` operator is functionally similar to the Boolean operator `!`. Because it's used by popular search engines such as Google, it may be more familiar to some user communities. |=== From 05fc4ae03452bc2d2cdc87f86f9f9b2379ac3599 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Fri, 9 Jun 2017 12:04:30 -0500 Subject: [PATCH 041/131] Ref Guide: escape brackets so they don't come out as arrows --- solr/solr-ref-guide/src/the-dismax-query-parser.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/the-dismax-query-parser.adoc b/solr/solr-ref-guide/src/the-dismax-query-parser.adoc index 64682bf5a4f..ea250b1182c 100644 --- a/solr/solr-ref-guide/src/the-dismax-query-parser.adoc +++ b/solr/solr-ref-guide/src/the-dismax-query-parser.adoc @@ -102,7 +102,7 @@ The table below explains the various ways that mm values can be specified. |Percentage |75% |Sets the minimum number of matching clauses to this percentage of the total number of optional clauses. The number computed from the percentage is rounded down and used as the minimum. |Negative percentage |-25% |Indicates that this percent of the total number of optional clauses can be missing. The number computed from the percentage is rounded down, before being subtracted from the total to determine the minimum number. |An expression beginning with a positive integer followed by a > or < sign and another value |3<90% |Defines a conditional expression indicating that if the number of optional clauses is equal to (or less than) the integer, they are all required, but if it's greater than the integer, the specification applies. In this example: if there are 1 to 3 clauses they are all required, but for 4 or more clauses only 90% are required. -|Multiple conditional expressions involving > or < signs |2<-25% 9<-3 |Defines multiple conditions, each one being valid only for numbers greater than the one before it. In the example at left, if there are 1 or 2 clauses, then both are required. If there are 3-9 clauses all but 25% are required. If there are more then 9 clauses, all but three are required. +|Multiple conditional expressions involving > or < signs |2\<-25% 9\<-3 |Defines multiple conditions, each one being valid only for numbers greater than the one before it. In the example at left, if there are 1 or 2 clauses, then both are required. If there are 3-9 clauses all but 25% are required. If there are more then 9 clauses, all but three are required. |=== When specifying `mm` values, keep in mind the following: From 2c0239c217a50034df80fb04e492fb69664c7ae1 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Fri, 9 Jun 2017 12:07:06 -0500 Subject: [PATCH 042/131] Ref Guide: reduce levels in PDF TOC; increase font size for literal text --- solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc | 2 +- solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc b/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc index 4ac125e177c..214bf0770d2 100644 --- a/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc +++ b/solr/solr-ref-guide/src/pdf/SolrRefGuide-all.adoc @@ -1,7 +1,7 @@ = Apache Solr Reference Guide: For Solr {solr-guide-version} :toc: :toc-title: Table of Contents -:toclevels: 2 +:toclevels: 1 :author: Written by the Apache Lucene/Solr Project :email: https://lucene.apache.org/solr :revdate: Published {build-date} diff --git a/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml b/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml index 0c10425caa2..ef3244a932c 100644 --- a/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml +++ b/solr/solr-ref-guide/src/pdf/themes/refguide-theme.yml @@ -45,7 +45,7 @@ base: border_color: '#eeeeee' border_radius: 4 border_width: 0.5 -vertical_rhythm: $base_line_height_length +vertical_rhythm: 10 horizontal_rhythm: $base_line_height_length vertical_spacing: $vertical_rhythm link: @@ -54,6 +54,7 @@ link: literal: font_color: '#333333' font_family: Inconsolata + font_size: 11 background_color: '#f5f5f5' # code is used for source code blocks code: From 8cd826f2936853d345c863e51c50eeaa9754a061 Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Fri, 9 Jun 2017 09:36:01 -0700 Subject: [PATCH 043/131] SOLR-10829: Fixed IndexSchema to enforce that uniqueKey can not be Points based for correctness --- solr/CHANGES.txt | 2 ++ .../org/apache/solr/schema/IndexSchema.java | 8 ++++++ .../conf/bad-schema-uniquekey-uses-points.xml | 28 +++++++++++++++++++ .../solr/schema/BadIndexSchemaTest.java | 2 ++ 4 files changed, 40 insertions(+) create mode 100644 solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-uses-points.xml diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index dc17616213e..e5eb0c19755 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -348,6 +348,8 @@ Bug Fixes * SOLR-10137: Ensure that ConfigSets created via API are mutable. (Hrishikesh via Mark Miller) +* SOLR-10829: Fixed IndexSchema to enforce that uniqueKey can not be Points based for correctness (hossman) + Optimizations ---------------------- * SOLR-10634: JSON Facet API: When a field/terms facet will retrieve all buckets (i.e. limit:-1) diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index 04f64d5d7a3..463df3ee681 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -534,6 +534,14 @@ public class IndexSchema { log.error(msg); throw new SolrException(ErrorCode.SERVER_ERROR, msg); } + + if (uniqueKeyField.getType().isPointField()) { + String msg = UNIQUE_KEY + " field ("+uniqueKeyFieldName+ + ") can not be configured to use a Points based FieldType: " + uniqueKeyField.getType().getTypeName(); + log.error(msg); + throw new SolrException(ErrorCode.SERVER_ERROR, msg); + } + uniqueKeyFieldName=uniqueKeyField.getName(); uniqueKeyFieldType=uniqueKeyField.getType(); diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-uses-points.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-uses-points.xml new file mode 100644 index 00000000000..ac4fd9c4a2d --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-uses-points.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + id + + diff --git a/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java b/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java index 2d5adf993d0..5545d959026 100644 --- a/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java +++ b/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java @@ -65,6 +65,8 @@ public class BadIndexSchemaTest extends AbstractBadConfigTestBase { "can not be configured with a default value"); doTest("bad-schema-uniquekey-multivalued.xml", "can not be configured to be multivalued"); + doTest("bad-schema-uniquekey-uses-points.xml", + "can not be configured to use a Points based FieldType"); } public void testMultivaluedCurrency() throws Exception { From 3033229766a99b9292b9a584aac12e36a5fd1028 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Fri, 9 Jun 2017 13:49:29 -0500 Subject: [PATCH 044/131] Ref Guide: fix incomplete source declaration --- solr/solr-ref-guide/src/collections-api.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc index 78869b69884..2c4e9cddc39 100644 --- a/solr/solr-ref-guide/src/collections-api.adoc +++ b/solr/solr-ref-guide/src/collections-api.adoc @@ -1398,7 +1398,7 @@ http://localhost:8983/solr/admin/collections?action=DELETESTATUS&requestid=bar *Input: Clear all the stored statuses* -[source +[source,text] ---- http://localhost:8983/solr/admin/collections?action=DELETESTATUS&flush=true ---- From c37b377438c868f4dd1854c0468bfa2bc0512abc Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Fri, 9 Jun 2017 15:19:00 -0400 Subject: [PATCH 045/131] Ref Guide: fix miscellaneous formatting issues/typos/etc. --- .../adding-custom-plugins-in-solrcloud-mode.adoc | 2 +- solr/solr-ref-guide/src/charfilterfactories.adoc | 2 +- solr/solr-ref-guide/src/cloud-screens.adoc | 2 +- solr/solr-ref-guide/src/codec-factory.adoc | 2 +- solr/solr-ref-guide/src/collections-api.adoc | 2 +- solr/solr-ref-guide/src/format-of-solr-xml.adoc | 2 +- .../src/hadoop-authentication-plugin.adoc | 2 +- .../src/implicit-requesthandlers.adoc | 2 +- .../src/kerberos-authentication-plugin.adoc | 14 +++++++------- .../solr-ref-guide/src/parallel-sql-interface.adoc | 2 +- .../src/performance-statistics-reference.adoc | 2 +- .../setting-up-an-external-zookeeper-ensemble.adoc | 8 ++++---- .../src/shards-and-indexing-data-in-solrcloud.adoc | 2 +- .../src/solr-control-script-reference.adoc | 8 ++++++-- .../src/solr-cores-and-solr-xml.adoc | 2 +- solr/solr-ref-guide/src/stream-evaluators.adoc | 2 +- ...ta-store-data-with-the-data-import-handler.adoc | 2 +- solr/solr-ref-guide/src/working-with-dates.adoc | 5 ++++- .../src/zookeeper-access-control.adoc | 2 +- 19 files changed, 36 insertions(+), 29 deletions(-) diff --git a/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc b/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc index 54647f9dce3..f9277f0f4b1 100644 --- a/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc +++ b/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc @@ -108,7 +108,7 @@ The `.der` files that are output from Step 2 should then be loaded to ZooKeeper At the current time, you can only use the ZooKeeper `zkCli.sh` (or `zkCli.cmd` on Windows) script to issue these commands (the Solr version has the same name, but is not the same). If you have your own ZooKeeper ensemble running already, you can find the script in `$ZK_INSTALL/bin/zkCli.sh` (or `zkCli.cmd` if you are using Windows). -NOTE: If you are running the embedded ZooKeeper that is included with Solr, you *do not* have this script already; in order to use it, you will need to download a copy of ZooKeeper v3.4.6 from http://zookeeper.apache.org/. Don't worry about configuring the download, you're just trying to get the command line utility script. When you start the script, you will connect to the embedded ZooKeeper. +NOTE: If you are running the embedded ZooKeeper that is included with Solr, you *do not* have this script already; in order to use it, you will need to download a copy of ZooKeeper v3.4.10 from http://zookeeper.apache.org/. Don't worry about configuring the download, you're just trying to get the command line utility script. When you start the script, you will connect to the embedded ZooKeeper. To load the keys, you will need to connect to ZooKeeper with `zkCli.sh`, create the directories, and then create the key file, as in the following example. diff --git a/solr/solr-ref-guide/src/charfilterfactories.adoc b/solr/solr-ref-guide/src/charfilterfactories.adoc index d25648cacee..5b64c875e41 100644 --- a/solr/solr-ref-guide/src/charfilterfactories.adoc +++ b/solr/solr-ref-guide/src/charfilterfactories.adoc @@ -100,7 +100,7 @@ The table below presents examples of HTML stripping. |`hello'); -->` |hello |`if a` |hello -|`a> mode, a "Cloud" option will appear in the Admin UI between <> and <>. -This screen provides status information about each collection & node in your cluster, as well as access to the low level data being stored in <>. +This screen provides status information about each collection & node in your cluster, as well as access to the low level data being stored in <>. .Only Visible When using SolrCloud [NOTE] diff --git a/solr/solr-ref-guide/src/codec-factory.adoc b/solr/solr-ref-guide/src/codec-factory.adoc index 8f470386ac2..388f5881f11 100644 --- a/solr/solr-ref-guide/src/codec-factory.adoc +++ b/solr/solr-ref-guide/src/codec-factory.adoc @@ -45,7 +45,7 @@ Example: === solr.SimpleTextCodecFactory -This factory for Lucene's {solr-javadocs}/solr-core/org/apache/solr/core/SimpleTextCodecFactory.html[`solr.SimpleTextCodecFactory`] produces a plain text human-readable index format. +This factory for Lucene's {solr-javadocs}/solr-core/org/apache/solr/core/SimpleTextCodecFactory.html[`SimpleTextCodecFactory`] produces a plain text human-readable index format. CAUTION: *FOR RECREATIONAL USE ONLY*. This codec should never be used in production. `SimpleTextCodec` is relatively slow and takes up a large amount of disk space. Its use should be limited to educational and debugging purposes. diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc index 2c4e9cddc39..33dcb946f3b 100644 --- a/solr/solr-ref-guide/src/collections-api.adoc +++ b/solr/solr-ref-guide/src/collections-api.adoc @@ -112,7 +112,7 @@ http://localhost:8983/solr/admin/collections?action=CREATE&name=newCollection&nu `/admin/collections?action=MODIFYCOLLECTION&collection=_&=&=_` -It's possible to edit multiple attributes at a time. Changing these values only updates the z-node on Zookeeper, they do not change the topology of the collection. For instance, increasing replicationFactor will _not_ automatically add more replicas to the collection but _will_ allow more ADDREPLICA commands to succeed. +It's possible to edit multiple attributes at a time. Changing these values only updates the z-node on ZooKeeper, they do not change the topology of the collection. For instance, increasing replicationFactor will _not_ automatically add more replicas to the collection but _will_ allow more ADDREPLICA commands to succeed. *Query Parameters* diff --git a/solr/solr-ref-guide/src/format-of-solr-xml.adoc b/solr/solr-ref-guide/src/format-of-solr-xml.adoc index 02e72bd7958..28dc3c481db 100644 --- a/solr/solr-ref-guide/src/format-of-solr-xml.adoc +++ b/solr/solr-ref-guide/src/format-of-solr-xml.adoc @@ -24,7 +24,7 @@ This section will describe the default `solr.xml` file included with Solr and ho == Defining solr.xml -You can find `solr.xml` in your `$SOLR_HOME` directory (usually `server/solr`) in standalone mode or in Zookeeper when using SolrCloud. The default `solr.xml` file looks like this: +You can find `solr.xml` in your `$SOLR_HOME` directory (usually `server/solr`) in standalone mode or in ZooKeeper when using SolrCloud. The default `solr.xml` file looks like this: [source,xml] ---- diff --git a/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc b/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc index cafab92d14d..2ac541a57af 100644 --- a/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc @@ -52,7 +52,7 @@ For most SolrCloud or standalone Solr setups, the `HadoopAuthPlugin` should suff |authConfigs |Yes |Configuration parameters required by the authentication scheme defined by the type property. For more details, see https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[Hadoop configuration] options. |defaultConfigs |No |Default values for the configuration parameters specified by the `authConfigs` property. The default values are specified as a collection of key-value pairs (i.e., `property-name:default_value`). |enableDelegationToken |No |Enable (or disable) the delegation tokens functionality. -|initKerberosZk |No |For enabling initialization of kerberos before connecting to Zookeeper (if applicable). +|initKerberosZk |No |For enabling initialization of kerberos before connecting to ZooKeeper (if applicable). |proxyUserConfigs |No |Configures proxy users for the underlying Hadoop authentication mechanism. This configuration is expressed as a collection of key-value pairs (i.e., `property-name:value`). |clientBuilderFactory |No |The `HttpClientBuilderFactory` implementation used for the Solr internal communication. Only applicable for `ConfigurableInternodeAuthHadoopPlugin`. |=== diff --git a/solr/solr-ref-guide/src/implicit-requesthandlers.adoc b/solr/solr-ref-guide/src/implicit-requesthandlers.adoc index 7e5163df4dd..3c87f8c9554 100644 --- a/solr/solr-ref-guide/src/implicit-requesthandlers.adoc +++ b/solr/solr-ref-guide/src/implicit-requesthandlers.adoc @@ -53,7 +53,7 @@ Solr ships with many out-of-the-box RequestHandlers, which are called implicit b |<> |{solr-javadocs}/solr-core/org/apache/solr/handler/UpdateRequestHandler.html[UpdateRequestHandler] |`_UPDATE` |Add, delete and update indexed documents formatted as SolrXML, CSV, SolrJSON or javabin. |<> |{solr-javadocs}/solr-core/org/apache/solr/handler/UpdateRequestHandler.html[UpdateRequestHandler] |`_UPDATE_CSV` |Add and update CSV-formatted documents. |<> |{solr-javadocs}/solr-core/org/apache/solr/handler/UpdateRequestHandler.html[UpdateRequestHandler] |`_UPDATE_JSON` |Add, delete and update SolrJSON-formatted documents. -|<> |{solr-javadocs}/solr-core/org/apache/solr/handler/UpdateRequestHandler.html[UpdateRequestHandler] |`_UPDATE_JSON_DOCS ` |Add and update custom JSON-formatted documents. +|<> |{solr-javadocs}/solr-core/org/apache/solr/handler/UpdateRequestHandler.html[UpdateRequestHandler] |`_UPDATE_JSON_DOCS` |Add and update custom JSON-formatted documents. |=== [[ImplicitRequestHandlers-HowtoViewtheConfiguration]] diff --git a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc index e86e0651309..396243349ce 100644 --- a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc @@ -59,11 +59,11 @@ Since a Solr cluster requires internode communication, each node must also be ab [[KerberosAuthenticationPlugin-KerberizedZooKeeper]] === Kerberized ZooKeeper -When setting up a kerberized SolrCloud cluster, it is recommended to enable Kerberos security for Zookeeper as well. +When setting up a kerberized SolrCloud cluster, it is recommended to enable Kerberos security for ZooKeeper as well. -In such a setup, the client principal used to authenticate requests with Zookeeper can be shared for internode communication as well. This has the benefit of not needing to renew the ticket granting tickets (TGTs) separately, since the Zookeeper client used by Solr takes care of this. To achieve this, a single JAAS configuration (with the app name as Client) can be used for the Kerberos plugin as well as for the Zookeeper client. +In such a setup, the client principal used to authenticate requests with ZooKeeper can be shared for internode communication as well. This has the benefit of not needing to renew the ticket granting tickets (TGTs) separately, since the Zookeeper client used by Solr takes care of this. To achieve this, a single JAAS configuration (with the app name as Client) can be used for the Kerberos plugin as well as for the Zookeeper client. -See the <> section below for an example of starting Zookeeper in Kerberos mode. +See the <> section below for an example of starting ZooKeeper in Kerberos mode. [[KerberosAuthenticationPlugin-BrowserConfiguration]] === Browser Configuration @@ -126,7 +126,7 @@ kadmin.local: quit Copy the keytab file from the KDC server’s `/tmp/107.keytab` location to the Solr host at `/keytabs/107.keytab`. Repeat this step for each Solr node. -You might need to take similar steps to create a Zookeeper service principal and keytab if it has not already been set up. In that case, the example below shows a different service principal for ZooKeeper, so the above might be repeated with `zookeeper/host1` as the service principal for one of the nodes +You might need to take similar steps to create a ZooKeeper service principal and keytab if it has not already been set up. In that case, the example below shows a different service principal for ZooKeeper, so the above might be repeated with `zookeeper/host1` as the service principal for one of the nodes [[KerberosAuthenticationPlugin-ZooKeeperConfiguration]] === ZooKeeper Configuration @@ -191,7 +191,7 @@ More details on how to use a `/security.json` file in Solr are available in the [IMPORTANT] ==== -If you already have a `/security.json` file in Zookeeper, download the file, add or modify the authentication section and upload it back to ZooKeeper using the <> available in Solr. +If you already have a `/security.json` file in ZooKeeper, download the file, add or modify the authentication section and upload it back to ZooKeeper using the <> available in Solr. ==== [[KerberosAuthenticationPlugin-DefineaJAASConfigurationFile]] @@ -201,7 +201,7 @@ The JAAS configuration file defines the properties to use for authentication, su The following example can be copied and modified slightly for your environment. The location of the file can be anywhere on the server, but it will be referenced when starting Solr so it must be readable on the filesystem. The JAAS file may contain multiple sections for different users, but each section must have a unique name so it can be uniquely referenced in each application. -In the below example, we have created a JAAS configuration file with the name and path of `/home/foo/jaas-client.conf`. We will use this name and path when we define the Solr start parameters in the next section. Note that the client `principal` here is the same as the service principal. This will be used to authenticate internode requests and requests to Zookeeper. Make sure to use the correct `principal` hostname and the `keyTab` file path. +In the below example, we have created a JAAS configuration file with the name and path of `/home/foo/jaas-client.conf`. We will use this name and path when we define the Solr start parameters in the next section. Note that the client `principal` here is the same as the service principal. This will be used to authenticate internode requests and requests to ZooKeeper. Make sure to use the correct `principal` hostname and the `keyTab` file path. [source,plain] ---- @@ -242,7 +242,7 @@ While starting up Solr, the following host-specific parameters need to be passed |`solr.kerberos.cookie.portaware` |No |When set to true, cookies are differentiated based on host and port, as opposed to standard cookies which are not port aware. This should be set if more than one Solr node is hosted on the same host. The default is false. |`solr.kerberos.principal` |Yes |The service principal. |`solr.kerberos.keytab` |Yes |Keytab file path containing service principal credentials. -|`solr.kerberos.jaas.appname` |No |The app name (section name) within the JAAS configuration file which is required for internode communication. Default is `Client`, which is used for Zookeeper authentication as well. If different users are used for ZooKeeper and Solr, they will need to have separate sections in the JAAS configuration file. +|`solr.kerberos.jaas.appname` |No |The app name (section name) within the JAAS configuration file which is required for internode communication. Default is `Client`, which is used for ZooKeeper authentication as well. If different users are used for ZooKeeper and Solr, they will need to have separate sections in the JAAS configuration file. |`java.security.auth.login.config` |Yes |Path to the JAAS configuration file for configuring a Solr client for internode communication. |=== diff --git a/solr/solr-ref-guide/src/parallel-sql-interface.adoc b/solr/solr-ref-guide/src/parallel-sql-interface.adoc index c527e05d8a6..5dccfce08fd 100644 --- a/solr/solr-ref-guide/src/parallel-sql-interface.adoc +++ b/solr/solr-ref-guide/src/parallel-sql-interface.adoc @@ -239,7 +239,7 @@ The parallel SQL interface supports and pushes down most common SQL operators, s |> |Greater than |`fielda > 10` | `fielda:{10 TO *]` |>= |Greater than or equals |`fielda >= 10` | `fielda:[10 TO *]` |< |Less than |`fielda < 10` | `fielda:[* TO 10}` -|<= |Less than or equals |`fielda <= 10` | `fielda:[* TO 10]` +|\<= |Less than or equals |`fielda \<= 10` | `fielda:[* TO 10]` |=== Some operators that are not supported are BETWEEN, LIKE and IN. However, there are workarounds for BETWEEN and LIKE. diff --git a/solr/solr-ref-guide/src/performance-statistics-reference.adoc b/solr/solr-ref-guide/src/performance-statistics-reference.adoc index 3c5360138f2..50bc6011b64 100644 --- a/solr/solr-ref-guide/src/performance-statistics-reference.adoc +++ b/solr/solr-ref-guide/src/performance-statistics-reference.adoc @@ -20,7 +20,7 @@ This page explains some of the <> statistics that Solr exposes. -The same statistics are also exposed via the<> when statistics are requested. +The same statistics are also exposed via the <> when statistics are requested. These statistics are per core. When you are running in SolrCloud mode these statistics would co-relate to each performance of an individual replica. diff --git a/solr/solr-ref-guide/src/setting-up-an-external-zookeeper-ensemble.adoc b/solr/solr-ref-guide/src/setting-up-an-external-zookeeper-ensemble.adoc index 9aa4d05de9b..ab548362ebd 100644 --- a/solr/solr-ref-guide/src/setting-up-an-external-zookeeper-ensemble.adoc +++ b/solr/solr-ref-guide/src/setting-up-an-external-zookeeper-ensemble.adoc @@ -25,7 +25,7 @@ Shutting down a redundant Solr instance will also shut down its ZooKeeper server The solution to this problem is to set up an external ZooKeeper ensemble. Fortunately, while this process can seem intimidating due to the number of powerful options, setting up a simple ensemble is actually quite straightforward, as described below. .How Many ZooKeepers? -[quote,ZooKeeper Administrator's Guide,http://zookeeper.apache.org/doc/r3.4.6/zookeeperAdmin.html] +[quote,ZooKeeper Administrator's Guide,http://zookeeper.apache.org/doc/r3.4.10/zookeeperAdmin.html] ____ "For a ZooKeeper service to be active, there must be a majority of non-failing machines that can communicate with each other. *To create a deployment that can tolerate the failure of F machines, you should count on deploying 2xF+1 machines*. Thus, a deployment that consists of three machines can handle one failure, and a deployment of five machines can handle two failures. Note that a deployment of six machines can only handle two failures since three machines is not a majority. @@ -38,7 +38,7 @@ It is generally recommended to have an odd number of ZooKeeper servers in your e For example, if you only have two ZooKeeper nodes and one goes down, 50% of available servers is not a majority, so ZooKeeper will no longer serve requests. However, if you have three ZooKeeper nodes and one goes down, you have 66% of available servers available, and ZooKeeper will continue normally while you repair the one down node. If you have 5 nodes, you could continue operating with two down nodes if necessary. -More information on ZooKeeper clusters is available from the ZooKeeper documentation at http://zookeeper.apache.org/doc/r3.4.6/zookeeperAdmin.html#sc_zkMulitServerSetup. +More information on ZooKeeper clusters is available from the ZooKeeper documentation at http://zookeeper.apache.org/doc/r3.4.10/zookeeperAdmin.html#sc_zkMulitServerSetup. [[SettingUpanExternalZooKeeperEnsemble-DownloadApacheZooKeeper]] == Download Apache ZooKeeper @@ -49,7 +49,7 @@ The first step in setting up Apache ZooKeeper is, of course, to download the sof ==== When using stand-alone ZooKeeper, you need to take care to keep your version of ZooKeeper updated with the latest version distributed with Solr. Since you are using it as a stand-alone application, it does not get upgraded when you upgrade Solr. -Solr currently uses Apache ZooKeeper v3.4.6. +Solr currently uses Apache ZooKeeper v3.4.10. ==== [[SettingUpanExternalZooKeeperEnsemble-SettingUpaSingleZooKeeper]] @@ -92,7 +92,7 @@ Again, ZooKeeper provides a great deal of power through additional configuration Pointing Solr at the ZooKeeper instance you've created is a simple matter of using the `-z` parameter when using the bin/solr script. For example, in order to point the Solr instance to the ZooKeeper you've started on port 2181, this is what you'd need to do: -Starting `cloud` example with Zookeeper already running at port 2181 (with all other defaults): +Starting `cloud` example with ZooKeeper already running at port 2181 (with all other defaults): [source,bash] ---- diff --git a/solr/solr-ref-guide/src/shards-and-indexing-data-in-solrcloud.adoc b/solr/solr-ref-guide/src/shards-and-indexing-data-in-solrcloud.adoc index dff118bf1b5..d2dbcf7dd65 100644 --- a/solr/solr-ref-guide/src/shards-and-indexing-data-in-solrcloud.adoc +++ b/solr/solr-ref-guide/src/shards-and-indexing-data-in-solrcloud.adoc @@ -30,7 +30,7 @@ Before SolrCloud, Solr supported Distributed Search, which allowed one query to SolrCloud fixes all those problems. There is support for distributing both the index process and the queries automatically, and ZooKeeper provides failover and load balancing. Additionally, every shard can also have multiple replicas for additional robustness. -In SolrCloud there are no masters or slaves. Instead, every shard consists of at least one physical *replica*, exactly one of which is a *leader*. Leaders are automatically elected, initially on a first-come-first-served basis, and then based on the Zookeeper process described at http://zookeeper.apache.org/doc/trunk/recipes.html#sc_leaderElection[http://zookeeper.apache.org/doc/trunk/recipes.html#sc_leaderElection.]. +In SolrCloud there are no masters or slaves. Instead, every shard consists of at least one physical *replica*, exactly one of which is a *leader*. Leaders are automatically elected, initially on a first-come-first-served basis, and then based on the ZooKeeper process described at http://zookeeper.apache.org/doc/trunk/recipes.html#sc_leaderElection[http://zookeeper.apache.org/doc/trunk/recipes.html#sc_leaderElection.]. If a leader goes down, one of the other replicas is automatically elected as the new leader. diff --git a/solr/solr-ref-guide/src/solr-control-script-reference.adoc b/solr/solr-ref-guide/src/solr-control-script-reference.adoc index fd27d0a9169..4c915e3fc68 100644 --- a/solr/solr-ref-guide/src/solr-control-script-reference.adoc +++ b/solr/solr-ref-guide/src/solr-control-script-reference.adoc @@ -96,7 +96,7 @@ Sets the solr.solr.home system property; Solr will create core directories under This parameter is ignored when running examples (-e), as the solr.solr.home depends on which example is run. |`bin/solr start -s newHome` -|-v |Be more verbose. This changes the logging level of log4j from `INFO` to `DEBUG`., having the same effect as if you edited `log4j.properties` accordingly. |`bin/solr start -f -v` +|-v |Be more verbose. This changes the logging level of log4j from `INFO` to `DEBUG`, having the same effect as if you edited `log4j.properties` accordingly. |`bin/solr start -f -v` |-q |Be more quiet. This changes the logging level of log4j from `INFO` to `WARN`, having the same effect as if you edited `log4j.properties` accordingly. This can be useful in a production setting where you want to limit logging to warnings and errors. |`bin/solr start -f -q` |-V |Start Solr with verbose messages from the start script. |`bin/solr start -V` |-z |Start Solr with the defined ZooKeeper connection string. This option is only used with the -c option, to start Solr in SolrCloud mode. If this option is not provided, Solr will start the embedded ZooKeeper instance and use that instance for SolrCloud operations. |`bin/solr start -c -z server1:2181,server2:2181` @@ -577,7 +577,11 @@ The path to write the downloaded configuration set into. If just a name is suppl In either case, _pre-existing configurations at the destination will be overwritten!_ - |`-d directory_under_configsets` `-d /path/to/configset/destination` + a| +`-d directory_under_configsets` + +`-d /path/to/configset/destination` + |-z |The ZooKeeper connection string. Unnecessary if ZK_HOST is defined in `solr.in.sh` or `solr.in.cmd`. |`-z 123.321.23.43:2181` |=== diff --git a/solr/solr-ref-guide/src/solr-cores-and-solr-xml.adoc b/solr/solr-ref-guide/src/solr-cores-and-solr-xml.adoc index ee9a7c76c1f..4fb477f2212 100644 --- a/solr/solr-ref-guide/src/solr-cores-and-solr-xml.adoc +++ b/solr/solr-ref-guide/src/solr-cores-and-solr-xml.adoc @@ -23,7 +23,7 @@ In Solr, the term _core_ is used to refer to a single index and associated trans Cores can be created using `bin/solr` script or as part of SolrCloud collection creation using the APIs. Core-specific properties (such as the directories to use for the indexes or configuration files, the core name, and other options) are defined in a `core.properties` file. Any `core.properties` file in any directory of your Solr installation (or in a directory under where `solr_home` is defined) will be found by Solr and the defined properties will be used for the core named in the file. -In standalone mode, `solr.xml` must reside in `solr_home`. In SolrCloud mode, `solr.xml` will be loaded from Zookeeper if it exists, with fallback to `solr_home`. +In standalone mode, `solr.xml` must reside in `solr_home`. In SolrCloud mode, `solr.xml` will be loaded from ZooKeeper if it exists, with fallback to `solr_home`. [NOTE] ==== diff --git a/solr/solr-ref-guide/src/stream-evaluators.adoc b/solr/solr-ref-guide/src/stream-evaluators.adoc index 8571b0e1751..3702f391c95 100644 --- a/solr/solr-ref-guide/src/stream-evaluators.adoc +++ b/solr/solr-ref-guide/src/stream-evaluators.adoc @@ -335,7 +335,7 @@ The expressions below show the various ways in which you can use the `gt` evalua ---- gt(1,2) // 1 > 2 gt(1,fieldA) // 1 > fieldA -gt(fieldA,val(foo)) fieldA > "foo" +gt(fieldA,val(foo)) // fieldA > "foo" gt(add(fieldA,fieldB),6) // fieldA + fieldB > 6 ---- diff --git a/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc b/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc index 33d23620bb5..52804ed5e41 100644 --- a/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc +++ b/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc @@ -135,7 +135,7 @@ Request parameters can be substituted in configuration with placeholder `${datai + password="${dataimporter.request.jdbcpassword}" /> ---- These parameters can then be passed to the `full-import` command or defined in the `` section in `solrconfig.xml`. This example shows the parameters with the full-import command: diff --git a/solr/solr-ref-guide/src/working-with-dates.adoc b/solr/solr-ref-guide/src/working-with-dates.adoc index 5426c943f7c..f0ac2422931 100644 --- a/solr/solr-ref-guide/src/working-with-dates.adoc +++ b/solr/solr-ref-guide/src/working-with-dates.adoc @@ -53,7 +53,10 @@ As you can see, the date format includes colon characters separating the hours, This is normally an invalid query: `datefield:1972-05-20T17:33:18.772Z` -These are valid queries: `datefield:1972-05-20T17\:33\:18.772Z` `datefield:"1972-05-20T17:33:18.772Z"` `datefield:[1972-05-20T17:33:18.772Z TO *]` +These are valid queries: + +`datefield:1972-05-20T17\:33\:18.772Z` + +`datefield:"1972-05-20T17:33:18.772Z"` + +`datefield:[1972-05-20T17:33:18.772Z TO *]` ==== diff --git a/solr/solr-ref-guide/src/zookeeper-access-control.adoc b/solr/solr-ref-guide/src/zookeeper-access-control.adoc index 342021fe03e..d3d6b710e64 100644 --- a/solr/solr-ref-guide/src/zookeeper-access-control.adoc +++ b/solr/solr-ref-guide/src/zookeeper-access-control.adoc @@ -18,7 +18,7 @@ // specific language governing permissions and limitations // under the License. -This section describes using ZooKeeper access control lists (ACLs) with Solr. For information about ZooKeeper ACLs, see the ZooKeeper documentation at http://zookeeper.apache.org/doc/r3.4.6/zookeeperProgrammers.html#sc_ZooKeeperAccessControl. +This section describes using ZooKeeper access control lists (ACLs) with Solr. For information about ZooKeeper ACLs, see the ZooKeeper documentation at http://zookeeper.apache.org/doc/r3.4.10/zookeeperProgrammers.html#sc_ZooKeeperAccessControl. [[ZooKeeperAccessControl-AboutZooKeeperACLs]] == About ZooKeeper ACLs From 5844ed4ac95373cbdb512e84b8ad08f78c2baf57 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Fri, 9 Jun 2017 23:52:19 +0200 Subject: [PATCH 046/131] LUCENE-7854: Add a new DelimitedTermFrequencyTokenFilter that allows to mark tokens with a custom term frequency --- lucene/CHANGES.txt | 6 ++ .../DelimitedTermFrequencyTokenFilter.java | 75 ++++++++++++++++++ ...imitedTermFrequencyTokenFilterFactory.java | 53 +++++++++++++ ...he.lucene.analysis.util.TokenFilterFactory | 1 + ...DelimitedTermFrequencyTokenFilterTest.java | 77 +++++++++++++++++++ 5 files changed, 212 insertions(+) create mode 100644 lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilter.java create mode 100644 lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilterFactory.java create mode 100644 lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilterTest.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 02512434896..12e50001a97 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -18,6 +18,12 @@ New Features with a custom token stream allows indexing custom term frequencies (Mike McCandless) +* LUCENE-7866: Add a new DelimitedTermFrequencyTokenFilter that allows to + mark tokens with a custom term frequency (LUCENE-7854). It parses a numeric + value after a separator char ('|') at the end of each token and changes + the term frequency to this value. (Uwe Schindler, Robert Muir, + Mike McCandless) + API Changes * LUCENE-2605: Classic QueryParser no longer splits on whitespace by default. diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilter.java new file mode 100644 index 00000000000..e2095ad56a2 --- /dev/null +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilter.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.analysis.miscellaneous; + +import java.io.IOException; + +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; + + +/** + * Characters before the delimiter are the "token", the textual integer after is the term frequency. + * To use this {@code TokenFilter} the field must be indexed with + * {@link IndexOptions#DOCS_AND_FREQS} but no positions or offsets. + *

    + * For example, if the delimiter is '|', then for the string "foo|5", "foo" is the token + * and "5" is a term frequency. If there is no delimiter, the TokenFilter does not modify + * the term frequency. + *

    + * Note make sure your Tokenizer doesn't split on the delimiter, or this won't work + */ +public final class DelimitedTermFrequencyTokenFilter extends TokenFilter { + public static final char DEFAULT_DELIMITER = '|'; + + private final char delimiter; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final TermFrequencyAttribute tfAtt = addAttribute(TermFrequencyAttribute.class); + + + public DelimitedTermFrequencyTokenFilter(TokenStream input) { + this(input, DEFAULT_DELIMITER); + } + + public DelimitedTermFrequencyTokenFilter(TokenStream input, char delimiter) { + super(input); + this.delimiter = delimiter; + } + + @Override + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + final char[] buffer = termAtt.buffer(); + final int length = termAtt.length(); + for (int i = 0; i < length; i++) { + if (buffer[i] == delimiter) { + termAtt.setLength(i); // simply set a new length + i++; + tfAtt.setTermFrequency(ArrayUtil.parseInt(buffer, i, length - i)); + return true; + } + } + return true; + } + return false; + } +} diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilterFactory.java new file mode 100644 index 00000000000..af5c0fa8bdd --- /dev/null +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilterFactory.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.analysis.miscellaneous; + +import java.util.Map; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.util.TokenFilterFactory; + +/** + * Factory for {@link DelimitedTermFrequencyTokenFilter}. The field must have {@code omitPositions=true}. + * 

    + * <fieldType name="text_tfdl" class="solr.TextField" omitPositions="true">
+ *   <analyzer>
+ *     <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ *     <filter class="solr.DelimitedTermFrequencyTokenFilterFactory" delimiter="|"/>
+ *   </analyzer>
+ * </fieldType>
    + */ +public class DelimitedTermFrequencyTokenFilterFactory extends TokenFilterFactory { + public static final String DELIMITER_ATTR = "delimiter"; + + private final char delimiter; + + /** Creates a new DelimitedPayloadTokenFilterFactory */ + public DelimitedTermFrequencyTokenFilterFactory(Map args) { + super(args); + delimiter = getChar(args, DELIMITER_ATTR, DelimitedTermFrequencyTokenFilter.DEFAULT_DELIMITER); + if (!args.isEmpty()) { + throw new IllegalArgumentException("Unknown parameters: " + args); + } + } + + @Override + public DelimitedTermFrequencyTokenFilter create(TokenStream input) { + return new DelimitedTermFrequencyTokenFilter(input, delimiter); + } +} \ No newline at end of file diff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory index 4e33006a922..bc19c4ac320 100644 --- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory +++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory @@ -63,6 +63,7 @@ org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory org.apache.lucene.analysis.miscellaneous.CapitalizationFilterFactory org.apache.lucene.analysis.miscellaneous.CodepointCountFilterFactory org.apache.lucene.analysis.miscellaneous.DateRecognizerFilterFactory +org.apache.lucene.analysis.miscellaneous.DelimitedTermFrequencyTokenFilterFactory org.apache.lucene.analysis.miscellaneous.FingerprintFilterFactory org.apache.lucene.analysis.miscellaneous.FixBrokenOffsetsFilterFactory org.apache.lucene.analysis.miscellaneous.HyphenatedWordsFilterFactory diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilterTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilterTest.java new file mode 100644 index 00000000000..7609f6e3d91 --- /dev/null +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/DelimitedTermFrequencyTokenFilterTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.analysis.miscellaneous; + +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; + +public class DelimitedTermFrequencyTokenFilterTest extends BaseTokenStreamTestCase { + + public void testTermFrequency() throws Exception { + String test = "The quick|40 red|4 fox|06 jumped|1 over the lazy|2 brown|123 dogs|1024"; + DelimitedTermFrequencyTokenFilter filter = + new DelimitedTermFrequencyTokenFilter(whitespaceMockTokenizer(test)); + CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class); + TermFrequencyAttribute tfAtt = filter.getAttribute(TermFrequencyAttribute.class); + filter.reset(); + assertTermEquals("The", filter, termAtt, tfAtt, 1); + assertTermEquals("quick", filter, termAtt, tfAtt, 40); + assertTermEquals("red", filter, termAtt, tfAtt, 4); + assertTermEquals("fox", filter, termAtt, tfAtt, 6); + assertTermEquals("jumped", filter, termAtt, tfAtt, 1); + assertTermEquals("over", filter, termAtt, tfAtt, 1); + assertTermEquals("the", filter, termAtt, tfAtt, 1); + assertTermEquals("lazy", filter, termAtt, tfAtt, 2); + assertTermEquals("brown", filter, termAtt, tfAtt, 123); + assertTermEquals("dogs", filter, termAtt, tfAtt, 1024); + assertFalse(filter.incrementToken()); + filter.end(); + filter.close(); + } + + public void testInvalidNegativeTf() throws Exception { + String test = "foo bar|-20"; + DelimitedTermFrequencyTokenFilter filter = + new DelimitedTermFrequencyTokenFilter(whitespaceMockTokenizer(test)); + CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class); + TermFrequencyAttribute tfAtt = filter.getAttribute(TermFrequencyAttribute.class); + filter.reset(); + assertTermEquals("foo", filter, termAtt, tfAtt, 1); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, filter::incrementToken); + assertEquals("Term frequency must be 1 or greater; got -20", iae.getMessage()); + } + + public void testInvalidFloatTf() throws Exception { + String test = "foo bar|1.2"; + DelimitedTermFrequencyTokenFilter filter = + new DelimitedTermFrequencyTokenFilter(whitespaceMockTokenizer(test)); + CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class); + TermFrequencyAttribute tfAtt = filter.getAttribute(TermFrequencyAttribute.class); + filter.reset(); + assertTermEquals("foo", filter, termAtt, tfAtt, 1); + expectThrows(NumberFormatException.class, filter::incrementToken); + } + + void assertTermEquals(String expected, TokenStream stream, CharTermAttribute termAtt, TermFrequencyAttribute tfAtt, int expectedTf) throws Exception { + assertTrue(stream.incrementToken()); + assertEquals(expected, termAtt.toString()); + assertEquals(expectedTf, tfAtt.getTermFrequency()); + } +} From b3b8344a720f8f7df34523126a46fb95fd1661e7 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Fri, 9 Jun 2017 23:56:24 +0200 Subject: [PATCH 047/131] LUCENE-7866: Add a new DelimitedTermFrequencyTokenFilter that allows to mark tokens with a custom term frequency (2nd commit to fix issue number) --- lucene/CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 12e50001a97..05dc0f83d1d 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -21,8 +21,8 @@ New Features * LUCENE-7866: Add a new DelimitedTermFrequencyTokenFilter that allows to mark tokens with a custom term frequency (LUCENE-7854). It parses a numeric value after a separator char ('|') at the end of each token and changes - the term frequency to this value. (Uwe Schindler, Robert Muir, - Mike McCandless) + the term frequency to this value. (Uwe Schindler, Robert Muir, Mike + McCandless) API Changes From 072e72097cac1515199fb032b2b0cbdd9f6d6331 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Fri, 9 Jun 2017 16:40:32 -0500 Subject: [PATCH 048/131] SOLR-10863: remove feed.xml from HTML version of Ref Guide --- solr/solr-ref-guide/src/_config.yml.template | 6 +---- solr/solr-ref-guide/src/_includes/head.html | 2 -- solr/solr-ref-guide/src/feed.xml | 28 -------------------- 3 files changed, 1 insertion(+), 35 deletions(-) delete mode 100755 solr/solr-ref-guide/src/feed.xml diff --git a/solr/solr-ref-guide/src/_config.yml.template b/solr/solr-ref-guide/src/_config.yml.template index eac2d2812f8..0db495e78a8 100755 --- a/solr/solr-ref-guide/src/_config.yml.template +++ b/solr/solr-ref-guide/src/_config.yml.template @@ -54,11 +54,7 @@ defaults: layout: "post" search: true - -# the description is used in the feed.xml file -description: "The Apache Solr Reference Guide is the official documentation for the Apache Solr project." - -# needed for sitemap.xml and feed.xml files +# needed for sitemap.xml url: https://home.apache.org/~ctargett/RefGuidePOC/jekyll-full # Asciidoc settings - disabled so we can use asciidoctor instead diff --git a/solr/solr-ref-guide/src/_includes/head.html b/solr/solr-ref-guide/src/_includes/head.html index 47bf8679ba7..de0bd440ebd 100755 --- a/solr/solr-ref-guide/src/_includes/head.html +++ b/solr/solr-ref-guide/src/_includes/head.html @@ -31,5 +31,3 @@ - - diff --git a/solr/solr-ref-guide/src/feed.xml b/solr/solr-ref-guide/src/feed.xml deleted file mode 100755 index d9faae8e25b..00000000000 --- a/solr/solr-ref-guide/src/feed.xml +++ /dev/null @@ -1,28 +0,0 @@ ---- -search: exclude -layout: none ---- - - - - - {{ site.site_title | xml_escape }} - {{ site.description | xml_escape }} - {{ site.url }} - - {{ site.time | date_to_rfc822 }} - {{ site.time | date_to_rfc822 }} - Jekyll v{{ jekyll.version }} - {% for page in site.pages limit:10 %} - - {{ page.title | xml_escape }} - {{ page.content | xml_escape }} - {{ page.url | prepend: site.url }} - {{ page.url | prepend: site.url }} - {% for tag in page.tags %} - {{ tag | xml_escape }} - {% endfor %} - - {% endfor %} - - From f434e34d5c7b33783875e77f919c05f0cd806b3f Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Fri, 9 Jun 2017 17:06:27 -0500 Subject: [PATCH 049/131] SOLR-10616: parameterize Guide HTML path in sitemap.xml --- solr/solr-ref-guide/build.xml | 31 +++++++++++++------- solr/solr-ref-guide/src/_config.yml.template | 3 +- solr/solr-ref-guide/src/sitemap.xml | 2 +- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/solr/solr-ref-guide/build.xml b/solr/solr-ref-guide/build.xml index 4e009929de3..6a610b1e5f3 100644 --- a/solr/solr-ref-guide/build.xml +++ b/solr/solr-ref-guide/build.xml @@ -44,34 +44,45 @@ + + + + + + + + + + + diff --git a/solr/solr-ref-guide/src/_config.yml.template b/solr/solr-ref-guide/src/_config.yml.template index 0db495e78a8..9d40209324b 100755 --- a/solr/solr-ref-guide/src/_config.yml.template +++ b/solr/solr-ref-guide/src/_config.yml.template @@ -55,7 +55,7 @@ defaults: search: true # needed for sitemap.xml -url: https://home.apache.org/~ctargett/RefGuidePOC/jekyll-full +url: https://lucene.apache.org/solr/guide/ # Asciidoc settings - disabled so we can use asciidoctor instead asciidoc: {} @@ -71,6 +71,7 @@ asciidoc: {} solr-attributes: &solr-attributes-ref solr-guide-draft-status: "${solr-guide-draft-status}" solr-guide-version: "${solr-guide-version}" + solr-guide-version-path: "${solr-guide-version-path}" solr-docs-version: "${solr-docs-version}" solr-javadocs: "${solr-javadocs}" lucene-javadocs: "${lucene-javadocs}" diff --git a/solr/solr-ref-guide/src/sitemap.xml b/solr/solr-ref-guide/src/sitemap.xml index d5fa97aae93..ba2032d4027 100755 --- a/solr/solr-ref-guide/src/sitemap.xml +++ b/solr/solr-ref-guide/src/sitemap.xml @@ -10,7 +10,7 @@ search: exclude {% for page in site.pages %} {% unless page.search == "exclude" %} - {{site.url}}{{page.url}} + {{site.url}}{{site.solr-attributes.solr-guide-version-path}}{{page.url}} {% endunless %} {% endfor %} From 566fcfce1337d09553a85d17bb4f552b14768646 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Fri, 9 Jun 2017 19:43:01 -0400 Subject: [PATCH 050/131] SOLR-10761: Switch trie numeric/date fields to points in data-driven-enabled example and test schemas --- solr/CHANGES.txt | 3 ++ ...AddSchemaFieldsUpdateProcessorFactory.java | 12 +++---- ...ema-add-schema-fields-update-processor.xml | 22 +++++++++++++ .../solr/collection1/conf/schema-rest.xml | 33 ++++++++++++++++++- ...-schema-fields-update-processor-chains.xml | 30 ++++++++--------- .../conf/solrconfig-schemaless.xml | 6 ++-- .../schema/TestFieldCollectionResource.java | 10 +++--- ...chemaFieldsUpdateProcessorFactoryTest.java | 16 ++++----- solr/example/files/conf/managed-schema | 20 +++++++++++ solr/example/files/conf/solrconfig.xml | 6 ++-- .../basic_configs/conf/solrconfig.xml | 6 ++-- .../conf/solrconfig.xml | 6 ++-- solr/solr-ref-guide/src/schemaless-mode.adoc | 6 ++-- 13 files changed, 126 insertions(+), 50 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index e5eb0c19755..d99a075f4e0 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -389,6 +389,9 @@ Other Changes * SOLR-10501: Test sortMissing{First,Last} with points fields. (Steve Rowe) +* SOLR-10761: Switch trie numeric/date fields to points in data-driven-enabled example and test schemas. + (Steve Rowe) + ================== 6.6.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java index 47589729085..c4a122db157 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java @@ -92,31 +92,31 @@ import static org.apache.solr.core.ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG; * <str name="defaultFieldType">text_general</str> * <lst name="typeMapping"> * <str name="valueClass">Boolean</str> - * <str name="fieldType">boolean</str> + * <str name="fieldType">booleans</str> * </lst> * <lst name="typeMapping"> * <str name="valueClass">Integer</str> - * <str name="fieldType">tint</str> + * <str name="fieldType">pints</str> * </lst> * <lst name="typeMapping"> * <str name="valueClass">Float</str> - * <str name="fieldType">tfloat</str> + * <str name="fieldType">pfloats</str> * </lst> * <lst name="typeMapping"> * <str name="valueClass">Date</str> - * <str name="fieldType">tdate</str> + * <str name="fieldType">pdates</str> * </lst> * <lst name="typeMapping"> * <str name="valueClass">Long</str> * <str name="valueClass">Integer</str> - * <str name="fieldType">tlong</str> + * <str name="fieldType">plongs</str> * </lst> * <lst name="typeMapping"> * <arr name="valueClass"> * <str>Double</str> * <str>Float</str> * </arr> - * <str name="fieldType">tdouble</str> + * <str name="fieldType">pdoubles</str> * </lst> * </processor> */ diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-add-schema-fields-update-processor.xml b/solr/core/src/test-files/solr/collection1/conf/schema-add-schema-fields-update-processor.xml index 7f2494403fd..aa8d8601fdc 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-add-schema-fields-update-processor.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-add-schema-fields-update-processor.xml @@ -33,6 +33,17 @@     + + + + + + + + + + + @@ -44,6 +55,17 @@ + + + + + + + + + + + id diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml b/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml index 58b05eb1cb8..89c2623cbd0 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml @@ -90,6 +90,7 @@ + + + + + + + + + + + + + + @@ -681,6 +700,18 @@ + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml index e5745752a91..4541fbc521c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml @@ -40,27 +40,27 @@ java.lang.Integer - tint + pints java.lang.Float - tfloat + pfloats java.util.Date - tdate + pdates java.lang.Long java.lang.Integer - tlong + plongs java.lang.Double java.lang.Float - tdouble + pdoubles @@ -74,24 +74,24 @@ java.lang.Integer - tint + pints java.lang.Float - tfloat + pfloats java.util.Date - tdate + pdates java.lang.Long java.lang.Integer - tlong + plongs java.lang.Number - tdouble + pdoubles @@ -132,24 +132,24 @@ java.lang.Integer - tint + pints java.lang.Float - tfloat + pfloats java.util.Date - tdate + pdates java.lang.Long java.lang.Integer - tlong + plongs java.lang.Number - tdouble + pdoubles diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml index 8247d692db0..ac8321b726c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml @@ -85,16 +85,16 @@ java.util.Date - tdate + pdates java.lang.Long java.lang.Integer - tlong + plongs java.lang.Number - tdouble + pdoubles diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java index f3de92bee55..8d2b55536d4 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java @@ -77,11 +77,11 @@ public class TestFieldCollectionResource extends SolrRestletTestBase { "/fields/[0]/name=='HTMLstandardtok'", "/fields/[1]/name=='HTMLwhitetok'", "/fields/[2]/name=='_version_'", - "/fields/[98]/name=='*_d'", - "/fields/[97]/name=='*_f'", - "/fields/[96]/name=='*_b'", - "/fields/[95]/name=='*_t'", - "/fields/[94]/name=='*_l'" + "/fields/[108]/name=='*_d'", + "/fields/[107]/name=='*_f'", + "/fields/[106]/name=='*_b'", + "/fields/[105]/name=='*_t'", + "/fields/[104]/name=='*_l'" ); } diff --git a/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java index 8b59d0e51c9..64d1abf82de 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java @@ -67,7 +67,7 @@ public class AddSchemaFieldsUpdateProcessorFactoryTest extends UpdateProcessorTe assertNotNull(d); schema = h.getCore().getLatestSchema(); assertNotNull(schema.getFieldOrNull(fieldName)); - assertEquals("tdate", schema.getFieldType(fieldName).getTypeName()); + assertEquals("pdates", schema.getFieldType(fieldName).getTypeName()); } public void testSingleFieldRoundTrip() throws Exception { @@ -79,7 +79,7 @@ public class AddSchemaFieldsUpdateProcessorFactoryTest extends UpdateProcessorTe assertNotNull(d); schema = h.getCore().getLatestSchema(); assertNotNull(schema.getFieldOrNull(fieldName)); - assertEquals("tfloat", schema.getFieldType(fieldName).getTypeName()); + assertEquals("pfloats", schema.getFieldType(fieldName).getTypeName()); assertU(commit()); assertQ(req("id:2"), "//arr[@name='" + fieldName + "']/float[.='" + floatValue.toString() + "']"); } @@ -95,7 +95,7 @@ public class AddSchemaFieldsUpdateProcessorFactoryTest extends UpdateProcessorTe assertNotNull(d); schema = h.getCore().getLatestSchema(); assertNotNull(schema.getFieldOrNull(fieldName)); - assertEquals("tdouble", schema.getFieldType(fieldName).getTypeName()); + assertEquals("pdoubles", schema.getFieldType(fieldName).getTypeName()); assertU(commit()); assertQ(req("id:3") ,"//arr[@name='" + fieldName + "']/double[.='" + fieldValue1.toString() + "']" @@ -141,8 +141,8 @@ public class AddSchemaFieldsUpdateProcessorFactoryTest extends UpdateProcessorTe schema = h.getCore().getLatestSchema(); assertNotNull(schema.getFieldOrNull(fieldName1)); assertNotNull(schema.getFieldOrNull(fieldName2)); - assertEquals("tdouble", schema.getFieldType(fieldName1).getTypeName()); - assertEquals("tlong", schema.getFieldType(fieldName2).getTypeName()); + assertEquals("pdoubles", schema.getFieldType(fieldName1).getTypeName()); + assertEquals("plongs", schema.getFieldType(fieldName2).getTypeName()); assertU(commit()); assertQ(req("id:5") ,"//arr[@name='" + fieldName1 + "']/double[.='" + field1Value1.toString() + "']" @@ -194,10 +194,10 @@ public class AddSchemaFieldsUpdateProcessorFactoryTest extends UpdateProcessorTe assertNotNull(schema.getFieldOrNull(fieldName2)); assertNotNull(schema.getFieldOrNull(fieldName3)); assertNotNull(schema.getFieldOrNull(fieldName4)); - assertEquals("tdouble", schema.getFieldType(fieldName1).getTypeName()); - assertEquals("tlong", schema.getFieldType(fieldName2).getTypeName()); + assertEquals("pdoubles", schema.getFieldType(fieldName1).getTypeName()); + assertEquals("plongs", schema.getFieldType(fieldName2).getTypeName()); assertEquals("text", schema.getFieldType(fieldName3).getTypeName()); - assertEquals("tdate", schema.getFieldType(fieldName4).getTypeName()); + assertEquals("pdates", schema.getFieldType(fieldName4).getTypeName()); assertU(commit()); assertQ(req("id:6") ,"//arr[@name='" + fieldName1 + "']/double[.='" + field1Value1.toString() + "']" diff --git a/solr/example/files/conf/managed-schema b/solr/example/files/conf/managed-schema index 3c47c35a881..bf154f47c0b 100644 --- a/solr/example/files/conf/managed-schema +++ b/solr/example/files/conf/managed-schema @@ -47,6 +47,16 @@ + + + + + + + + + + @@ -493,6 +503,7 @@ + @@ -500,6 +511,11 @@ + + + + + @@ -512,6 +528,10 @@ + + + + diff --git a/solr/example/files/conf/solrconfig.xml b/solr/example/files/conf/solrconfig.xml index 71da5108944..35ec28edee2 100644 --- a/solr/example/files/conf/solrconfig.xml +++ b/solr/example/files/conf/solrconfig.xml @@ -1229,16 +1229,16 @@ java.util.Date - tdates + pdates java.lang.Long java.lang.Integer - tlongs + plongs java.lang.Number - tdoubles + pdoubles diff --git a/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml b/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml index 53f74c3d4e7..770ac60c411 100644 --- a/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml @@ -1223,16 +1223,16 @@ java.util.Date - tdates + pdates java.lang.Long java.lang.Integer - tlongs + plongs java.lang.Number - tdoubles + pdoubles diff --git a/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml b/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml index 48d47e3ee6c..33ef77320d9 100644 --- a/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml @@ -1222,16 +1222,16 @@ java.util.Date - tdates + pdates java.lang.Long java.lang.Integer - tlongs + plongs java.lang.Number - tdoubles + pdoubles diff --git a/solr/solr-ref-guide/src/schemaless-mode.adoc b/solr/solr-ref-guide/src/schemaless-mode.adoc index fdf97ba538c..45ae8dd7073 100644 --- a/solr/solr-ref-guide/src/schemaless-mode.adoc +++ b/solr/solr-ref-guide/src/schemaless-mode.adoc @@ -139,16 +139,16 @@ The UpdateRequestProcessorChain allows Solr to guess field types, and you can de java.util.Date - tdates + pdates java.lang.Long java.lang.Integer - tlongs + plongs java.lang.Number - tdoubles + pdoubles From 5c781d5679716dc46a28015c03ef9bcae824e58c Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 9 Jun 2017 23:39:23 -0400 Subject: [PATCH 051/131] SOLR-10836: igain, significantTerms, and tlogit assumed existing terms --- solr/CHANGES.txt | 5 +++++ .../org/apache/solr/search/IGainTermsQParserPlugin.java | 5 ++--- .../solr/search/SignificantTermsQParserPlugin.java | 9 ++++----- .../solr/search/TextLogisticRegressionQParserPlugin.java | 5 ++--- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d99a075f4e0..039a2c499d5 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -350,6 +350,11 @@ Bug Fixes * SOLR-10829: Fixed IndexSchema to enforce that uniqueKey can not be Points based for correctness (hossman) +* SOLR-10836: The query parsers igain, significantTerms, and tlogit (used by streaming expressions by + the same name) might throw a NullPointerException if the referenced field had no indexed data in some + shards. The fix included an optimization to use Solr's cached AtomicReader instead of re-calculating. + (David Smiley) + Optimizations ---------------------- * SOLR-10634: JSON Facet API: When a field/terms facet will retrieve all buckets (i.e. limit:-1) diff --git a/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java index bccbe065589..053f50f4055 100644 --- a/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java @@ -23,7 +23,6 @@ import java.util.TreeSet; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; @@ -162,8 +161,8 @@ public class IGainTermsQParserPlugin extends QParserPlugin { double pc = numPositiveDocs / numDocs; double entropyC = binaryEntropy(pc); - Terms terms = MultiFields.getFields(searcher.getIndexReader()).terms(field); - TermsEnum termsEnum = terms.iterator(); + Terms terms = ((SolrIndexSearcher)searcher).getSlowAtomicReader().terms(field); + TermsEnum termsEnum = terms == null ? TermsEnum.EMPTY : terms.iterator(); BytesRef term; PostingsEnum postingsEnum = null; while ((term = termsEnum.next()) != null) { diff --git a/solr/core/src/java/org/apache/solr/search/SignificantTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/SignificantTermsQParserPlugin.java index 9b9a46bb934..83463161abc 100644 --- a/solr/core/src/java/org/apache/solr/search/SignificantTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/SignificantTermsQParserPlugin.java @@ -19,12 +19,11 @@ package org.apache.solr.search; import java.io.IOException; -import java.util.TreeSet; -import java.util.List; import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -146,8 +145,8 @@ public class SignificantTermsQParserPlugin extends QParserPlugin { //TODO: Use a priority queue TreeSet topTerms = new TreeSet<>(); - Terms terms = MultiFields.getFields(searcher.getIndexReader()).terms(field); - TermsEnum termsEnum = terms.iterator(); + Terms terms = ((SolrIndexSearcher)searcher).getSlowAtomicReader().terms(field); + TermsEnum termsEnum = terms == null ? TermsEnum.EMPTY : terms.iterator(); BytesRef term; PostingsEnum postingsEnum = null; diff --git a/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java index c1b89061b22..842f746c56a 100644 --- a/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java @@ -26,7 +26,6 @@ import java.util.Map; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; @@ -173,8 +172,8 @@ public class TextLogisticRegressionQParserPlugin extends QParserPlugin { public void finish() throws IOException { Map docVectors = new HashMap<>(); - Terms terms = MultiFields.getFields(searcher.getIndexReader()).terms(trainingParams.feature); - TermsEnum termsEnum = terms.iterator(); + Terms terms = ((SolrIndexSearcher)searcher).getSlowAtomicReader().terms(trainingParams.feature); + TermsEnum termsEnum = terms == null ? TermsEnum.EMPTY : terms.iterator(); PostingsEnum postingsEnum = null; int termIndex = 0; for (String termStr : trainingParams.terms) { From 1b9f060e25f09445e6f60956793d049dfca7a774 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sat, 10 Jun 2017 10:30:54 +0200 Subject: [PATCH 052/131] LUCENE-7866: Exclude DelimitedTermFrequencyTokenFilter from random data tests in random chains and factory tests --- .../lucene/analysis/core/TestFactories.java | 46 +++++++++++++------ .../analysis/core/TestRandomChains.java | 3 ++ 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java index 956a3941b78..499774e5999 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java @@ -21,13 +21,17 @@ import java.io.IOException; import java.io.Reader; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.miscellaneous.DelimitedTermFrequencyTokenFilterFactory; import org.apache.lucene.analysis.util.AbstractAnalysisFactory; import org.apache.lucene.analysis.util.CharFilterFactory; import org.apache.lucene.analysis.util.MultiTermAwareComponent; @@ -49,6 +53,12 @@ import org.apache.lucene.util.Version; // TODO: fix this to use CustomAnalyzer instead of its own FactoryAnalyzer public class TestFactories extends BaseTokenStreamTestCase { + + /** Factories that are excluded from testing it with random data */ + private static final Set> EXCLUDE_FACTORIES_RANDOM_DATA = new HashSet<>(Arrays.asList( + DelimitedTermFrequencyTokenFilterFactory.class + )); + public void test() throws IOException { for (String tokenizer : TokenizerFactory.availableTokenizers()) { doTestTokenizer(tokenizer); @@ -77,11 +87,13 @@ public class TestFactories extends BaseTokenStreamTestCase { assertFalse(mtc instanceof CharFilterFactory); } - // beast it just a little, it shouldnt throw exceptions: - // (it should have thrown them in initialize) - Analyzer a = new FactoryAnalyzer(factory, null, null); - checkRandomData(random(), a, 20, 20, false, false); - a.close(); + if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) { + // beast it just a little, it shouldnt throw exceptions: + // (it should have thrown them in initialize) + Analyzer a = new FactoryAnalyzer(factory, null, null); + checkRandomData(random(), a, 20, 20, false, false); + a.close(); + } } } @@ -99,11 +111,13 @@ public class TestFactories extends BaseTokenStreamTestCase { assertTrue(mtc instanceof TokenFilterFactory); } - // beast it just a little, it shouldnt throw exceptions: - // (it should have thrown them in initialize) - Analyzer a = new FactoryAnalyzer(assertingTokenizer, factory, null); - checkRandomData(random(), a, 20, 20, false, false); - a.close(); + if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) { + // beast it just a little, it shouldnt throw exceptions: + // (it should have thrown them in initialize) + Analyzer a = new FactoryAnalyzer(assertingTokenizer, factory, null); + checkRandomData(random(), a, 20, 20, false, false); + a.close(); + } } } @@ -121,11 +135,13 @@ public class TestFactories extends BaseTokenStreamTestCase { assertTrue(mtc instanceof CharFilterFactory); } - // beast it just a little, it shouldnt throw exceptions: - // (it should have thrown them in initialize) - Analyzer a = new FactoryAnalyzer(assertingTokenizer, null, factory); - checkRandomData(random(), a, 20, 20, false, false); - a.close(); + if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) { + // beast it just a little, it shouldnt throw exceptions: + // (it should have thrown them in initialize) + Analyzer a = new FactoryAnalyzer(assertingTokenizer, null, factory); + checkRandomData(random(), a, 20, 20, false, false); + a.close(); + } } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java index 34c31d2dc58..0162ac74687 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java @@ -73,6 +73,7 @@ import org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter; import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree; import org.apache.lucene.analysis.hunspell.Dictionary; import org.apache.lucene.analysis.hunspell.TestHunspellStemFilter; +import org.apache.lucene.analysis.miscellaneous.DelimitedTermFrequencyTokenFilter; import org.apache.lucene.analysis.miscellaneous.HyphenatedWordsFilter; import org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilter; import org.apache.lucene.analysis.miscellaneous.LimitTokenOffsetFilter; @@ -159,6 +160,8 @@ public class TestRandomChains extends BaseTokenStreamTestCase { WordDelimiterFilter.class, // Cannot correct offsets when a char filter had changed them: WordDelimiterGraphFilter.class, + // requires a special encoded token value, so it may fail with random data: + DelimitedTermFrequencyTokenFilter.class, // clones of core's filters: org.apache.lucene.analysis.core.StopFilter.class, org.apache.lucene.analysis.core.LowerCaseFilter.class)) { From f43f89ee6ca276644f4cfcd751f81cd14c92e02a Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Sun, 11 Jun 2017 09:42:28 +0930 Subject: [PATCH 053/131] SOLR-10868: add solrj resource dir to module paths --- dev-tools/idea/solr/solrj/src/java/solrj.iml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev-tools/idea/solr/solrj/src/java/solrj.iml b/dev-tools/idea/solr/solrj/src/java/solrj.iml index bf819133f0c..e5dde1d705c 100644 --- a/dev-tools/idea/solr/solrj/src/java/solrj.iml +++ b/dev-tools/idea/solr/solrj/src/java/solrj.iml @@ -6,6 +6,9 @@ + + + From d7808ebc6029a51c6f80e6700ca53f7c9a6523f0 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sun, 11 Jun 2017 14:25:03 +0200 Subject: [PATCH 054/131] LUCENE-7866: Apply changes also to cloned test (TODO: fix this!) --- .../lucene/analysis/ja/TestFactories.java | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestFactories.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestFactories.java index bae8ffc8f53..0a467a7e178 100644 --- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestFactories.java +++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestFactories.java @@ -21,13 +21,17 @@ import java.io.IOException; import java.io.Reader; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.miscellaneous.DelimitedTermFrequencyTokenFilterFactory; import org.apache.lucene.analysis.util.AbstractAnalysisFactory; import org.apache.lucene.analysis.util.CharFilterFactory; import org.apache.lucene.analysis.util.MultiTermAwareComponent; @@ -46,6 +50,12 @@ import org.apache.lucene.util.Version; // TODO: fix this to use CustomAnalyzer instead of its own FactoryAnalyzer public class TestFactories extends BaseTokenStreamTestCase { + + /** Factories that are excluded from testing it with random data */ + private static final Set> EXCLUDE_FACTORIES_RANDOM_DATA = new HashSet<>(Arrays.asList( + DelimitedTermFrequencyTokenFilterFactory.class + )); + public void test() throws IOException { for (String tokenizer : TokenizerFactory.availableTokenizers()) { doTestTokenizer(tokenizer); @@ -74,11 +84,13 @@ public class TestFactories extends BaseTokenStreamTestCase { assertFalse(mtc instanceof CharFilterFactory); } - // beast it just a little, it shouldnt throw exceptions: - // (it should have thrown them in initialize) - Analyzer a = new FactoryAnalyzer(factory, null, null); - checkRandomData(random(), a, 20, 20, false, false); - a.close(); + if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) { + // beast it just a little, it shouldnt throw exceptions: + // (it should have thrown them in initialize) + Analyzer a = new FactoryAnalyzer(factory, null, null); + checkRandomData(random(), a, 20, 20, false, false); + a.close(); + } } } @@ -96,11 +108,13 @@ public class TestFactories extends BaseTokenStreamTestCase { assertTrue(mtc instanceof TokenFilterFactory); } - // beast it just a little, it shouldnt throw exceptions: - // (it should have thrown them in initialize) - Analyzer a = new FactoryAnalyzer(assertingTokenizer, factory, null); - checkRandomData(random(), a, 20, 20, false, false); - a.close(); + if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) { + // beast it just a little, it shouldnt throw exceptions: + // (it should have thrown them in initialize) + Analyzer a = new FactoryAnalyzer(assertingTokenizer, factory, null); + checkRandomData(random(), a, 20, 20, false, false); + a.close(); + } } } @@ -118,11 +132,13 @@ public class TestFactories extends BaseTokenStreamTestCase { assertTrue(mtc instanceof CharFilterFactory); } - // beast it just a little, it shouldnt throw exceptions: - // (it should have thrown them in initialize) - Analyzer a = new FactoryAnalyzer(assertingTokenizer, null, factory); - checkRandomData(random(), a, 20, 20, false, false); - a.close(); + if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) { + // beast it just a little, it shouldnt throw exceptions: + // (it should have thrown them in initialize) + Analyzer a = new FactoryAnalyzer(assertingTokenizer, null, factory); + checkRandomData(random(), a, 20, 20, false, false); + a.close(); + } } } From 95841d9ed406def6be62ab039dba8c4dd4b552a8 Mon Sep 17 00:00:00 2001 From: Erick Date: Sun, 11 Jun 2017 12:13:25 -0700 Subject: [PATCH 055/131] SOLR-10857: Solr loads UNLOADed core on request. --- solr/CHANGES.txt | 7 +++ .../org/apache/solr/core/CoreContainer.java | 9 ++-- .../handler/admin/CoreAdminHandlerTest.java | 52 +++++++++++++++++++ 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 039a2c499d5..275710da640 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -397,6 +397,13 @@ Other Changes * SOLR-10761: Switch trie numeric/date fields to points in data-driven-enabled example and test schemas. (Steve Rowe) +================== 6.6.1 ================== + +Bug Fixes +---------------------- + +* SOLR-10857: standalone Solr loads UNLOADed core on request (Erick Erickson, Mikhail Khludnev) + ================== 6.6.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 37842f87e62..6055acf6efc 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -1285,6 +1285,8 @@ public class CoreContainer { boolean close = solrCores.isLoadedNotPendingClose(name); SolrCore core = solrCores.remove(name); + + solrCores.removeCoreDescriptor(cd); coresLocator.delete(this, cd); if (core == null) { // transient core @@ -1298,8 +1300,8 @@ public class CoreContainer { if (zkSys.getZkController() != null) { // cancel recovery in cloud mode core.getSolrCoreState().cancelRecovery(); - if (core.getCoreDescriptor().getCloudDescriptor().getReplicaType() == Replica.Type.PULL - || core.getCoreDescriptor().getCloudDescriptor().getReplicaType() == Replica.Type.TLOG) { + if (cd.getCloudDescriptor().getReplicaType() == Replica.Type.PULL + || cd.getCloudDescriptor().getReplicaType() == Replica.Type.TLOG) { // Stop replication if this is part of a pull/tlog replica before closing the core zkSys.getZkController().stopReplicationFromLeader(name); } @@ -1319,9 +1321,6 @@ public class CoreContainer { throw new SolrException(ErrorCode.SERVER_ERROR, "Error unregistering core [" + name + "] from cloud state", e); } } - if (deleteInstanceDir) { // we aren't going to reload this if we delete the instance dir. - solrCores.removeCoreDescriptor(cd); - } } public void rename(String name, String toName) { diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java index a81cf13389d..f1770f8acbe 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java @@ -27,10 +27,12 @@ import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; import org.apache.commons.io.FileUtils; import org.apache.lucene.util.Constants; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.CoreStatus; +import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CoreAdminParams; @@ -282,6 +284,56 @@ public class CoreAdminHandlerTest extends SolrTestCaseJ4 { } + @Test + public void testUnloadForever() throws Exception { + File solrHomeDirectory = new File(initCoreDataDir, getClass().getName() + "-corex-" + + System.nanoTime()); + solrHomeDirectory.mkdirs(); + copySolrHomeToTemp(solrHomeDirectory, "corex"); + File corex = new File(solrHomeDirectory, "corex"); + FileUtils.write(new File(corex, "core.properties"), "", StandardCharsets.UTF_8); + JettySolrRunner runner = new JettySolrRunner(solrHomeDirectory.getAbsolutePath(), buildJettyConfig("/solr")); + runner.start(); + + try (HttpSolrClient client = getHttpSolrClient(runner.getBaseUrl() + "/corex")) { + client.setConnectionTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); + client.setSoTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "123"); + client.add(doc); + client.commit(); + } + + try (HttpSolrClient client = getHttpSolrClient(runner.getBaseUrl() + "/corex")) { + client.setConnectionTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); + client.setSoTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); + QueryResponse result = client.query(new SolrQuery("id:*")); + assertEquals(1,result.getResults().getNumFound()); + } + + try (HttpSolrClient client = getHttpSolrClient(runner.getBaseUrl().toString())) { + client.setConnectionTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); + client.setSoTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); + CoreAdminRequest.Unload req = new CoreAdminRequest.Unload(false); + req.setDeleteInstanceDir(false);//random().nextBoolean()); + req.setCoreName("corex"); + req.process(client); + } + + try (HttpSolrClient client = getHttpSolrClient(runner.getBaseUrl() + "/corex")) { + client.setConnectionTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); + client.setSoTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT*1000); + QueryResponse result = client.query(new SolrQuery("id:*")); + //assertEquals(1,result.getResults().getNumFound()); + fail("expect 404"); + }catch(Exception e){ + e.printStackTrace(); + } + finally{ + runner.stop(); + } + } + @Test public void testDeleteInstanceDirAfterCreateFailure() throws Exception { assumeFalse("Ignore test on windows because it does not delete data directory immediately after unload", Constants.WINDOWS); From 6075956b65ab378f67c3c7c2d3116ec8d43a312e Mon Sep 17 00:00:00 2001 From: Cao Manh Dat Date: Mon, 12 Jun 2017 10:32:45 +0700 Subject: [PATCH 056/131] SOLR-10715: /v2/ should not be an alias for /v2/collections --- .../java/org/apache/solr/api/V2HttpCall.java | 11 ++++- .../solr/handler/V2ApiIntegrationTest.java | 8 +++ .../apache/solr/handler/V2StandaloneTest.java | 49 +++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/handler/V2StandaloneTest.java diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java index baf4abc9972..da2781e967b 100644 --- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java +++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java @@ -80,8 +80,15 @@ public class V2HttpCall extends HttpSolrCall { try { pieces = getPathSegments(path); if (pieces.size() == 0) { - prefix = "c"; - path = "/c"; + api = new Api(null) { + @Override + public void call(SolrQueryRequest req, SolrQueryResponse rsp) { + rsp.add("documentation", "https://cwiki.apache.org/confluence/display/solr/v2+API"); + rsp.add("description", "V2 API root path"); + } + }; + initAdminRequest(path); + return; } else { prefix = pieces.get(0); } diff --git a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java index 70b7f8a9757..4741ef4edac 100644 --- a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java @@ -51,6 +51,14 @@ public class V2ApiIntegrationTest extends SolrCloudTestCase { .process(cluster.getSolrClient()); } + @Test + public void testWelcomeMessage() throws Exception { + NamedList res = cluster.getSolrClient().request( + new V2Request.Builder("").build()); + NamedList header = (NamedList) res.get("responseHeader"); + assertEquals(0, header.get("status")); + } + @Test public void testIntrospect() throws Exception { ModifiableSolrParams params = new ModifiableSolrParams(); diff --git a/solr/core/src/test/org/apache/solr/handler/V2StandaloneTest.java b/solr/core/src/test/org/apache/solr/handler/V2StandaloneTest.java new file mode 100644 index 00000000000..2d5b2a96fa1 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/V2StandaloneTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler; + +import java.io.File; + +import org.apache.commons.io.FileUtils; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.common.util.NamedList; +import org.junit.Test; + +public class V2StandaloneTest extends SolrTestCaseJ4{ + + @Test + public void testWelcomeMessage() throws Exception { + File solrHomeTmp = createTempDir().toFile().getAbsoluteFile(); + FileUtils.copyDirectory(new File(TEST_HOME(), "configsets/minimal/conf"), new File(solrHomeTmp,"/conf")); + FileUtils.copyFile(new File(TEST_HOME(), "solr.xml"), new File(solrHomeTmp, "solr.xml")); + + JettySolrRunner jetty = new JettySolrRunner(solrHomeTmp.getAbsolutePath(), buildJettyConfig("/solr")); + jetty.start(); + + try (HttpSolrClient client = getHttpSolrClient(buildUrl(jetty.getLocalPort(),"/solr/"))) { + NamedList res = client.request(new V2Request.Builder("/").build()); + NamedList header = (NamedList) res.get("responseHeader"); + assertEquals(0, header.get("status")); + } + + jetty.stop(); + } +} From 41d12df6d247b678b1a24c26567a08c2135b68ce Mon Sep 17 00:00:00 2001 From: Cao Manh Dat Date: Mon, 12 Jun 2017 10:46:38 +0700 Subject: [PATCH 057/131] SOLR-10715: Welcome message should work for _introspect --- solr/CHANGES.txt | 2 ++ solr/core/src/java/org/apache/solr/api/V2HttpCall.java | 2 +- .../test/org/apache/solr/handler/V2ApiIntegrationTest.java | 4 ++++ .../src/test/org/apache/solr/handler/V2StandaloneTest.java | 4 ++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 275710da640..6eb2152f2e7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -156,6 +156,8 @@ Bug Fixes * SOLR-10223: Allow running examples as root on Linux with -force option (janhoy) +* SOLR-10715: /v2/ should not be an alias for /v2/collections (Cao Manh Dat) + Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java index da2781e967b..5e7c0b16e42 100644 --- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java +++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java @@ -79,7 +79,7 @@ public class V2HttpCall extends HttpSolrCall { String fullPath = path = path.substring(7);//strip off '/____v2' try { pieces = getPathSegments(path); - if (pieces.size() == 0) { + if (pieces.size() == 0 || (pieces.size() == 1 && path.endsWith(CommonParams.INTROSPECT))) { api = new Api(null) { @Override public void call(SolrQueryRequest req, SolrQueryResponse rsp) { diff --git a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java index 4741ef4edac..66f5dbe7a57 100644 --- a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java @@ -57,6 +57,10 @@ public class V2ApiIntegrationTest extends SolrCloudTestCase { new V2Request.Builder("").build()); NamedList header = (NamedList) res.get("responseHeader"); assertEquals(0, header.get("status")); + + res = cluster.getSolrClient().request(new V2Request.Builder("/_introspect").build()); + header = (NamedList) res.get("responseHeader"); + assertEquals(0, header.get("status")); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/V2StandaloneTest.java b/solr/core/src/test/org/apache/solr/handler/V2StandaloneTest.java index 2d5b2a96fa1..ccf2a580dcf 100644 --- a/solr/core/src/test/org/apache/solr/handler/V2StandaloneTest.java +++ b/solr/core/src/test/org/apache/solr/handler/V2StandaloneTest.java @@ -42,6 +42,10 @@ public class V2StandaloneTest extends SolrTestCaseJ4{ NamedList res = client.request(new V2Request.Builder("/").build()); NamedList header = (NamedList) res.get("responseHeader"); assertEquals(0, header.get("status")); + + res = client.request(new V2Request.Builder("/_introspect").build()); + header = (NamedList) res.get("responseHeader"); + assertEquals(0, header.get("status")); } jetty.stop(); From e11eb63f60acf78278ba88cb777b17bb1daba314 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Mon, 12 Jun 2017 18:01:09 +0930 Subject: [PATCH 058/131] SOLR-10870: SolrRequestParsers does unnecessary check for /config prefix --- .../org/apache/solr/servlet/SolrRequestParsers.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java index 9aae8e9f2a9..1300bd9872b 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java @@ -16,8 +16,7 @@ */ package org.apache.solr.servlet; -import static org.apache.solr.common.params.CommonParams.PATH; - +import javax.servlet.http.HttpServletRequest; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -39,8 +38,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import javax.servlet.http.HttpServletRequest; - import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.apache.commons.fileupload.servlet.ServletFileUpload; @@ -53,6 +50,7 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.MultiMapSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.common.util.FastInputStream; @@ -61,10 +59,11 @@ import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; -import org.apache.solr.common.util.CommandOperation; import org.apache.solr.util.RTimerTree; import org.apache.solr.util.SolrFileCleaningTracker; +import static org.apache.solr.common.params.CommonParams.PATH; + public class SolrRequestParsers { @@ -746,10 +745,6 @@ public class SolrRequestParsers if (idx >= 0 && uri.endsWith("/schema") || uri.contains("/schema/")) { restletPath = true; } - idx = uri.indexOf("/config"); - if (idx >= 0 && uri.endsWith("/config") || uri.contains("/config/")) { - restletPath = true; - } if (restletPath) { return parseQueryString(req.getQueryString()); From 661a3c46d42114b5d3c3f20b0c412b4b5deb5afb Mon Sep 17 00:00:00 2001 From: Cao Manh Dat Date: Mon, 12 Jun 2017 16:44:42 +0700 Subject: [PATCH 059/131] SOLR-10715: Move change entry to 6.7.0 --- solr/CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 6eb2152f2e7..9ba932fdab9 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -156,8 +156,6 @@ Bug Fixes * SOLR-10223: Allow running examples as root on Linux with -force option (janhoy) -* SOLR-10715: /v2/ should not be an alias for /v2/collections (Cao Manh Dat) - Optimizations ---------------------- @@ -357,6 +355,8 @@ Bug Fixes shards. The fix included an optimization to use Solr's cached AtomicReader instead of re-calculating. (David Smiley) +* SOLR-10715: /v2/ should not be an alias for /v2/collections (Cao Manh Dat) + Optimizations ---------------------- * SOLR-10634: JSON Facet API: When a field/terms facet will retrieve all buckets (i.e. limit:-1) From b2ea95c9ecbf68c35ff485b0f16ed756a6105487 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Mon, 12 Jun 2017 11:48:21 +0200 Subject: [PATCH 060/131] SOLR-10849: MoreLikeThisComponent should expose setMaxDocFreqPct (maxDoc frequency percentage). --- solr/CHANGES.txt | 3 +++ .../java/org/apache/solr/handler/MoreLikeThisHandler.java | 7 +++++++ .../org/apache/solr/common/params/MoreLikeThisParams.java | 1 + 3 files changed, 11 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 9ba932fdab9..e9a22db2953 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -284,6 +284,9 @@ Upgrade Notes New Features ---------------------- +* SOLR-10849: MoreLikeThisComponent should expose setMaxDocFreqPct (maxDoc + frequency percentage). (Dawid Weiss) + * SOLR-10307: Allow Passing SSL passwords through environment variables. (Mano Kovacs via Mark Miller) * SOLR-10721: Provide a way to know when Core Discovery is finished and when all async cores are done loading diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index 50ea711e9d5..0d1213695b1 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -335,6 +335,13 @@ public class MoreLikeThisHandler extends RequestHandlerBase mlt.setMaxQueryTerms( params.getInt(MoreLikeThisParams.MAX_QUERY_TERMS, MoreLikeThis.DEFAULT_MAX_QUERY_TERMS)); mlt.setMaxNumTokensParsed(params.getInt(MoreLikeThisParams.MAX_NUM_TOKENS_PARSED, MoreLikeThis.DEFAULT_MAX_NUM_TOKENS_PARSED)); mlt.setBoost( params.getBool(MoreLikeThisParams.BOOST, false ) ); + + // There is no default for maxDocFreqPct. Also, it's a bit oddly expressed as an integer value + // (percentage of the collection's documents count). We keep Lucene's convention here. + if (params.getInt(MoreLikeThisParams.MAX_DOC_FREQ_PCT) != null) { + mlt.setMaxDocFreqPct(params.getInt(MoreLikeThisParams.MAX_DOC_FREQ_PCT)); + } + boostFields = SolrPluginUtils.parseFieldBoosts(params.getParams(MoreLikeThisParams.QF)); } diff --git a/solr/solrj/src/java/org/apache/solr/common/params/MoreLikeThisParams.java b/solr/solrj/src/java/org/apache/solr/common/params/MoreLikeThisParams.java index c898fdb8dae..919f786ee06 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/MoreLikeThisParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/MoreLikeThisParams.java @@ -31,6 +31,7 @@ public interface MoreLikeThisParams public final static String SIMILARITY_FIELDS = PREFIX + "fl"; public final static String MIN_TERM_FREQ = PREFIX + "mintf"; public final static String MAX_DOC_FREQ = PREFIX + "maxdf"; + public final static String MAX_DOC_FREQ_PCT = PREFIX + "maxdfpct"; public final static String MIN_DOC_FREQ = PREFIX + "mindf"; public final static String MIN_WORD_LEN = PREFIX + "minwl"; public final static String MAX_WORD_LEN = PREFIX + "maxwl"; From 5a737a3aab969b120a84dbc7cd7ed351796576b3 Mon Sep 17 00:00:00 2001 From: Erick Erickson Date: Mon, 12 Jun 2017 09:40:34 -0700 Subject: [PATCH 061/131] SOLR-10857: Solr loads UNLOADed core on request, cleaned up printStackTrace --- .../handler/admin/CoreAdminHandlerTest.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java index f1770f8acbe..06c30e63421 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java @@ -320,18 +320,16 @@ public class CoreAdminHandlerTest extends SolrTestCaseJ4 { req.process(client); } - try (HttpSolrClient client = getHttpSolrClient(runner.getBaseUrl() + "/corex")) { - client.setConnectionTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); - client.setSoTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT*1000); - QueryResponse result = client.query(new SolrQuery("id:*")); - //assertEquals(1,result.getResults().getNumFound()); - fail("expect 404"); - }catch(Exception e){ - e.printStackTrace(); - } - finally{ - runner.stop(); - } + HttpSolrClient.RemoteSolrException rse = expectThrows(HttpSolrClient.RemoteSolrException.class, () -> { + try (HttpSolrClient client = getHttpSolrClient(runner.getBaseUrl() + "/corex")) { + client.setConnectionTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); + client.setSoTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT * 1000); + client.query(new SolrQuery("id:*")); + } finally { + runner.stop(); + } + }); + assertTrue(rse.getMessage().contains("Can not find: /solr/corex/select")); } @Test From 0411504dd8dfccb2d997546759c1f26d7bb0eae9 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Mon, 12 Jun 2017 12:30:51 -0500 Subject: [PATCH 062/131] SOLR-10871: remove backticks for monospace type in headings --- .../src/common-query-parameters.adoc | 30 +++++------ solr/solr-ref-guide/src/config-api.adoc | 2 +- .../src/configuring-solrconfig-xml.adoc | 2 +- solr/solr-ref-guide/src/coreadmin-api.adoc | 8 +-- ...ir-and-directoryfactory-in-solrconfig.adoc | 2 +- solr/solr-ref-guide/src/de-duplication.adoc | 4 +- .../detecting-languages-during-indexing.adoc | 2 +- ...istributed-search-with-index-sharding.adoc | 2 +- solr/solr-ref-guide/src/enabling-ssl.adoc | 6 +-- .../src/exporting-result-sets.adoc | 2 +- solr/solr-ref-guide/src/faceting.adoc | 54 +++++++++---------- ...field-type-definitions-and-properties.adoc | 2 +- solr/solr-ref-guide/src/highlighting.adoc | 4 +- .../solr-ref-guide/src/index-replication.adoc | 2 +- .../src/indexconfig-in-solrconfig.adoc | 14 ++--- .../src/introduction-to-solr-indexing.adoc | 2 +- .../src/local-parameters-in-queries.adoc | 2 +- .../major-changes-from-solr-5-to-solr-6.adoc | 2 +- solr/solr-ref-guide/src/merging-indexes.adoc | 2 +- .../src/near-real-time-searching.adoc | 10 ++-- .../src/parallel-sql-interface.adoc | 16 +++--- .../src/query-settings-in-solrconfig.adoc | 22 ++++---- .../read-and-write-side-fault-tolerance.adoc | 4 +- .../src/request-parameters-api.adoc | 2 +- .../src/requestdispatcher-in-solrconfig.adoc | 8 +-- solr/solr-ref-guide/src/response-writers.adoc | 8 +-- ...hema-factory-definition-in-solrconfig.adoc | 6 +-- solr/solr-ref-guide/src/spatial-search.adoc | 6 +-- solr/solr-ref-guide/src/spell-checking.adoc | 40 +++++++------- .../src/the-dismax-query-parser.adoc | 20 +++---- .../src/the-extended-dismax-query-parser.adoc | 26 ++++----- .../src/the-query-elevation-component.adoc | 14 ++--- .../src/the-standard-query-parser.adoc | 12 ++--- .../src/transforming-result-documents.adoc | 31 +++++------ .../src/updatehandlers-in-solrconfig.adoc | 6 +-- .../uploading-data-with-index-handlers.adoc | 2 +- ...data-with-solr-cell-using-apache-tika.adoc | 4 +- ...okeeper-to-manage-configuration-files.adoc | 2 +- .../src/working-with-dates.adoc | 8 +-- .../src/working-with-enum-fields.adoc | 6 +-- ...ing-with-external-files-and-processes.adoc | 4 +- 41 files changed, 199 insertions(+), 202 deletions(-) diff --git a/solr/solr-ref-guide/src/common-query-parameters.adoc b/solr/solr-ref-guide/src/common-query-parameters.adoc index 5b6c7010efe..826cbe2cb36 100644 --- a/solr/solr-ref-guide/src/common-query-parameters.adoc +++ b/solr/solr-ref-guide/src/common-query-parameters.adoc @@ -46,7 +46,7 @@ The table below summarizes Solr's common query parameters, which are supported b The following sections describe these parameters in detail. [[CommonQueryParameters-ThedefTypeParameter]] -== The `defType` Parameter +== The defType Parameter The defType parameter selects the query parser that Solr should use to process the main query parameter (`q`) in the request. For example: @@ -55,7 +55,7 @@ The defType parameter selects the query parser that Solr should use to process t If no defType param is specified, then by default, the <> is used. (eg: `defType=lucene`) [[CommonQueryParameters-ThesortParameter]] -== The `sort` Parameter +== The sort Parameter The `sort` parameter arranges search results in either ascending (`asc`) or descending (`desc`) order. The parameter can be used with either numerical or alphabetical content. The directions can be entered in either all lowercase or all uppercase letters (i.e., both `asc` or `ASC`). @@ -88,7 +88,7 @@ Regarding the sort parameter's arguments: ** When more than one sort criteria is provided, the second entry will only be used if the first entry results in a tie. If there is a third entry, it will only be used if the first AND second entries are tied. This pattern continues with further entries. [[CommonQueryParameters-ThestartParameter]] -== The `start` Parameter +== The start Parameter When specified, the `start` parameter specifies an offset into a query's result set and instructs Solr to begin displaying results from this offset. @@ -99,14 +99,14 @@ Setting the `start` parameter to some other number, such as 3, causes Solr to sk You can use the `start` parameter this way for paging. For example, if the `rows` parameter is set to 10, you could display three successive pages of results by setting start to 0, then re-issuing the same query and setting start to 10, then issuing the query again and setting start to 20. [[CommonQueryParameters-TherowsParameter]] -== The `rows` Parameter +== The rows Parameter You can use the rows parameter to paginate results from a query. The parameter specifies the maximum number of documents from the complete result set that Solr should return to the client at one time. The default value is 10. That is, by default, Solr returns 10 documents at a time in response to a query. [[CommonQueryParameters-Thefq_FilterQuery_Parameter]] -== The `fq` (Filter Query) Parameter +== The fq (Filter Query) Parameter The `fq` parameter defines a query that can be used to restrict the superset of documents that can be returned, without influencing score. It can be very useful for speeding up complex queries, since the queries specified with `fq` are cached independently of the main query. When a later query uses the same filter, there's a cache hit, and filter results are returned quickly from the cache. @@ -132,7 +132,7 @@ fq=+popularity:[10 TO *] +section:0 * As with all parameters: special characters in an URL need to be properly escaped and encoded as hex values. Online tools are available to help you with URL-encoding. For example: http://meyerweb.com/eric/tools/dencoder/. [[CommonQueryParameters-Thefl_FieldList_Parameter]] -== The `fl` (Field List) Parameter +== The fl (Field List) Parameter The `fl` parameter limits the information included in a query response to a specified list of fields. The fields need to either be `stored="true"` or `docValues="true"``.` @@ -204,7 +204,7 @@ fl=id,sales_price:price,secret_sauce:prod(price,popularity),why_score:[explain s ---- [[CommonQueryParameters-ThedebugParameter]] -== The `debug` Parameter +== The debug Parameter The `debug` parameter can be specified multiple times and supports the following arguments: @@ -219,7 +219,7 @@ For backwards compatibility with older versions of Solr, `debugQuery=true` may i The default behavior is not to include debugging information. [[CommonQueryParameters-TheexplainOtherParameter]] -== The `explainOther` Parameter +== The explainOther Parameter The `explainOther` parameter specifies a Lucene query in order to identify a set of documents. If this parameter is included and is set to a non-blank value, the query will return debugging information, along with the "explain info" of each document that matches the Lucene query, relative to the main query (which is specified by the q parameter). For example: @@ -233,7 +233,7 @@ The query above allows you to examine the scoring explain info of the top matchi The default value of this parameter is blank, which causes no extra "explain info" to be returned. [[CommonQueryParameters-ThetimeAllowedParameter]] -== The `timeAllowed` Parameter +== The timeAllowed Parameter This parameter specifies the amount of time, in milliseconds, allowed for a search to complete. If this time expires before the search is complete, any partial results will be returned, but values such as `numFound`, <> counts, and result <> may not be accurate for the entire result set. @@ -245,7 +245,7 @@ This value is only checked at the time of: As this check is periodically performed, the actual time for which a request can be processed before it is aborted would be marginally greater than or equal to the value of `timeAllowed`. If the request consumes more time in other stages, e.g., custom components, etc., this parameter is not expected to abort the request. [[CommonQueryParameters-ThesegmentTerminateEarlyParameter]] -== The `segmentTerminateEarly` Parameter +== The segmentTerminateEarly Parameter This parameter may be set to either true or false. @@ -258,19 +258,19 @@ Similar to using <>. [[CommonQueryParameters-Thecache_falseParameter]] -== The `cache=false` Parameter +== The cache=false Parameter Solr caches the results of all queries and filter queries by default. To disable result caching, set the `cache=false` parameter. @@ -296,7 +296,7 @@ fq={!frange l=10 u=100 cache=false cost=100}mul(popularity,price) ---- [[CommonQueryParameters-ThelogParamsListParameter]] -== The `logParamsList` Parameter +== The logParamsList Parameter By default, Solr logs all parameters of requests. Set this parameter to restrict which parameters of a request are logged. This may help control logging to only those parameters considered important to your organization. @@ -314,7 +314,7 @@ This parameter does not only apply to query requests, but to any kind of request ==== [[CommonQueryParameters-TheechoParamsParameter]] -== The `echoParams` Parameter +== The echoParams Parameter The `echoParams` parameter controls what information about request parameters is included in the response header. diff --git a/solr/solr-ref-guide/src/config-api.adoc b/solr/solr-ref-guide/src/config-api.adoc index 13d523b05fd..1b330d0201a 100644 --- a/solr/solr-ref-guide/src/config-api.adoc +++ b/solr/solr-ref-guide/src/config-api.adoc @@ -212,7 +212,7 @@ For more information about user-defined properties, see the section <> below for examples of how to use this type of command. [[ConfigAPI-HowtoMapsolrconfig.xmlPropertiestoJSON]] -== How to Map `solrconfig.xml` Properties to JSON +== How to Map solrconfig.xml Properties to JSON By using this API, you will be generating JSON representations of properties defined in `solrconfig.xml`. To understand how properties should be represented with the API, let's take a look at a few examples. diff --git a/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc b/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc index c3a3021f0ff..48f9084452e 100644 --- a/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc +++ b/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc @@ -121,7 +121,7 @@ The path and name of the `solrcore.properties` file can be overridden using the ==== [[Configuringsolrconfig.xml-Userdefinedpropertiesfromcore.properties]] -=== User-Defined Properties in `core.properties` +=== User-Defined Properties in core.properties Every Solr core has a `core.properties` file, automatically created when using the APIs. When you create a SolrCloud collection, you can pass through custom parameters to go into each core.properties that will be created, by prefixing the parameter name with "property." as a URL parameter. Example: diff --git a/solr/solr-ref-guide/src/coreadmin-api.adoc b/solr/solr-ref-guide/src/coreadmin-api.adoc index a2e3aa3a0a6..fef4929bcc9 100644 --- a/solr/solr-ref-guide/src/coreadmin-api.adoc +++ b/solr/solr-ref-guide/src/coreadmin-api.adoc @@ -289,25 +289,25 @@ Either `path` or `targetCore` parameter must be specified but not both. The rang The `core` index will be split into as many pieces as the number of `path` or `targetCore` parameters. -==== Usage with two `targetCore` parameters: +==== Usage with two targetCore parameters: `\http://localhost:8983/solr/admin/cores?action=SPLIT&core=core0&targetCore=core1&targetCore=core2` Here the `core` index will be split into two pieces and merged into the two `targetCore` indexes. -==== Usage with two `path` parameters: +==== Usage with two path parameters: `\http://localhost:8983/solr/admin/cores?action=SPLIT&core=core0&path=/path/to/index/1&path=/path/to/index/2` The `core` index will be split into two pieces and written into the two directory paths specified. -==== Usage with the `split.key` parameter: +==== Usage with the split.key parameter: `\http://localhost:8983/solr/admin/cores?action=SPLIT&core=core0&targetCore=core1&split.key=A!` Here all documents having the same route key as the `split.key` i.e. 'A!' will be split from the `core` index and written to the `targetCore`. -==== Usage with `ranges` parameter: +==== Usage with ranges parameter: `\http://localhost:8983/solr/admin/cores?action=SPLIT&core=core0&targetCore=core1&targetCore=core2&targetCore=core3&ranges=0-1f4,1f5-3e8,3e9-5dc` diff --git a/solr/solr-ref-guide/src/datadir-and-directoryfactory-in-solrconfig.adoc b/solr/solr-ref-guide/src/datadir-and-directoryfactory-in-solrconfig.adoc index 496830d692b..8ee43be3b59 100644 --- a/solr/solr-ref-guide/src/datadir-and-directoryfactory-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/datadir-and-directoryfactory-in-solrconfig.adoc @@ -20,7 +20,7 @@ Where and how Solr stores its indexes are configurable options. -== Specifying a Location for Index Data with the `dataDir` Parameter +== Specifying a Location for Index Data with the dataDir Parameter By default, Solr stores its index data in a directory called `/data` under the core's instance directory (`instanceDir`). If you would like to specify a different directory for storing index data, you can configure the `dataDir` in the `core.properties` file for the core, or use the `` parameter in the `solrconfig.xml` file. You can specify another directory either with an absolute path or a pathname relative to the instanceDir of the SolrCore. For example: diff --git a/solr/solr-ref-guide/src/de-duplication.adoc b/solr/solr-ref-guide/src/de-duplication.adoc index 2a3b9020202..8f4d01a2c9d 100644 --- a/solr/solr-ref-guide/src/de-duplication.adoc +++ b/solr/solr-ref-guide/src/de-duplication.adoc @@ -47,7 +47,7 @@ Of course the `signatureField` could be the unique field, but generally you want There are two places in Solr to configure de-duplication: in `solrconfig.xml` and in `schema.xml`. [[De-Duplication-Insolrconfig.xml]] -=== In `solrconfig.xml` +=== In solrconfig.xml The `SignatureUpdateProcessorFactory` has to be registered in `solrconfig.xml` as part of an <>, as in this example: @@ -87,7 +87,7 @@ A Signature implementation for generating a signature hash. The full classpath o |=== [[De-Duplication-Inschema.xml]] -=== In `schema.xml` +=== In schema.xml If you are using a separate field for storing the signature, you must have it indexed: diff --git a/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc b/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc index 26786e425e9..b73fdf77777 100644 --- a/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc +++ b/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc @@ -67,7 +67,7 @@ Here is an example of a minimal LangDetect `langid` configuration in `solrconfig ---- [[DetectingLanguagesDuringIndexing-langidParameters]] -== `langid` Parameters +== langid Parameters As previously mentioned, both implementations of the `langid` UpdateRequestProcessor take the same parameters. diff --git a/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc b/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc index 9b2ca46d1db..b1ad8dc8b27 100644 --- a/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc +++ b/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc @@ -34,7 +34,7 @@ When not using SolrCloud, it is up to you to get all your documents indexed on e In the legacy distributed mode, Solr does not calculate universal term/doc frequencies. For most large-scale implementations, it is not likely to matter that Solr calculates TF/IDF at the shard level. However, if your collection is heavily skewed in its distribution across servers, you may find misleading relevancy results in your searches. In general, it is probably best to randomly distribute documents to your shards. [[DistributedSearchwithIndexSharding-ExecutingDistributedSearcheswiththeshardsParameter]] -== Executing Distributed Searches with the `shards` Parameter +== Executing Distributed Searches with the shards Parameter If a query request includes the `shards` parameter, the Solr server distributes the request across all the shards listed as arguments to the parameter. The `shards` parameter uses this syntax: diff --git a/solr/solr-ref-guide/src/enabling-ssl.adoc b/solr/solr-ref-guide/src/enabling-ssl.adoc index c26e4af8423..32b3cc04bea 100644 --- a/solr/solr-ref-guide/src/enabling-ssl.adoc +++ b/solr/solr-ref-guide/src/enabling-ssl.adoc @@ -253,7 +253,7 @@ curl -E solr-ssl.keystore.p12:secret --cacert solr-ssl.cacert.pem ... NOTE: If your operating system does not include cURL, you can download binaries here: http://curl.haxx.se/download.html -=== Create a SolrCloud Collection using `bin/solr` +=== Create a SolrCloud Collection using bin/solr Create a 2-shard, replicationFactor=1 collection named mycollection using the default configset (data_driven_schema_configs): @@ -318,7 +318,7 @@ You should get a response that looks like this: ---- [[EnablingSSL-Indexdocumentsusingpost.jar]] -=== Index Documents using `post.jar` +=== Index Documents using post.jar Use `post.jar` to index some example documents to the SolrCloud collection created above: @@ -340,7 +340,7 @@ curl -E solr-ssl.pem:secret --cacert solr-ssl.pem "https://localhost:8984/solr/m ---- [[EnablingSSL-IndexadocumentusingCloudSolrClient]] -=== Index a document using `CloudSolrClient` +=== Index a document using CloudSolrClient From a java client using SolrJ, index a document. In the code below, the `javax.net.ssl.*` system properties are set programmatically, but you could instead specify them on the java command line, as in the `post.jar` example above: diff --git a/solr/solr-ref-guide/src/exporting-result-sets.adoc b/solr/solr-ref-guide/src/exporting-result-sets.adoc index 37401a5a457..33852fadb1f 100644 --- a/solr/solr-ref-guide/src/exporting-result-sets.adoc +++ b/solr/solr-ref-guide/src/exporting-result-sets.adoc @@ -31,7 +31,7 @@ The cases where this functionality may be useful include: session analysis, dist All the fields being sorted and exported must have docValues set to true. For more information, see the section on <>. [[ExportingResultSets-The_exportRequestHandler]] -== The `/export` RequestHandler +== The /export RequestHandler The `/export` request handler with the appropriate configuration is one of Solr's out-of-the-box request handlers - see <> for more information. diff --git a/solr/solr-ref-guide/src/faceting.adoc b/solr/solr-ref-guide/src/faceting.adoc index d61dbcfb99b..158b2dbacb7 100644 --- a/solr/solr-ref-guide/src/faceting.adoc +++ b/solr/solr-ref-guide/src/faceting.adoc @@ -29,12 +29,12 @@ Searchers are presented with the indexed terms, along with numerical counts of h There are two general parameters for controlling faceting. [[Faceting-ThefacetParameter]] -=== The `facet` Parameter +=== The facet Parameter If set to *true*, this parameter enables facet counts in the query response. If set to *false*, a blank or missing value, this parameter disables faceting. None of the other parameters listed below will have any effect unless this parameter is set to *true*. The default value is blank (false). [[Faceting-Thefacet.queryParameter]] -=== The `facet.query` Parameter +=== The facet.query Parameter This parameter allows you to specify an arbitrary query in the Lucene default syntax to generate a facet count. @@ -83,7 +83,7 @@ The table below summarizes Solr's field value faceting parameters. These parameters are described in the sections below. [[Faceting-Thefacet.fieldParameter]] -=== The `facet.field` Parameter +=== The facet.field Parameter The `facet.field` parameter identifies a field that should be treated as a facet. It iterates over each Term in the field and generate a facet count using that Term as the constraint. This parameter can be specified multiple times in a query to select multiple facet fields. @@ -93,28 +93,28 @@ If you do not set this parameter to at least one field in the schema, none of th ==== [[Faceting-Thefacet.prefixParameter]] -=== The `facet.prefix` Parameter +=== The facet.prefix Parameter The `facet.prefix` parameter limits the terms on which to facet to those starting with the given string prefix. This does not limit the query in any way, only the facets that would be returned in response to the query. This parameter can be specified on a per-field basis with the syntax of `f..facet.prefix`. [[Faceting-Thefacet.containsParameter]] -=== The `facet.contains` Parameter +=== The facet.contains Parameter The `facet.contains` parameter limits the terms on which to facet to those containing the given substring. This does not limit the query in any way, only the facets that would be returned in response to the query. This parameter can be specified on a per-field basis with the syntax of `f..facet.contains`. [[Faceting-Thefacet.contains.ignoreCaseParameter]] -=== The `facet.contains.ignoreCase` Parameter +=== The facet.contains.ignoreCase Parameter If `facet.contains` is used, the `facet.contains.ignoreCase` parameter causes case to be ignored when matching the given substring against candidate facet terms. This parameter can be specified on a per-field basis with the syntax of `f..facet.contains.ignoreCase`. [[Faceting-Thefacet.sortParameter]] -=== The `facet.sort` Parameter +=== The facet.sort Parameter This parameter determines the ordering of the facet field constraints. @@ -128,7 +128,7 @@ The default is `count` if `facet.limit` is greater than 0, otherwise, the defaul This parameter can be specified on a per-field basis with the syntax of `f..facet.sort`. [[Faceting-Thefacet.limitParameter]] -=== The `facet.limit` Parameter +=== The facet.limit Parameter This parameter specifies the maximum number of constraint counts (essentially, the number of facets for a field that are returned) that should be returned for the facet fields. A negative value means that Solr will return unlimited number of constraint counts. @@ -137,7 +137,7 @@ The default value is 100. This parameter can be specified on a per-field basis to apply a distinct limit to each field with the syntax of `f..facet.limit`. [[Faceting-Thefacet.offsetParameter]] -=== The `facet.offset` Parameter +=== The facet.offset Parameter The `facet.offset` parameter indicates an offset into the list of constraints to allow paging. @@ -146,7 +146,7 @@ The default value is 0. This parameter can be specified on a per-field basis with the syntax of `f..facet.offset`. [[Faceting-Thefacet.mincountParameter]] -=== The `facet.mincount` Parameter +=== The facet.mincount Parameter The `facet.mincount` parameter specifies the minimum counts required for a facet field to be included in the response. If a field's counts are below the minimum, the field's facet is not returned. @@ -155,7 +155,7 @@ The default value is 0. This parameter can be specified on a per-field basis with the syntax of `f..facet.mincount`. [[Faceting-Thefacet.missingParameter]] -=== The `facet.missing` Parameter +=== The facet.missing Parameter If set to true, this parameter indicates that, in addition to the Term-based constraints of a facet field, a count of all results that match the query but which have no facet value for the field should be computed and returned in the response. @@ -164,7 +164,7 @@ The default value is false. This parameter can be specified on a per-field basis with the syntax of `f..facet.missing`. [[Faceting-Thefacet.methodParameter]] -=== The `facet.method` Parameter +=== The facet.method Parameter The facet.method parameter selects the type of algorithm or method Solr should use when faceting a field. @@ -189,7 +189,7 @@ The default value is `fc` (except for fields using the `BoolField` field type an This parameter can be specified on a per-field basis with the syntax of `f..facet.method`. [[Faceting-Thefacet.enum.cache.minDfParameter]] -=== The `facet.enum.cache.minDf` Parameter +=== The facet.enum.cache.minDf Parameter This parameter indicates the minimum document frequency (the number of documents matching a term) for which the filterCache should be used when determining the constraint count for that term. This is only used with the `facet.method=enum` method of faceting. @@ -200,14 +200,14 @@ The default value is 0, causing the filterCache to be used for all terms in the This parameter can be specified on a per-field basis with the syntax of `f..facet.enum.cache.minDf`. [[Faceting-Thefacet.existsParameter]] -=== The `facet.exists` Parameter +=== The facet.exists Parameter To cap facet counts by 1, specify `facet.exists=true`. It can be used with `facet.method=enum` or when it's omitted. It can be used only on non-trie fields (such as strings). It may speed up facet counting on large indices and/or high-cardinality facet values.. This parameter can be specified on a per-field basis with the syntax of `f..facet.exists` or via local parameter` facet.field={!facet.method=enum facet.exists=true}size`. [[Faceting-Thefacet.excludeTermsParameter]] -=== The `facet.excludeTerms` Parameter +=== The facet.excludeTerms Parameter If you want to remove terms from facet counts but keep them in the index, the `facet.excludeTerms` parameter allows you to do that. @@ -219,7 +219,7 @@ In some situations, the accuracy in selecting the "top" constraints returned for In some situations, depending on how your docs are partitioned across your shards, and what `facet.limit` value you used, you may find it advantageous to increase or decrease the amount of over-requesting Solr does. This can be achieved by setting the `facet.overrequest.count` (defaults to 10) and `facet.overrequest.ratio` (defaults to 1.5) parameters. [[Faceting-Thefacet.threadsParameter]] -=== The `facet.threads` Parameter +=== The facet.threads Parameter This param will cause loading the underlying fields used in faceting to be executed in parallel with the number of threads specified. Specify as `facet.threads=N` where `N` is the maximum number of threads used. Omitting this parameter or specifying the thread count as 0 will not spawn any threads, and only the main request thread will be used. Specifying a negative number of threads will create up to Integer.MAX_VALUE threads. @@ -244,7 +244,7 @@ You can use Range Faceting on any date field or any numeric field that supports |=== [[Faceting-Thefacet.rangeParameter]] -=== The `facet.range` Parameter +=== The facet.range Parameter The `facet.range` parameter defines the field for which Solr should create range facets. For example: @@ -253,7 +253,7 @@ The `facet.range` parameter defines the field for which Solr should create range `facet.range=lastModified_dt` [[Faceting-Thefacet.range.startParameter]] -=== The `facet.range.start` Parameter +=== The facet.range.start Parameter The `facet.range.start` parameter specifies the lower bound of the ranges. You can specify this parameter on a per field basis with the syntax of `f..facet.range.start`. For example: @@ -262,7 +262,7 @@ The `facet.range.start` parameter specifies the lower bound of the ranges. You c `f.lastModified_dt.facet.range.start=NOW/DAY-30DAYS` [[Faceting-Thefacet.range.endParameter]] -=== The `facet.range.end` Parameter +=== The facet.range.end Parameter The facet.range.end specifies the upper bound of the ranges. You can specify this parameter on a per field basis with the syntax of `f..facet.range.end`. For example: @@ -271,7 +271,7 @@ The facet.range.end specifies the upper bound of the ranges. You can specify thi `f.lastModified_dt.facet.range.end=NOW/DAY+30DAYS` [[Faceting-Thefacet.range.gapParameter]] -=== The `facet.range.gap` Parameter +=== The facet.range.gap Parameter The span of each range expressed as a value to be added to the lower bound. For date fields, this should be expressed using the {solr-javadocs}/solr-core/org/apache/solr/util/DateMathParser.html[`DateMathParser` syntax] (such as, `facet.range.gap=%2B1DAY ... '+1DAY'`). You can specify this parameter on a per-field basis with the syntax of `f..facet.range.gap`. For example: @@ -280,7 +280,7 @@ The span of each range expressed as a value to be added to the lower bound. For `f.lastModified_dt.facet.range.gap=+1DAY` [[Faceting-Thefacet.range.hardendParameter]] -=== The `facet.range.hardend` Parameter +=== The facet.range.hardend Parameter The `facet.range.hardend` parameter is a Boolean parameter that specifies how Solr should handle cases where the `facet.range.gap` does not divide evenly between `facet.range.start` and `facet.range.end`. @@ -289,7 +289,7 @@ If *true*, the last range constraint will have the `facet.range.end` value as an This parameter can be specified on a per field basis with the syntax `f..facet.range.hardend`. [[Faceting-Thefacet.range.includeParameter]] -=== The `facet.range.include` Parameter +=== The facet.range.include Parameter By default, the ranges used to compute range faceting between `facet.range.start` and `facet.range.end` are inclusive of their lower bounds and exclusive of the upper bounds. The "before" range defined with the `facet.range.other` parameter is exclusive and the "after" range is inclusive. This default, equivalent to "lower" below, will not result in double counting at the boundaries. You can use the `facet.range.include` parameter to modify this behavior using the following options: @@ -313,7 +313,7 @@ To ensure you avoid double-counting, do not choose both `lower` and `upper`, do ==== [[Faceting-Thefacet.range.otherParameter]] -=== The `facet.range.other` Parameter +=== The facet.range.other Parameter The `facet.range.other` parameter specifies that in addition to the counts for each range constraint between `facet.range.start` and `facet.range.end`, counts should also be computed for these options: @@ -332,7 +332,7 @@ The `facet.range.other` parameter specifies that in addition to the counts for e This parameter can be specified on a per field basis with the syntax of `f..facet.range.other`. In addition to the `all` option, this parameter can be specified multiple times to indicate multiple choices, but `none` will override all other options. [[Faceting-Thefacet.range.methodParameter]] -=== The `facet.range.method` Parameter +=== The facet.range.method Parameter The `facet.range.method` parameter selects the type of algorithm or method Solr should use for range faceting. Both methods produce the same results, but performance may vary. @@ -343,7 +343,7 @@ dv:: This method iterates the documents that match the main query, and for each Default value for this parameter is "filter". [[Faceting-Thefacet.mincountParameterinRangeFaceting]] -=== The `facet.mincount` Parameter in Range Faceting +=== The facet.mincount Parameter in Range Faceting The `facet.mincount` parameter, the same one as used in field faceting is also applied to range faceting. When used, no ranges with a count below the minimum will be included in the response. @@ -653,14 +653,14 @@ If you are concerned about the performance of your searches you should test with This method will use <> if they are enabled for the field, will use fieldCache otherwise. [[Faceting-Thefacet.intervalparameter]] -=== The `facet.interval` parameter +=== The facet.interval parameter This parameter Indicates the field where interval faceting must be applied. It can be used multiple times in the same request to indicate multiple fields. `facet.interval=price&facet.interval=size` [[Faceting-Thefacet.interval.setparameter]] -=== The `facet.interval.set` parameter +=== The facet.interval.set parameter This parameter is used to set the intervals for the field, it can be specified multiple times to indicate multiple intervals. This parameter is global, which means that it will be used for all fields indicated with `facet.interval` unless there is an override for a specific field. To override this parameter on a specific field you can use: `f..facet.interval.set`, for example: diff --git a/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc b/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc index 92b32a3c63c..12d291306cf 100644 --- a/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc +++ b/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc @@ -28,7 +28,7 @@ A field type definition can include four types of information: * Field type properties - depending on the implementation class, some properties may be mandatory. [[FieldTypeDefinitionsandProperties-FieldTypeDefinitionsinschema.xml]] -== Field Type Definitions in `schema.xml` +== Field Type Definitions in schema.xml Field types are defined in `schema.xml`. Each field type is defined between `fieldType` elements. They can optionally be grouped within a `types` element. Here is an example of a field type definition for a type called `text_general`: diff --git a/solr/solr-ref-guide/src/highlighting.adoc b/solr/solr-ref-guide/src/highlighting.adoc index 77fae072608..117ccbcf7f8 100644 --- a/solr/solr-ref-guide/src/highlighting.adoc +++ b/solr/solr-ref-guide/src/highlighting.adoc @@ -251,7 +251,7 @@ The FastVector Highlighter will occasionally truncate highlighted words. To prev Solr supports two boundary scanners: `breakIterator` and `simple`. [[Highlighting-ThebreakIteratorBoundaryScanner]] -==== The `breakIterator` Boundary Scanner +==== The breakIterator Boundary Scanner The `breakIterator` boundary scanner offers excellent performance right out of the box by taking locale and boundary type into account. In most cases you will want to use the `breakIterator` boundary scanner. To implement the `breakIterator` boundary scanner, add this code to the `highlighting` section of your `solrconfig.xml` file, adjusting the type, language, and country values as appropriate to your application: @@ -269,7 +269,7 @@ The `breakIterator` boundary scanner offers excellent performance right out of t Possible values for the `hl.bs.type` parameter are WORD, LINE, SENTENCE, and CHARACTER. [[Highlighting-ThesimpleBoundaryScanner]] -==== The `simple` Boundary Scanner +==== The simple Boundary Scanner The `simple` boundary scanner scans term boundaries for a specified maximum character value (`hl.bs.maxScan`) and for common delimiters such as punctuation marks (`hl.bs.chars`). The `simple` boundary scanner may be useful for some custom To implement the `simple` boundary scanner, add this code to the `highlighting` section of your `solrconfig.xml` file, adjusting the values as appropriate to your application: diff --git a/solr/solr-ref-guide/src/index-replication.adoc b/solr/solr-ref-guide/src/index-replication.adoc index 90fa4703996..df8e9c60371 100644 --- a/solr/solr-ref-guide/src/index-replication.adoc +++ b/solr/solr-ref-guide/src/index-replication.adoc @@ -111,7 +111,7 @@ The example below shows a possible 'master' configuration for the `ReplicationHa ---- [[IndexReplication-Replicatingsolrconfig.xml]] -==== Replicating `solrconfig.xml` +==== Replicating solrconfig.xml In the configuration file on the master server, include a line like the following: diff --git a/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc b/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc index 74d5f977f77..cfc1a78b5c7 100644 --- a/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc @@ -33,7 +33,7 @@ By default, the settings are commented out in the sample `solrconfig.xml` includ == Writing New Segments [[IndexConfiginSolrConfig-ramBufferSizeMB]] -=== `ramBufferSizeMB` +=== ramBufferSizeMB Once accumulated document updates exceed this much memory space (defined in megabytes), then the pending updates are flushed. This can also create new segments or trigger a merge. Using this setting is generally preferable to `maxBufferedDocs`. If both `maxBufferedDocs` and `ramBufferSizeMB` are set in `solrconfig.xml`, then a flush will occur when either limit is reached. The default is 100Mb. @@ -43,7 +43,7 @@ Once accumulated document updates exceed this much memory space (defined in mega ---- [[IndexConfiginSolrConfig-maxBufferedDocs]] -=== `maxBufferedDocs` +=== maxBufferedDocs Sets the number of document updates to buffer in memory before they are flushed as a new segment. This may also trigger a merge. The default Solr configuration sets to flush by RAM usage (`ramBufferSizeMB`). @@ -66,7 +66,7 @@ Controls whether newly written (and not yet merged) index segments should use th == Merging Index Segments [[IndexConfiginSolrConfig-mergePolicyFactory]] -=== `mergePolicyFactory` +=== mergePolicyFactory Defines how merging segments is done. @@ -118,7 +118,7 @@ If the configuration options for the built-in merge policies do not fully suit y The example above shows Solr's {solr-javadocs}/solr-core/org/apache/solr/index/SortingMergePolicyFactory.html[`SortingMergePolicyFactory`] being configured to sort documents in merged segments by `"timestamp desc"`, and wrapped around a `TieredMergePolicyFactory` configured to use the values `maxMergeAtOnce=10` and `segmentsPerTier=10` via the `inner` prefix defined by `SortingMergePolicyFactory`'s `wrapped.prefix` option. For more information on using `SortingMergePolicyFactory`, see <>. [[IndexConfiginSolrConfig-mergeScheduler]] -=== `mergeScheduler` +=== mergeScheduler The merge scheduler controls how merges are performed. The default `ConcurrentMergeScheduler` performs merges in the background using separate threads. The alternative, `SerialMergeScheduler`, does not perform merges with separate threads. @@ -128,7 +128,7 @@ The merge scheduler controls how merges are performed. The default `ConcurrentMe ---- [[IndexConfiginSolrConfig-mergedSegmentWarmer]] -=== `mergedSegmentWarmer` +=== mergedSegmentWarmer When using Solr in for <> a merged segment warmer can be configured to warm the reader on the newly merged segment, before the merge commits. This is not required for near real-time search, but will reduce search latency on opening a new near real-time reader after a merge completes. @@ -159,7 +159,7 @@ Many <> implementatio == Index Locks [[IndexConfiginSolrConfig-lockType]] -=== `lockType` +=== lockType The LockFactory options specify the locking implementation to use. @@ -178,7 +178,7 @@ For more information on the nuances of each LockFactory, see http://wiki.apache. ---- [[IndexConfiginSolrConfig-writeLockTimeout]] -=== `writeLockTimeout` +=== writeLockTimeout The maximum time to wait for a write lock on an IndexWriter. The default is 1000, expressed in milliseconds. diff --git a/solr/solr-ref-guide/src/introduction-to-solr-indexing.adoc b/solr/solr-ref-guide/src/introduction-to-solr-indexing.adoc index 2412f0173e5..888d8db9b1a 100644 --- a/solr/solr-ref-guide/src/introduction-to-solr-indexing.adoc +++ b/solr/solr-ref-guide/src/introduction-to-solr-indexing.adoc @@ -44,7 +44,7 @@ For more information on indexing in Solr, see the https://wiki.apache.org/solr/F When starting Solr with the "-e" option, the `example/` directory will be used as base directory for the example Solr instances that are created. This directory also includes an `example/exampledocs/` subdirectory containing sample documents in a variety of formats that you can use to experiment with indexing into the various examples. [[IntroductiontoSolrIndexing-ThecurlUtilityforTransferringFiles]] -== The `curl` Utility for Transferring Files +== The curl Utility for Transferring Files Many of the instructions and examples in this section make use of the `curl` utility for transferring content through a URL. `curl` posts and retrieves data over HTTP, FTP, and many other protocols. Most Linux distributions include a copy of `curl`. You'll find curl downloads for Linux, Windows, and many other operating systems at http://curl.haxx.se/download.html. Documentation for `curl` is available here: http://curl.haxx.se/docs/manpage.html. diff --git a/solr/solr-ref-guide/src/local-parameters-in-queries.adoc b/solr/solr-ref-guide/src/local-parameters-in-queries.adoc index 4618ec32111..1ed8eeacd52 100644 --- a/solr/solr-ref-guide/src/local-parameters-in-queries.adoc +++ b/solr/solr-ref-guide/src/local-parameters-in-queries.adoc @@ -64,7 +64,7 @@ is equivilent to: `fq={!type=lucene df=summary}solr rocks` -== Specifying the Parameter Value with the `v` Key +== Specifying the Parameter Value with the v Key A special key of `v` within local parameters is an alternate way to specify the value of that parameter. diff --git a/solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adoc b/solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adoc index 7c2f7d603ff..9b58cef26c0 100644 --- a/solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adoc +++ b/solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adoc @@ -84,6 +84,6 @@ Regardless of whether it is explicitly declared, or used as an implicit global d DELETESHARD and DELETEREPLICA now default to deleting the instance directory, data directory, and index directory for any replica they delete. Please review the <> documentation for details on new request parameters to prevent this behavior if you wish to keep all data on disk when using these commands -== `facet.date.*` Parameters Removed +== facet.date.* Parameters Removed The `facet.date` parameter (and associated `facet.date.*` parameters) that were deprecated in Solr 3.x have been removed completely. If you have not yet switched to using the equivalent <> functionality you must do so now before upgrading. diff --git a/solr/solr-ref-guide/src/merging-indexes.adoc b/solr/solr-ref-guide/src/merging-indexes.adoc index e57e3ab5359..49afe4e3115 100644 --- a/solr/solr-ref-guide/src/merging-indexes.adoc +++ b/solr/solr-ref-guide/src/merging-indexes.adoc @@ -28,7 +28,7 @@ To merge indexes, they must meet these requirements: Optimally, the two indexes should be built using the same schema. [[MergingIndexes-UsingIndexMergeTool]] -== Using `IndexMergeTool` +== Using IndexMergeTool To merge the indexes, do the following: diff --git a/solr/solr-ref-guide/src/near-real-time-searching.adoc b/solr/solr-ref-guide/src/near-real-time-searching.adoc index 641208e029b..8d87d5477e9 100644 --- a/solr/solr-ref-guide/src/near-real-time-searching.adoc +++ b/solr/solr-ref-guide/src/near-real-time-searching.adoc @@ -31,7 +31,7 @@ However, pay special attention to cache and autowarm settings as they can have a A commit operation makes index changes visible to new search requests. A *hard commit* uses the transaction log to get the id of the latest document changes, and also calls `fsync` on the index files to ensure they have been flushed to stable storage and no data loss will result from a power failure. The current transaction log is closed and a new one is opened. See the "transaction log" discussion below for data loss issues. -A *soft commit* is much faster since it only makes index changes visible and does not `fsync` index files, or write a new index descriptor or start a new transaction log. Search collections that have NRT requirements (that want index changes to be quickly visible to searches) will want to soft commit often but hard commit less frequently. A softCommit may be "less expensive", but it is not free, since it can slow throughput. See the "transaction log" discussion below for data loss issues. +A *soft commit* is much faster since it only makes index changes visible and does not `fsync` index files, or write a new index descriptor or start a new transaction log. Search collections that have NRT requirements (that want index changes to be quickly visible to searches) will want to soft commit often but hard commit less frequently. A softCommit may be "less expensive", but it is not free, since it can slow throughput. See the "transaction log" discussion below for data loss issues. An *optimize* is like a *hard commit* except that it forces all of the index segments to be merged into a single segment first. Depending on the use, this operation should be performed infrequently (e.g., nightly), if at all, since it involves reading and re-writing the entire index. Segments are normally merged over time anyway (as determined by the merge policy), and optimize just forces these merges to occur immediately. @@ -52,7 +52,7 @@ Use `maxDocs` and `maxTime` judiciously to fine-tune your commit strategies. === Transaction Logs (tlogs) Transaction logs are a "rolling window" of at least the last `N` (default 100) documents indexed. Tlogs are configured in solrconfig.xml, including the value of `N`. The current transaction log is closed and a new one opened each time any variety of hard commit occurs. Soft commits have no effect on the transaction log. - + When tlogs are enabled, documents being added to the index are written to the tlog before the indexing call returns to the client. In the event of an un-graceful shutdown (power loss, JVM crash, `kill -9` etc) any documents written to the tlog that was open when Solr stopped are replayed on startup. When Solr is shut down gracefully (i.e. using the `bin/solr stop` command and the like) Solr will close the tlog file and index segments so no replay will be necessary on startup. @@ -76,7 +76,7 @@ For example: It's better to use `maxTime` rather than `maxDocs` to modify an `autoSoftCommit`, especially when indexing a large number of documents through the commit operation. It's also better to turn off `autoSoftCommit` for bulk indexing. [[NearRealTimeSearching-OptionalAttributesforcommitandoptimize]] -=== Optional Attributes for `commit` and `optimize` +=== Optional Attributes for commit and optimize // TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed @@ -100,7 +100,7 @@ Example of `commit` and `optimize` with optional attributes: ---- [[NearRealTimeSearching-PassingcommitandcommitWithinparametersaspartoftheURL]] -=== Passing `commit` and `commitWithin` Parameters as Part of the URL +=== Passing commit and commitWithin Parameters as Part of the URL Update handlers can also get `commit`-related parameters as part of the update URL. This example adds a small test document and causes an explicit commit to happen immediately afterwards: @@ -133,7 +133,7 @@ curl http://localhost:8983/solr/my_collection/update?commitWithin=10000 ---- [[NearRealTimeSearching-ChangingdefaultcommitWithinBehavior]] -=== Changing default `commitWithin` Behavior +=== Changing default commitWithin Behavior The `commitWithin` settings allow forcing document commits to happen in a defined time period. This is used most frequently with <>, and for that reason the default is to perform a soft commit. This does not, however, replicate new documents to slave servers in a master/slave environment. If that's a requirement for your implementation, you can force a hard commit by adding a parameter, as in this example: diff --git a/solr/solr-ref-guide/src/parallel-sql-interface.adoc b/solr/solr-ref-guide/src/parallel-sql-interface.adoc index 5dccfce08fd..57cea47412d 100644 --- a/solr/solr-ref-guide/src/parallel-sql-interface.adoc +++ b/solr/solr-ref-guide/src/parallel-sql-interface.adoc @@ -180,7 +180,7 @@ SELECT fieldA as fa, fieldB as fb, fieldC as fc FROM tableA WHERE fieldC = 'term We've covered many syntax options with this example, so let's walk through what's possible below. -=== `WHERE` Clause and Boolean Predicates +=== WHERE Clause and Boolean Predicates [IMPORTANT] ==== @@ -226,7 +226,7 @@ To specify NOT queries, you use the `AND NOT` syntax as follows: WHERE (fieldA = 'term1') AND NOT (fieldB = 'term2') ---- -==== Supported `WHERE` Operators +==== Supported WHERE Operators The parallel SQL interface supports and pushes down most common SQL operators, specifically: @@ -247,7 +247,7 @@ Some operators that are not supported are BETWEEN, LIKE and IN. However, there a * BETWEEN can be supported with a range query, such as `field = [50 TO 100]`. * A simplistic LIKE can be used with a wildcard, such as `field = 'sam*'`. -=== `ORDER BY` Clause +=== ORDER BY Clause The `ORDER BY` clause maps directly to Solr fields. Multiple `ORDER BY` fields and directions are supported. @@ -257,7 +257,7 @@ If the `ORDER BY` clause contains the exact fields in the `GROUP BY` clause, the Order by fields are case sensitive. -=== `LIMIT` Clause +=== LIMIT Clause Limits the result set to the specified size. In the example above the clause `LIMIT 100` will limit the result set to 100 records. @@ -267,7 +267,7 @@ There are a few differences to note between limited and unlimited queries: * Limited queries allow any stored field in the field list. Unlimited queries require the fields to be stored as a DocValues field. * Limited queries allow any indexed field in the `ORDER BY` list. Unlimited queries require the fields to be stored as a DocValues field. -=== `SELECT DISTINCT` Queries +=== SELECT DISTINCT Queries The SQL interface supports both MapReduce and Facet implementations for `SELECT DISTINCT` queries. @@ -293,13 +293,13 @@ Because these functions never require data to be shuffled, the aggregations are SELECT count(*) as count, sum(fieldB) as sum FROM tableA WHERE fieldC = 'Hello' ---- -=== `GROUP BY` Aggregations +=== GROUP BY Aggregations The SQL interface also supports `GROUP BY` aggregate queries. As with `SELECT DISTINCT` queries, the SQL interface supports both a MapReduce implementation and a Facet implementation. The MapReduce implementation can build aggregations over extremely high cardinality fields. The Facet implementations provides high performance aggregation over fields with moderate levels of cardinality. -==== Basic `GROUP BY` with Aggregates +==== Basic GROUP BY with Aggregates Here is a basic example of a GROUP BY query that requests aggregations: @@ -327,7 +327,7 @@ The non-function fields in the field list determine the fields to calculate the The `GROUP BY` clause can contain up to 4 fields in the Solr index. These fields should correspond with the non-function fields in the field list. -=== `HAVING` Clause +=== HAVING Clause The `HAVING` clause may contain any function listed in the field list. Complex `HAVING` clauses such as this are supported: diff --git a/solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc b/solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc index f380110643b..1a6b315ea0a 100644 --- a/solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc @@ -55,7 +55,7 @@ FastLRUCache and LFUCache support `showItems` attribute. This is the number of c Details of each cache are described below. [[QuerySettingsinSolrConfig-filterCache]] -=== `filterCache` +=== filterCache This cache is used by `SolrIndexSearcher` for filters (DocSets) for unordered sets of all documents that match a query. The numeric attributes control the number of entries in the cache. @@ -72,7 +72,7 @@ Solr also uses this cache for faceting when the configuration parameter `facet.m ---- [[QuerySettingsinSolrConfig-queryResultCache]] -=== `queryResultCache` +=== queryResultCache This cache holds the results of previous searches: ordered lists of document IDs (DocList) based on a query, a sort, and the range of documents requested. @@ -88,7 +88,7 @@ The `queryResultCache` has an additional (optional) setting to limit the maximum ---- [[QuerySettingsinSolrConfig-documentCache]] -=== `documentCache` +=== documentCache This cache holds Lucene Document objects (the stored fields for each document). Since Lucene internal document IDs are transient, this cache is not auto-warmed. The size for the `documentCache` should always be greater than `max_results` times the `max_concurrent_queries`, to ensure that Solr does not need to refetch a document during a request. The more fields you store in your documents, the higher the memory usage of this cache will be. @@ -120,7 +120,7 @@ If you want auto-warming of your cache, include a `regenerator` attribute with t == Query Sizing and Warming [[QuerySettingsinSolrConfig-maxBooleanClauses]] -=== `maxBooleanClauses` +=== maxBooleanClauses This sets the maximum number of clauses allowed in a boolean query. This can affect range or prefix queries that expand to a query with a large number of boolean terms. If this limit is exceeded, an exception is thrown. @@ -135,7 +135,7 @@ This option modifies a global property that effects all Solr cores. If multiple ==== [[QuerySettingsinSolrConfig-enableLazyFieldLoading]] -=== `enableLazyFieldLoading` +=== enableLazyFieldLoading If this parameter is set to true, then fields that are not directly requested will be loaded lazily as needed. This can boost performance if the most common queries only need a small subset of fields, especially if infrequently accessed fields are large in size. @@ -145,7 +145,7 @@ If this parameter is set to true, then fields that are not directly requested wi ---- [[QuerySettingsinSolrConfig-useFilterForSortedQuery]] -=== `useFilterForSortedQuery` +=== useFilterForSortedQuery This parameter configures Solr to use a filter to satisfy a search. If the requested sort does not include "score", the `filterCache` will be checked for a filter matching the query. For most situations, this is only useful if the same search is requested often with different sort options and none of them ever use "score". @@ -155,7 +155,7 @@ This parameter configures Solr to use a filter to satisfy a search. If the reque ---- [[QuerySettingsinSolrConfig-queryResultWindowSize]] -=== `queryResultWindowSize` +=== queryResultWindowSize Used with the `queryResultCache`, this will cache a superset of the requested number of document IDs. For example, if the a search in response to a particular query requests documents 10 through 19, and `queryWindowSize` is 50, documents 0 through 49 will be cached. @@ -165,7 +165,7 @@ Used with the `queryResultCache`, this will cache a superset of the requested nu ---- [[QuerySettingsinSolrConfig-queryResultMaxDocsCached]] -=== `queryResultMaxDocsCached` +=== queryResultMaxDocsCached This parameter sets the maximum number of documents to cache for any entry in the `queryResultCache`. @@ -175,7 +175,7 @@ This parameter sets the maximum number of documents to cache for any entry in th ---- [[QuerySettingsinSolrConfig-useColdSearcher]] -=== `useColdSearcher` +=== useColdSearcher This setting controls whether search requests for which there is not a currently registered searcher should wait for a new searcher to warm up (false) or proceed immediately (true). When set to "false", requests will block until the searcher has warmed its caches. @@ -185,7 +185,7 @@ This setting controls whether search requests for which there is not a currently ---- [[QuerySettingsinSolrConfig-maxWarmingSearchers]] -=== `maxWarmingSearchers` +=== maxWarmingSearchers This parameter sets the maximum number of searchers that may be warming up in the background at any given time. Exceeding this limit will raise an error. For read-only slaves, a value of two is reasonable. Masters should probably be set a little higher. @@ -227,7 +227,7 @@ The (commented out) examples below can be found in the `solrconfig.xml` file of ==== The above code comes from a _sample_ `solrconfig.xml`. -A key best practice is to modify these defaults before taking your application to production, but please note: while the sample queries are commented out in the section for the "newSearcher", the sample query is not commented out for the "firstSearcher" event. +A key best practice is to modify these defaults before taking your application to production, but please note: while the sample queries are commented out in the section for the "newSearcher", the sample query is not commented out for the "firstSearcher" event. There is no point in auto-warming your Index Searcher with the query string "static firstSearcher warming in solrconfig.xml" if that is not relevant to your search application. ==== diff --git a/solr/solr-ref-guide/src/read-and-write-side-fault-tolerance.adoc b/solr/solr-ref-guide/src/read-and-write-side-fault-tolerance.adoc index 07ebd167f50..947c76017f2 100644 --- a/solr/solr-ref-guide/src/read-and-write-side-fault-tolerance.adoc +++ b/solr/solr-ref-guide/src/read-and-write-side-fault-tolerance.adoc @@ -30,7 +30,7 @@ In a SolrCloud cluster each individual node load balances read requests across a Even if some nodes in the cluster are offline or unreachable, a Solr node will be able to correctly respond to a search request as long as it can communicate with at least one replica of every shard, or one replica of every _relevant_ shard if the user limited the search via the `shards` or `\_route_` parameters. The more replicas there are of every shard, the more likely that the Solr cluster will be able to handle search results in the event of node failures. [[ReadandWriteSideFaultTolerance-zkConnected]] -=== `zkConnected` +=== zkConnected A Solr node will return the results of a search request as long as it can communicate with at least one replica of every shard that it knows about, even if it can _not_ communicate with ZooKeeper at the time it receives the request. This is normally the preferred behavior from a fault tolerance standpoint, but may result in stale or incorrect results if there have been major changes to the collection structure that the node has not been informed of via ZooKeeper (i.e., shards may have been added or removed, or split into sub-shards) @@ -57,7 +57,7 @@ A `zkConnected` header is included in every search response indicating if the no ---- [[ReadandWriteSideFaultTolerance-shards.tolerant]] -=== `shards.tolerant` +=== shards.tolerant In the event that one or more shards queried are completely unavailable, then Solr's default behavior is to fail the request. However, there are many use-cases where partial results are acceptable and so Solr provides a boolean `shards.tolerant` parameter (default `false`). diff --git a/solr/solr-ref-guide/src/request-parameters-api.adoc b/solr/solr-ref-guide/src/request-parameters-api.adoc index 0c53e98598a..45275d0760e 100644 --- a/solr/solr-ref-guide/src/request-parameters-api.adoc +++ b/solr/solr-ref-guide/src/request-parameters-api.adoc @@ -148,7 +148,7 @@ curl http://localhost:8983/solr/techproducts/config/params/myQueries ---- [[RequestParametersAPI-TheuseParamsParameter]] -== The `useParams` Parameter +== The useParams Parameter When making a request, the `useParams` parameter applies the request parameters sent to the request. This is translated at request time to the actual parameters. diff --git a/solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adoc b/solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adoc index 3243f4a40ac..2883f9af368 100644 --- a/solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adoc @@ -23,7 +23,7 @@ The `requestDispatcher` element of `solrconfig.xml` controls the way the Solr HT Included are parameters for defining if it should handle `/select` urls (for Solr 1.1 compatibility), if it will support remote streaming, the maximum size of file uploads and how it will respond to HTTP cache headers in requests. [[RequestDispatcherinSolrConfig-handleSelectElement]] -== `handleSelect` Element +== handleSelect Element [IMPORTANT] ==== @@ -42,7 +42,7 @@ In recent versions of Solr, a `/select` requestHandler is defined by default, so ---- [[RequestDispatcherinSolrConfig-requestParsersElement]] -== `requestParsers` Element +== requestParsers Element The `` sub-element controls values related to parsing requests. This is an empty XML element that doesn't have any content, only attributes. @@ -65,7 +65,7 @@ The attribute `addHttpRequestToContext` can be used to indicate that the origina ---- [[RequestDispatcherinSolrConfig-httpCachingElement]] -== `httpCaching` Element +== httpCaching Element The `` element controls HTTP cache control headers. Do not confuse these settings with Solr's internal cache configuration. This element controls caching of HTTP responses as defined by the W3C HTTP specifications. @@ -91,7 +91,7 @@ This element allows for three attributes and one sub-element. The attributes of ---- [[RequestDispatcherinSolrConfig-cacheControlElement]] -=== `cacheControl` Element +=== cacheControl Element In addition to these attributes, `` accepts one child element: ``. The content of this element will be sent as the value of the Cache-Control header on HTTP responses. This header is used to modify the default caching behavior of the requesting client. The possible values for the Cache-Control header are defined by the HTTP 1.1 specification in http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9[Section 14.9]. diff --git a/solr/solr-ref-guide/src/response-writers.adoc b/solr/solr-ref-guide/src/response-writers.adoc index 2cb03eea4a3..8d113705ab1 100644 --- a/solr/solr-ref-guide/src/response-writers.adoc +++ b/solr/solr-ref-guide/src/response-writers.adoc @@ -51,7 +51,7 @@ Note that the XSLT Response Writer can be used to convert the XML produced by th The behavior of the XML Response Writer can be driven by the following query parameters. [[ResponseWriters-TheversionParameter]] -=== The `version` Parameter +=== The version Parameter The `version` parameter determines the XML protocol used in the response. Clients are strongly encouraged to _always_ specify the protocol version, so as to ensure that the format of the response they receive does not change unexpectedly if the Solr server is upgraded and a new default format is introduced. @@ -66,7 +66,7 @@ Currently supported version values are: The default value is the latest supported. [[ResponseWriters-ThestylesheetParameter]] -=== The `stylesheet` Parameter +=== The stylesheet Parameter The `stylesheet` parameter can be used to direct Solr to include a `` declaration in the XML response it returns. @@ -78,7 +78,7 @@ Use of the `stylesheet` parameter is discouraged, as there is currently no way t ==== [[ResponseWriters-TheindentParameter]] -=== The `indent` Parameter +=== The indent Parameter If the `indent` parameter is used, and has a non-blank value, then Solr will make some attempts at indenting its XML response to make it more readable by humans. @@ -90,7 +90,7 @@ The default behavior is not to indent. The XSLT Response Writer applies an XML stylesheet to output. It can be used for tasks such as formatting results for an RSS feed. [[ResponseWriters-trParameter]] -=== `tr` Parameter +=== tr Parameter The XSLT Response Writer accepts one parameter: the `tr` parameter, which identifies the XML transformation to use. The transformation must be found in the Solr `conf/xslt` directory. diff --git a/solr/solr-ref-guide/src/schema-factory-definition-in-solrconfig.adoc b/solr/solr-ref-guide/src/schema-factory-definition-in-solrconfig.adoc index 4254e610111..9d0e60d6705 100644 --- a/solr/solr-ref-guide/src/schema-factory-definition-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/schema-factory-definition-in-solrconfig.adoc @@ -55,7 +55,7 @@ If you wish to explicitly configure `ManagedIndexSchemaFactory` the following op With the default configuration shown above, you can use the <> to modify the schema as much as you want, and then later change the value of `mutable` to *false* if you wish to "lock" the schema in place and prevent future changes. [[SchemaFactoryDefinitioninSolrConfig-Classicschema.xml]] -== Classic `schema.xml` +== Classic schema.xml An alternative to using a managed schema is to explicitly configure a `ClassicIndexSchemaFactory`. `ClassicIndexSchemaFactory` requires the use of a `schema.xml` configuration file, and disallows any programatic changes to the Schema at run time. The `schema.xml` file must be edited manually and is only loaded only when the collection is loaded. @@ -65,7 +65,7 @@ An alternative to using a managed schema is to explicitly configure a `ClassicIn ---- [[SchemaFactoryDefinitioninSolrConfig-Switchingfromschema.xmltoManagedSchema]] -=== Switching from `schema.xml` to Managed Schema +=== Switching from schema.xml to Managed Schema If you have an existing Solr collection that uses `ClassicIndexSchemaFactory`, and you wish to convert to use a managed schema, you can simply modify the `solrconfig.xml` to specify the use of the `ManagedIndexSchemaFactory`. @@ -79,7 +79,7 @@ Once Solr is restarted and it detects that a `schema.xml` file exists, but the ` You are now free to use the <> as much as you want to make changes, and remove the `schema.xml.bak`. [[SchemaFactoryDefinitioninSolrConfig-SwitchingfromManagedSchematoManuallyEditedschema.xml]] -=== Switching from Managed Schema to Manually Edited `schema.xml` +=== Switching from Managed Schema to Manually Edited schema.xml If you have started Solr with managed schema enabled and you would like to switch to manually editing a `schema.xml` file, you should take the following steps: diff --git a/solr/solr-ref-guide/src/spatial-search.adoc b/solr/solr-ref-guide/src/spatial-search.adoc index c44ed77d8af..69d130514a3 100644 --- a/solr/solr-ref-guide/src/spatial-search.adoc +++ b/solr/solr-ref-guide/src/spatial-search.adoc @@ -99,7 +99,7 @@ When used with `BBoxField`, additional options are supported: |=== [[SpatialSearch-geofilt]] -=== `geofilt` +=== geofilt The `geofilt` filter allows you to retrieve results based on the geospatial distance (AKA the "great circle distance") from a given point. Another way of looking at it is that it creates a circular shape filter. For example, to find all documents within five kilometers of a given lat/lon point, you could enter `&q=*:*&fq={!geofilt sfield=store}&pt=45.15,-93.85&d=5`. This filter returns all results within a circle of the given radius around the initial point: @@ -107,7 +107,7 @@ image::images/spatial-search/circle.png[image] [[SpatialSearch-bbox]] -=== `bbox` +=== bbox The `bbox` filter is very similar to `geofilt` except it uses the _bounding box_ of the calculated circle. See the blue box in the diagram below. It takes the same parameters as geofilt. @@ -162,7 +162,7 @@ There are four distance function queries: For more information about these function queries, see the section on <>. [[SpatialSearch-geodist]] -=== `geodist` +=== geodist `geodist` is a distance function that takes three optional parameters: `(sfield,latitude,longitude)`. You can use the `geodist` function to sort results by distance or score return results. diff --git a/solr/solr-ref-guide/src/spell-checking.adoc b/solr/solr-ref-guide/src/spell-checking.adoc index c3711926074..adb784a6e69 100644 --- a/solr/solr-ref-guide/src/spell-checking.adoc +++ b/solr/solr-ref-guide/src/spell-checking.adoc @@ -26,7 +26,7 @@ The basis for these suggestions can be terms in a field in Solr, externally crea == Configuring the SpellCheckComponent [[SpellChecking-DefineSpellCheckinsolrconfig.xml]] -=== Define Spell Check in `solrconfig.xml` +=== Define Spell Check in solrconfig.xml The first step is to specify the source of terms in `solrconfig.xml`. There are three approaches to spell checking in Solr, discussed below. @@ -205,12 +205,12 @@ The SpellCheck component accepts the parameters described in the table below. |=== [[SpellChecking-ThespellcheckParameter]] -=== The `spellcheck` Parameter +=== The spellcheck Parameter This parameter turns on SpellCheck suggestions for the request. If *true*, then spelling suggestions will be generated. [[SpellChecking-Thespellcheck.qorqParameter]] -=== The `spellcheck.q` or `q` Parameter +=== The spellcheck.q or q Parameter This parameter specifies the query to spellcheck. If `spellcheck.q` is defined, then it is used; otherwise the original input query is used. The `spellcheck.q` parameter is intended to be the original query, minus any extra markup like field names, boosts, and so on. If the `q` parameter is specified, then the `SpellingQueryConverter` class is used to parse it into tokens; otherwise the <> is used. The choice of which one to use is up to the application. Essentially, if you have a spelling "ready" version in your application, then it is probably better to use `spellcheck.q`. Otherwise, if you just want Solr to do the job, use the `q` parameter. @@ -220,44 +220,44 @@ The SpellingQueryConverter class does not deal properly with non-ASCII character ==== [[SpellChecking-Thespellcheck.buildParameter]] -=== The `spellcheck.build` Parameter +=== The spellcheck.build Parameter If set to *true*, this parameter creates the dictionary that the SolrSpellChecker will use for spell-checking. In a typical search application, you will need to build the dictionary before using the SolrSpellChecker. However, it's not always necessary to build a dictionary first. For example, you can configure the spellchecker to use a dictionary that already exists. The dictionary will take some time to build, so this parameter should not be sent with every request. [[SpellChecking-Thespellcheck.reloadParameter]] -=== The `spellcheck.reload` Parameter +=== The spellcheck.reload Parameter If set to true, this parameter reloads the spellchecker. The results depend on the implementation of `SolrSpellChecker.reload()`. In a typical implementation, reloading the spellchecker means reloading the dictionary. [[SpellChecking-Thespellcheck.countParameter]] -=== The `spellcheck.count` Parameter +=== The spellcheck.count Parameter This parameter specifies the maximum number of suggestions that the spellchecker should return for a term. If this parameter isn't set, the value defaults to 1. If the parameter is set but not assigned a number, the value defaults to 5. If the parameter is set to a positive integer, that number becomes the maximum number of suggestions returned by the spellchecker. [[SpellChecking-Thespellcheck.onlyMorePopularParameter]] -=== The `spellcheck.onlyMorePopular` Parameter +=== The spellcheck.onlyMorePopular Parameter If *true*, Solr will to return suggestions that result in more hits for the query than the existing query. Note that this will return more popular suggestions even when the given query term is present in the index and considered "correct". [[SpellChecking-Thespellcheck.maxResultsForSuggestParameter]] -=== The `spellcheck.maxResultsForSuggest` Parameter +=== The spellcheck.maxResultsForSuggest Parameter For example, if this is set to 5 and the user's query returns 5 or fewer results, the spellchecker will report "correctlySpelled=false" and also offer suggestions (and collations if requested). Setting this greater than zero is useful for creating "did-you-mean?" suggestions for queries that return a low number of hits. [[SpellChecking-Thespellcheck.alternativeTermCountParameter]] -=== The `spellcheck.alternativeTermCount` Parameter +=== The spellcheck.alternativeTermCount Parameter Specify the number of suggestions to return for each query term existing in the index and/or dictionary. Presumably, users will want fewer suggestions for words with docFrequency>0. Also setting this value turns "on" context-sensitive spell suggestions. [[SpellChecking-Thespellcheck.extendedResultsParameter]] -=== The `spellcheck.extendedResults` Parameter +=== The spellcheck.extendedResults Parameter This parameter causes to Solr to include additional information about the suggestion, such as the frequency in the index. [[SpellChecking-Thespellcheck.collateParameter]] -=== The `spellcheck.collate` Parameter +=== The spellcheck.collate Parameter If *true*, this parameter directs Solr to take the best suggestion for each token (if one exists) and construct a new query from the suggestions. For example, if the input query was "jawa class lording" and the best suggestion for "jawa" was "java" and "lording" was "loading", then the resulting collation would be "java class loading". @@ -266,27 +266,27 @@ The spellcheck.collate parameter only returns collations that are guaranteed to NOTE: This only returns a query to be used. It does not actually run the suggested query. [[SpellChecking-Thespellcheck.maxCollationsParameter]] -=== The `spellcheck.maxCollations` Parameter +=== The spellcheck.maxCollations Parameter The maximum number of collations to return. The default is *1*. This parameter is ignored if `spellcheck.collate` is false. [[SpellChecking-Thespellcheck.maxCollationTriesParameter]] -=== The `spellcheck.maxCollationTries` Parameter +=== The spellcheck.maxCollationTries Parameter This parameter specifies the number of collation possibilities for Solr to try before giving up. Lower values ensure better performance. Higher values may be necessary to find a collation that can return results. The default value is `0`, which maintains backwards-compatible (Solr 1.4) behavior (do not check collations). This parameter is ignored if `spellcheck.collate` is false. [[SpellChecking-Thespellcheck.maxCollationEvaluationsParameter]] -=== The `spellcheck.maxCollationEvaluations` Parameter +=== The spellcheck.maxCollationEvaluations Parameter This parameter specifies the maximum number of word correction combinations to rank and evaluate prior to deciding which collation candidates to test against the index. This is a performance safety-net in case a user enters a query with many misspelled words. The default is *10,000* combinations, which should work well in most situations. [[SpellChecking-Thespellcheck.collateExtendedResultsParameter]] -=== The `spellcheck.collateExtendedResults` Parameter +=== The spellcheck.collateExtendedResults Parameter If *true*, this parameter returns an expanded response format detailing the collations Solr found. The default value is *false* and this is ignored if `spellcheck.collate` is false. [[SpellChecking-Thespellcheck.collateMaxCollectDocsParameter]] -=== The `spellcheck.collateMaxCollectDocs` Parameter +=== The spellcheck.collateMaxCollectDocs Parameter This parameter specifies the maximum number of documents that should be collect when testing potential collations against the index. A value of *0* indicates that all documents should be collected, resulting in exact hit-counts. Otherwise an estimation is provided as a performance optimization in cases where exact hit-counts are unnecessary – the higher the value specified, the more precise the estimation. @@ -294,23 +294,23 @@ The default value for this parameter is *0*, but when `spellcheck.collateExtende [[SpellChecking-Thespellcheck.collateParam._ParameterPrefix]] -=== The `spellcheck.collateParam.*` Parameter Prefix +=== The spellcheck.collateParam.* Parameter Prefix This parameter prefix can be used to specify any additional parameters that you wish to the Spellchecker to use when internally validating collation queries. For example, even if your regular search results allow for loose matching of one or more query terms via parameters like `q.op=OR` and `mm=20%` you can specify override params such as `spellcheck.collateParam.q.op=AND&spellcheck.collateParam.mm=100%` to require that only collations consisting of words that are all found in at least one document may be returned. [[SpellChecking-Thespellcheck.dictionaryParameter]] -=== The `spellcheck.dictionary` Parameter +=== The spellcheck.dictionary Parameter This parameter causes Solr to use the dictionary named in the parameter's argument. The default setting is "default". This parameter can be used to invoke a specific spellchecker on a per request basis. [[SpellChecking-Thespellcheck.accuracyParameter]] -=== The `spellcheck.accuracy` Parameter +=== The spellcheck.accuracy Parameter Specifies an accuracy value to be used by the spell checking implementation to decide whether a result is worthwhile or not. The value is a float between 0 and 1. Defaults to `Float.MIN_VALUE`. [[spellcheck_DICT_NAME]] -=== The `spellcheck..key` Parameter +=== The spellcheck..key Parameter Specifies a key/value pair for the implementation handling a given dictionary. The value that is passed through is just `key=value` (`spellcheck..` is stripped off. diff --git a/solr/solr-ref-guide/src/the-dismax-query-parser.adoc b/solr/solr-ref-guide/src/the-dismax-query-parser.adoc index ea250b1182c..d624d7c7450 100644 --- a/solr/solr-ref-guide/src/the-dismax-query-parser.adoc +++ b/solr/solr-ref-guide/src/the-dismax-query-parser.adoc @@ -58,7 +58,7 @@ In addition to the common request parameter, highlighting parameters, and simple The sections below explain these parameters in detail. [[TheDisMaxQueryParser-TheqParameter]] -=== The `q` Parameter +=== The q Parameter The `q` parameter defines the main "query" constituting the essence of the search. The parameter supports raw input strings provided by users with no special escaping. The + and - characters are treated as "mandatory" and "prohibited" modifiers for terms. Text wrapped in balanced quote characters (for example, "San Jose") is treated as a phrase. Any query containing an odd number of quote characters is evaluated as if there were no quote characters at all. @@ -70,13 +70,13 @@ The `q` parameter does not support wildcard characters such as *. ==== [[TheDisMaxQueryParser-Theq.altParameter]] -=== The `q.alt` Parameter +=== The q.alt Parameter If specified, the `q.alt` parameter defines a query (which by default will be parsed using standard query parsing syntax) when the main q parameter is not specified or is blank. The `q.alt` parameter comes in handy when you need something like a query to match all documents (don't forget `&rows=0` for that one!) in order to get collection-wide faceting counts. [[TheDisMaxQueryParser-Theqf_QueryFields_Parameter]] -=== The `qf` (Query Fields) Parameter +=== The qf (Query Fields) Parameter The `qf` parameter introduces a list of fields, each of which is assigned a boost factor to increase or decrease that particular field's importance in the query. For example, the query below: @@ -86,7 +86,7 @@ assigns `fieldOne` a boost of 2.3, leaves `fieldTwo` with the default boost (bec [[TheDisMaxQueryParser-Themm_MinimumShouldMatch_Parameter]] -=== The `mm` (Minimum Should Match) Parameter +=== The mm (Minimum Should Match) Parameter When processing queries, Lucene/Solr recognizes three types of clauses: mandatory, prohibited, and "optional" (also known as "should" clauses). By default, all words or phrases specified in the `q` parameter are treated as "optional" clauses unless they are preceded by a "+" or a "-". When dealing with these "optional" clauses, the `mm` parameter makes it possible to say that a certain minimum number of those clauses must match. The DisMax query parser offers great flexibility in how the minimum number can be specified. @@ -116,7 +116,7 @@ The default value of `mm` is 100% (meaning that all clauses must match). [[TheDisMaxQueryParser-Thepf_PhraseFields_Parameter]] -=== The `pf` (Phrase Fields) Parameter +=== The pf (Phrase Fields) Parameter Once the list of matching documents has been identified using the `fq` and `qf` parameters, the `pf` parameter can be used to "boost" the score of documents in cases where all of the terms in the q parameter appear in close proximity. @@ -124,19 +124,19 @@ The format is the same as that used by the `qf` parameter: a list of fields and [[TheDisMaxQueryParser-Theps_PhraseSlop_Parameter]] -=== The `ps` (Phrase Slop) Parameter +=== The ps (Phrase Slop) Parameter The `ps` parameter specifies the amount of "phrase slop" to apply to queries specified with the pf parameter. Phrase slop is the number of positions one token needs to be moved in relation to another token in order to match a phrase specified in a query. [[TheDisMaxQueryParser-Theqs_QueryPhraseSlop_Parameter]] -=== The `qs` (Query Phrase Slop) Parameter +=== The qs (Query Phrase Slop) Parameter The `qs` parameter specifies the amount of slop permitted on phrase queries explicitly included in the user's query string with the `qf` parameter. As explained above, slop refers to the number of positions one token needs to be moved in relation to another token in order to match a phrase specified in a query. [[TheDisMaxQueryParser-Thetie_TieBreaker_Parameter]] -=== The `tie` (Tie Breaker) Parameter +=== The tie (Tie Breaker) Parameter The `tie` parameter specifies a float value (which should be something much less than 1) to use as tiebreaker in DisMax queries. @@ -146,7 +146,7 @@ A value of "0.0" - the default - makes the query a pure "disjunction max query": [[TheDisMaxQueryParser-Thebq_BoostQuery_Parameter]] -=== The `bq` (Boost Query) Parameter +=== The bq (Boost Query) Parameter The `bq` parameter specifies an additional, optional, query clause that will be added to the user's main query to influence the score. For example, if you wanted to add a relevancy boost for recent documents: @@ -160,7 +160,7 @@ You can specify multiple `bq` parameters. If you want your query to be parsed as [[TheDisMaxQueryParser-Thebf_BoostFunctions_Parameter]] -=== The `bf` (Boost Functions) Parameter +=== The bf (Boost Functions) Parameter The `bf` parameter specifies functions (with optional boosts) that will be used to construct FunctionQueries which will be added to the user's main query as optional clauses that will influence the score. Any function supported natively by Solr can be used, along with a boost value. For example: diff --git a/solr/solr-ref-guide/src/the-extended-dismax-query-parser.adoc b/solr/solr-ref-guide/src/the-extended-dismax-query-parser.adoc index fff5909d5ca..c0fdcaeeab5 100644 --- a/solr/solr-ref-guide/src/the-extended-dismax-query-parser.adoc +++ b/solr/solr-ref-guide/src/the-extended-dismax-query-parser.adoc @@ -39,64 +39,64 @@ In addition to supporting all the DisMax query parser parameters, Extended Disma In addition to all the <>, Extended DisMax includes these query parameters: [[TheExtendedDisMaxQueryParser-ThesowParameter]] -=== The `sow` Parameter +=== The sow Parameter Split on whitespace: if set to `false`, whitespace-separated term sequences will be provided to text analysis in one shot, enabling proper function of analysis filters that operate over term sequences, e.g. multi-word synonyms and shingles. Defaults to `true`: text analysis is invoked separately for each individual whitespace-separated term. [[TheExtendedDisMaxQueryParser-Themm.autoRelaxParameter]] -=== The `mm.autoRelax` Parameter +=== The mm.autoRelax Parameter If true, the number of clauses required (<>) will automatically be relaxed if a clause is removed (by e.g. stopwords filter) from some but not all <> fields. Use this parameter as a workaround if you experience that queries return zero hits due to uneven stopword removal between the `qf` fields. Note that relaxing mm may cause undesired side effects, hurting the precision of the search, depending on the nature of your index content. [[TheExtendedDisMaxQueryParser-TheboostParameter]] -=== The `boost` Parameter +=== The boost Parameter A multivalued list of strings parsed as queries with scores multiplied by the score from the main query for all matching documents. This parameter is shorthand for wrapping the query produced by eDisMax using the `BoostQParserPlugin` [[TheExtendedDisMaxQueryParser-ThelowercaseOperatorsParameter]] -=== The `lowercaseOperators` Parameter +=== The lowercaseOperators Parameter A Boolean parameter indicating if lowercase "and" and "or" should be treated the same as operators "AND" and "OR". [[TheExtendedDisMaxQueryParser-ThepsParameter]] -=== The `ps` Parameter +=== The ps Parameter Default amount of slop on phrase queries built with `pf`, `pf2` and/or `pf3` fields (affects boosting). [[TheExtendedDisMaxQueryParser-Thepf2Parameter]] -=== The `pf2` Parameter +=== The pf2 Parameter A multivalued list of fields with optional weights, based on pairs of word shingles. [[TheExtendedDisMaxQueryParser-Theps2Parameter]] -=== The `ps2` Parameter +=== The ps2 Parameter This is similar to `ps` but overrides the slop factor used for `pf2`. If not specified, `ps` is used. [[TheExtendedDisMaxQueryParser-Thepf3Parameter]] -=== The `pf3` Parameter +=== The pf3 Parameter A multivalued list of fields with optional weights, based on triplets of word shingles. Similar to `pf`, except that instead of building a phrase per field out of all the words in the input, it builds a set of phrases for each field out of each triplet of word shingles. [[TheExtendedDisMaxQueryParser-Theps3Parameter]] -=== The `ps3` Parameter +=== The ps3 Parameter This is similar to `ps` but overrides the slop factor used for `pf3`. If not specified, `ps` is used. [[TheExtendedDisMaxQueryParser-ThestopwordsParameter]] -=== The `stopwords` Parameter +=== The stopwords Parameter A Boolean parameter indicating if the `StopFilterFactory` configured in the query analyzer should be respected when parsing the query: if it is false, then the `StopFilterFactory` in the query analyzer is ignored. [[TheExtendedDisMaxQueryParser-TheufParameter]] -=== The `uf` Parameter +=== The uf Parameter Specifies which schema fields the end user is allowed to explicitly query. This parameter supports wildcards. The default is to allow all fields, equivalent to `uf=\*`. To allow only title field, use `uf=title`. To allow title and all fields ending with '_s', use `uf=title,*_s`. To allow all fields except title, use `uf=*,-title`. To disallow all fielded searches, use `uf=-*`. [[TheExtendedDisMaxQueryParser-Fieldaliasingusingper-fieldqfoverrides]] -=== Field aliasing using per-field `qf` overrides +=== Field aliasing using per-field qf overrides Per-field overrides of the `qf` parameter may be specified to provide 1-to-many aliasing from field names specified in the query string, to field names used in the underlying query. By default, no aliasing is used and field names specified in the query string are treated as literal field names in the index. @@ -223,7 +223,7 @@ Finally, in addition to the phrase fields (`pf`) parameter, `edismax` also suppo [[TheExtendedDisMaxQueryParser-Usingthe_magicfields__val_and_query_]] -== Using the 'magic fields' `\_val_` and `\_query_` +== Using the "magic fields" \_val_ and \_query_ The Solr Query Parser's use of `\_val_` and `\_query_` differs from the Lucene Query Parser in the following ways: diff --git a/solr/solr-ref-guide/src/the-query-elevation-component.adoc b/solr/solr-ref-guide/src/the-query-elevation-component.adoc index f70ed88b576..9898a082216 100644 --- a/solr/solr-ref-guide/src/the-query-elevation-component.adoc +++ b/solr/solr-ref-guide/src/the-query-elevation-component.adoc @@ -74,7 +74,7 @@ The Query Elevation Search Component takes the following arguments: |=== [[TheQueryElevationComponent-elevate.xml]] -=== `elevate.xml` +=== elevate.xml Elevated query results are configured in an external XML file specified in the `config-file` argument. An `elevate.xml` file might look like this: @@ -100,7 +100,7 @@ In this example, the query "foo bar" would first return documents 1, 2 and 3, th == Using the Query Elevation Component [[TheQueryElevationComponent-TheenableElevationParameter]] -=== The `enableElevation` Parameter +=== The enableElevation Parameter For debugging it may be useful to see results with and without the elevated docs. To hide results, use `enableElevation=false`: @@ -109,21 +109,21 @@ For debugging it may be useful to see results with and without the elevated docs `\http://localhost:8983/solr/techproducts/elevate?q=ipod&df=text&debugQuery=true&enableElevation=false` [[TheQueryElevationComponent-TheforceElevationParameter]] -=== The `forceElevation` Parameter +=== The forceElevation Parameter You can force elevation during runtime by adding `forceElevation=true` to the query URL: `\http://localhost:8983/solr/techproducts/elevate?q=ipod&df=text&debugQuery=true&enableElevation=true&forceElevation=true` [[TheQueryElevationComponent-TheexclusiveParameter]] -=== The `exclusive` Parameter +=== The exclusive Parameter You can force Solr to return only the results specified in the elevation file by adding `exclusive=true` to the URL: `\http://localhost:8983/solr/techproducts/elevate?q=ipod&df=text&debugQuery=true&exclusive=true` [[TheQueryElevationComponent-DocumentTransformersandthemarkExcludesParameter]] -=== Document Transformers and the `markExcludes` Parameter +=== Document Transformers and the markExcludes Parameter The `[elevated]` <> can be used to annotate each document with information about whether or not it was elevated: @@ -134,7 +134,7 @@ Likewise, it can be helpful when troubleshooting to see all matching documents `\http://localhost:8983/solr/techproducts/elevate?q=ipod&df=text&markExcludes=true&fl=id,[elevated],[excluded]` [[TheQueryElevationComponent-TheelevateIdsandexcludeIdsParameters]] -=== The `elevateIds` and `excludeIds` Parameters +=== The elevateIds and excludeIds Parameters When the elevation component is in use, the pre-configured list of elevations for a query can be overridden at request time to use the unique keys specified in these request parameters. @@ -149,6 +149,6 @@ For example, in the request below documents IW-02 and F8V7067-APL-KIT will be el `\http://localhost:8983/solr/techproducts/elevate?q=ipod&df=text&elevateIds=IW-02,F8V7067-APL-KIT` [[TheQueryElevationComponent-ThefqParameter]] -=== The `fq` Parameter +=== The fq Parameter Query elevation respects the standard filter query (`fq`) parameter. That is, if the query contains the `fq` parameter, all results will be within that filter even if `elevate.xml` adds other documents to the result set. diff --git a/solr/solr-ref-guide/src/the-standard-query-parser.adoc b/solr/solr-ref-guide/src/the-standard-query-parser.adoc index 94baedf6f25..f389b92a962 100644 --- a/solr/solr-ref-guide/src/the-standard-query-parser.adoc +++ b/solr/solr-ref-guide/src/the-standard-query-parser.adoc @@ -187,7 +187,7 @@ The brackets around a query determine its inclusiveness. [[TheStandardQueryParser-BoostingaTermwith_]] -=== Boosting a Term with `^` +=== Boosting a Term with "^" Lucene/Solr provides the relevance level of matching documents based on the terms found. To boost a term use the caret symbol `^` with a boost factor (a number) at the end of the term you are searching. The higher the boost factor, the more relevant the term will be. @@ -205,7 +205,7 @@ By default, the boost factor is 1. Although the boost factor must be positive, i [[TheStandardQueryParser-ConstantScorewith_]] -=== Constant Score with `^=` +=== Constant Score with "^=" Constant score queries are created with `^=`, which sets the entire clause to the specified score for any documents matching that clause. This is desirable when you only care about matches for a particular clause and don't want other relevancy factors such as term frequency (the number of times the term appears in the field) or inverse document frequency (a measure across the whole index for how rare a term is in a field). @@ -279,7 +279,7 @@ or [[TheStandardQueryParser-TheBooleanOperator_]] -=== The Boolean Operator `+` +=== The Boolean Operator "+" The `+` symbol (also known as the "required" operator) requires that the term after the `+` symbol exist somewhere in a field in at least one document in order for the query to return a match. @@ -296,7 +296,7 @@ This operator is supported by both the standard query parser and the DisMax quer [[TheStandardQueryParser-TheBooleanOperatorAND_]] -=== The Boolean Operator AND (`&&`) +=== The Boolean Operator AND ("&&") The AND operator matches documents where both terms exist anywhere in the text of a single document. This is equivalent to an intersection using sets. The symbol `&&` can be used in place of the word AND. @@ -308,7 +308,7 @@ To search for documents that contain "jakarta apache" and "Apache Lucene," use e [[TheStandardQueryParser-TheBooleanOperatorNOT_]] -=== The Boolean Operator NOT (`!`) +=== The Boolean Operator NOT ("!") The NOT operator excludes documents that contain the term after NOT. This is equivalent to a difference using sets. The symbol `!` can be used in place of the word NOT. @@ -319,7 +319,7 @@ The following queries search for documents that contain the phrase "jakarta apac `"jakarta apache" ! "Apache Lucene"` [[TheStandardQueryParser-TheBooleanOperator-]] -=== The Boolean Operator `-` +=== The Boolean Operator "-" The `-` symbol or "prohibit" operator excludes documents that contain the term after the `-` symbol. diff --git a/solr/solr-ref-guide/src/transforming-result-documents.adoc b/solr/solr-ref-guide/src/transforming-result-documents.adoc index 95d373e704e..feb69318a79 100644 --- a/solr/solr-ref-guide/src/transforming-result-documents.adoc +++ b/solr/solr-ref-guide/src/transforming-result-documents.adoc @@ -51,7 +51,7 @@ The sections below discuss exactly what these various transformers do. [[TransformingResultDocuments-_value_-ValueAugmenterFactory]] -=== `[value]` - ValueAugmenterFactory +=== [value] - ValueAugmenterFactory Modifies every document to include the exact same value, as if it were a stored field in every document: @@ -95,7 +95,7 @@ The "```value```" option forces an explicit value to always be used, while the " [[TransformingResultDocuments-_explain_-ExplainAugmenterFactory]] -=== `[explain]` - ExplainAugmenterFactory +=== [explain] - ExplainAugmenterFactory Augments each document with an inline explanation of its score exactly like the information available about each document in the debug section: @@ -116,7 +116,7 @@ Supported values for "```style```" are "```text```", and "```html```", and "nl" "value":1.052226, "description":"weight(features:cache in 2) [DefaultSimilarity], result of:", "details":[{ -... +}]}}]}} ---- A default style can be configured by specifying an "args" parameter in your configuration: @@ -130,7 +130,7 @@ A default style can be configured by specifying an "args" parameter in your conf [[TransformingResultDocuments-_child_-ChildDocTransformerFactory]] -=== `[child]` - ChildDocTransformerFactory +=== [child] - ChildDocTransformerFactory This transformer returns all <> of each parent document matching your query in a flat list nested inside the matching parent document. This is useful when you have indexed nested child documents and want to retrieve the child documents for the relevant parent documents for any type of search query. @@ -148,7 +148,7 @@ When using this transformer, the `parentFilter` parameter must be specified, and [[TransformingResultDocuments-_shard_-ShardAugmenterFactory]] -=== `[shard]` - ShardAugmenterFactory +=== [shard] - ShardAugmenterFactory This transformer adds information about what shard each individual document came from in a distributed request. @@ -156,7 +156,7 @@ ShardAugmenterFactory does not support any request parameters, or configuration [[TransformingResultDocuments-_docid_-DocIdAugmenterFactory]] -=== `[docid]` - DocIdAugmenterFactory +=== [docid] - DocIdAugmenterFactory This transformer adds the internal Lucene document id to each document – this is primarily only useful for debugging purposes. @@ -164,7 +164,7 @@ DocIdAugmenterFactory does not support any request parameters, or configuration [[TransformingResultDocuments-_elevated_and_excluded_]] -=== `[elevated]` and `[excluded]` +=== [elevated] and [excluded] These transformers are available only when using the <>. @@ -191,12 +191,12 @@ fl=id,[elevated],[excluded]&excludeIds=GB18030TEST&elevateIds=6H500F0&markExclud "id":"SP2514N", "[elevated]":false, "[excluded]":false}, -... +]}} ---- [[TransformingResultDocuments-_json_xml_]] -=== `[json]` / `[xml]` +=== [json] / [xml] These transformers replace field value containing a string representation of a valid XML or JSON structure with the actual raw XML or JSON structure rather than just the string value. Each applies only to the specific writer, such that `[json]` only applies to `wt=json` and `[xml]` only applies to `wt=xml`. @@ -207,7 +207,7 @@ fl=id,source_s:[json]&wt=json [[TransformingResultDocuments-_subquery_]] -=== `[subquery]` +=== [subquery] This transformer executes a separate query per transforming document passing document fields as an input for subquery parameters. It's usually used with `{!join}` and `{!parent}` query parsers, and is intended to be an improvement for `[child]`. @@ -246,17 +246,14 @@ Here is how it looks like in various formats: "id":1, "subject":["parentDocument"], "title":["xrxvomgu"], - "children":{ + "children":{ "numFound":1, "start":0, "docs":[ { "id":2, "cat":["childDocument"] } ] - }}, - { - "id":4, - ... + }}]}} ---- [source,java] @@ -311,7 +308,7 @@ If subquery collection has a different unique key field name (let's say `foo_id` [[TransformingResultDocuments-_geo_-Geospatialformatter]] -=== `[geo]` - Geospatial formatter +=== [geo] - Geospatial formatter Formats spatial data from a spatial field using a designated format type name. Two inner parameters are required: `f` for the field name, and `w` for the format name. Example: `geojson:[geo f=mySpatialField w=GeoJSON]`. @@ -321,7 +318,7 @@ In addition, this feature is very useful with the `RptWithGeometrySpatialField` [[TransformingResultDocuments-_features_-LTRFeatureLoggerTransformerFactory]] -=== `[features]` - LTRFeatureLoggerTransformerFactory +=== [features] - LTRFeatureLoggerTransformerFactory The "LTR" prefix stands for <>. This transformer returns the values of features and it can be used for feature extraction and feature logging. diff --git a/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc b/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc index 7afc2e15b42..664bd8c8e38 100644 --- a/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc @@ -33,7 +33,7 @@ The settings in this section are configured in the `` element in Data sent to Solr is not searchable until it has been _committed_ to the index. The reason for this is that in some cases commits can be slow and they should be done in isolation from other possible commit requests to avoid overwriting data. So, it's preferable to provide control over when data is committed. Several options are available to control the timing of commits. [[UpdateHandlersinSolrConfig-commitandsoftCommit]] -=== `commit` and `softCommit` +=== commit and softCommit In Solr, a `commit` is an action which asks Solr to "commit" those changes to the Lucene index files. By default commit actions result in a "hard commit" of all the Lucene index files to stable storage (disk). When a client includes a `commit=true` parameter with an update request, this ensures that all index segments affected by the adds & deletes on an update are written to disk as soon as index updates are completed. @@ -42,7 +42,7 @@ If an additional flag `softCommit=true` is specified, then Solr performs a 'soft For more information about Near Real Time operations, see <>. [[UpdateHandlersinSolrConfig-autoCommit]] -=== `autoCommit` +=== autoCommit These settings control how often pending updates will be automatically pushed to the index. An alternative to `autoCommit` is to use `commitWithin`, which can be defined when making the update request to Solr (i.e., when pushing documents), or in an update RequestHandler. @@ -79,7 +79,7 @@ You can also specify 'soft' autoCommits in the same way that you can specify 'so ---- [[UpdateHandlersinSolrConfig-commitWithin]] -=== `commitWithin` +=== commitWithin The `commitWithin` settings allow forcing document commits to happen in a defined time period. This is used most frequently with <>, and for that reason the default is to perform a soft commit. This does not, however, replicate new documents to slave servers in a master/slave environment. If that's a requirement for your implementation, you can force a hard commit by adding a parameter, as in this example: diff --git a/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc b/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc index 30a2c3ae0c2..8bad5f58062 100644 --- a/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc +++ b/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc @@ -148,7 +148,7 @@ When using the Join query parser in a Delete By Query, you should use the `score The rollback command rolls back all add and deletes made to the index since the last commit. It neither calls any event listeners nor creates a new searcher. Its syntax is simple: ``. [[UploadingDatawithIndexHandlers-UsingcurltoPerformUpdates]] -=== Using `curl` to Perform Updates +=== Using curl to Perform Updates You can use the `curl` utility to perform any of the above commands, using its `--data-binary` option to append the XML message to the `curl` command, and generating a HTTP POST request. For example: diff --git a/solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adoc b/solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adoc index e1c8414fb72..670ef2b90ed 100644 --- a/solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adoc +++ b/solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adoc @@ -48,7 +48,7 @@ While Apache Tika is quite powerful, it is not perfect and fails on some files. ==== [[UploadingDatawithSolrCellusingApacheTika-TryingoutTikawiththeSolrtechproductsExample]] -== Trying out Tika with the Solr `techproducts` Example +== Trying out Tika with the Solr techproducts Example You can try out the Tika framework using the `techproducts` example included in Solr. @@ -138,7 +138,7 @@ Here is the order in which the Solr Cell framework, using the Extracting Request 4. If `uprefix` is specified, any unknown field names are prefixed with that value, else if `defaultField` is specified, any unknown fields are copied to the default field. [[UploadingDatawithSolrCellusingApacheTika-ConfiguringtheSolrExtractingRequestHandler]] -== Configuring the Solr `ExtractingRequestHandler` +== Configuring the Solr ExtractingRequestHandler If you are not working with the supplied `sample_techproducts_configs `or` data_driven_schema_configs` <>, you must configure your own `solrconfig.xml` to know about the Jar's containing the `ExtractingRequestHandler` and its dependencies: diff --git a/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc b/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc index b2dc9d56b2c..a9e0d713c3a 100644 --- a/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc +++ b/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc @@ -58,7 +58,7 @@ It's a good idea to keep these files under version control. [[UsingZooKeepertoManageConfigurationFiles-UploadingConfigurationFilesusingbin_solrorSolrJ]] -== Uploading Configuration Files using `bin/solr` or SolrJ +== Uploading Configuration Files using bin/solr or SolrJ In production situations, <> can also be uploaded to ZooKeeper independent of collection creation using either Solr's <> or the {solr-javadocs}/solr-solrj/org/apache/solr/client/solrj/impl/CloudSolrClient.html[CloudSolrClient.uploadConfig] java method. diff --git a/solr/solr-ref-guide/src/working-with-dates.adoc b/solr/solr-ref-guide/src/working-with-dates.adoc index f0ac2422931..7d30b6605cd 100644 --- a/solr/solr-ref-guide/src/working-with-dates.adoc +++ b/solr/solr-ref-guide/src/working-with-dates.adoc @@ -53,8 +53,8 @@ As you can see, the date format includes colon characters separating the hours, This is normally an invalid query: `datefield:1972-05-20T17:33:18.772Z` -These are valid queries: + -`datefield:1972-05-20T17\:33\:18.772Z` + +These are valid queries: + +`datefield:1972-05-20T17\:33\:18.772Z` + `datefield:"1972-05-20T17:33:18.772Z"` + `datefield:[1972-05-20T17:33:18.772Z TO *]` @@ -108,7 +108,7 @@ Note that while date math is most commonly used relative to `NOW` it can be appl === Request Parameters That Affect Date Math [[WorkingwithDates-NOW]] -==== `NOW` +==== NOW The `NOW` parameter is used internally by Solr to ensure consistent date math expression parsing across multiple nodes in a distributed request. But it can be specified to instruct Solr to use an arbitrary moment in time (past or future) to override for all situations where the the special value of "```NOW```" would impact date math expressions. @@ -119,7 +119,7 @@ Example: `q=solr&fq=start_date:[* TO NOW]&NOW=1384387200000` [[WorkingwithDates-TZ]] -==== `TZ` +==== TZ By default, all date math expressions are evaluated relative to the UTC TimeZone, but the `TZ` parameter can be specified to override this behaviour, by forcing all date based addition and rounding to be relative to the specified http://docs.oracle.com/javase/8/docs/api/java/util/TimeZone.html[time zone]. diff --git a/solr/solr-ref-guide/src/working-with-enum-fields.adoc b/solr/solr-ref-guide/src/working-with-enum-fields.adoc index c4d49b339e5..8931543a036 100644 --- a/solr/solr-ref-guide/src/working-with-enum-fields.adoc +++ b/solr/solr-ref-guide/src/working-with-enum-fields.adoc @@ -21,7 +21,7 @@ The EnumField type allows defining a field whose values are a closed set, and the sort order is pre-determined but is not alphabetic nor numeric. Examples of this are severity lists, or risk definitions. [[WorkingwithEnumFields-DefininganEnumFieldinschema.xml]] -== Defining an EnumField in `schema.xml` +== Defining an EnumField in schema.xml The EnumField type definition is quite simple, as in this example defining field types for "priorityLevel" and "riskLevel" enumerations: @@ -52,7 +52,7 @@ In this example, there are two value lists defined. Each list is between `enum` Low Medium High - Urgent + Urgent Unknown @@ -60,7 +60,7 @@ In this example, there are two value lists defined. Each list is between `enum` Low Medium High - Critical + Critical ---- diff --git a/solr/solr-ref-guide/src/working-with-external-files-and-processes.adoc b/solr/solr-ref-guide/src/working-with-external-files-and-processes.adoc index 31abc186672..3aa0195dc3b 100644 --- a/solr/solr-ref-guide/src/working-with-external-files-and-processes.adoc +++ b/solr/solr-ref-guide/src/working-with-external-files-and-processes.adoc @@ -19,7 +19,7 @@ // under the License. [[WorkingwithExternalFilesandProcesses-TheExternalFileFieldType]] -== The `ExternalFileField` Type +== The ExternalFileField Type The `ExternalFileField` type makes it possible to specify the values for a field in a file outside the Solr index. For such a field, the file contains mappings from a key field to the field value. Another way to think of this is that, instead of specifying the field in documents as they are indexed, Solr finds values for this field in the external file. @@ -74,7 +74,7 @@ It's possible to define an event listener to reload an external file when either ---- [[WorkingwithExternalFilesandProcesses-ThePreAnalyzedFieldType]] -== The `PreAnalyzedField` Type +== The PreAnalyzedField Type The `PreAnalyzedField` type provides a way to send to Solr serialized token streams, optionally with independent stored values of a field, and have this information stored and indexed without any additional text processing applied in Solr. This is useful if user wants to submit field content that was already processed by some existing external text processing pipeline (e.g., it has been tokenized, annotated, stemmed, synonyms inserted, etc.), while using all the rich attributes that Lucene's TokenStream provides (per-token attributes). From c51f6fae75b7b2de1cbe77a13b76d6e08e9fa35c Mon Sep 17 00:00:00 2001 From: Tomas Fernandez Lobbe Date: Mon, 12 Jun 2017 14:04:16 -0700 Subject: [PATCH 063/131] SOLR-10835: Add support for point fields in Export Handler --- solr/CHANGES.txt | 2 + .../org/apache/solr/handler/ExportWriter.java | 105 ++++++--- .../conf/schema-sortingresponse.xml | 71 +++++- .../solr/response/TestExportWriter.java | 203 +++++++++++++++++- 4 files changed, 330 insertions(+), 51 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index e9a22db2953..9135cfa567b 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -360,6 +360,8 @@ Bug Fixes * SOLR-10715: /v2/ should not be an alias for /v2/collections (Cao Manh Dat) +* SOLR-10835: Add support for point fields in Export Handler (Tomás Fernández Löbbe) + Optimizations ---------------------- * SOLR-10634: JSON Facet API: When a field/terms facet will retrieve all buckets (i.e. limit:-1) diff --git a/solr/core/src/java/org/apache/solr/handler/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/ExportWriter.java index 46ec3a4a394..61f937ce05e 100644 --- a/solr/core/src/java/org/apache/solr/handler/ExportWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/ExportWriter.java @@ -17,6 +17,10 @@ package org.apache.solr.handler; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.apache.solr.common.util.Utils.makeMap; + import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; @@ -26,6 +30,7 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.Date; import java.util.List; +import java.util.function.LongFunction; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexableField; @@ -34,6 +39,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Sort; @@ -44,6 +50,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRefBuilder; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.LongValues; +import org.apache.lucene.util.NumericUtils; import org.apache.solr.client.solrj.impl.BinaryResponseParser; import org.apache.solr.common.IteratorWriter; import org.apache.solr.common.MapWriter; @@ -60,25 +67,21 @@ import org.apache.solr.response.JSONResponseWriter; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.BoolField; +import org.apache.solr.schema.DateValueFieldType; +import org.apache.solr.schema.DoubleValueFieldType; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.FloatValueFieldType; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.IntValueFieldType; +import org.apache.solr.schema.LongValueFieldType; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.StrField; -import org.apache.solr.schema.TrieDateField; -import org.apache.solr.schema.TrieDoubleField; -import org.apache.solr.schema.TrieFloatField; -import org.apache.solr.schema.TrieIntField; -import org.apache.solr.schema.TrieLongField; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SortSpec; import org.apache.solr.search.SyntaxError; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; -import static org.apache.solr.common.util.Utils.makeMap; - public class ExportWriter implements SolrCore.RawWriter, Closeable { private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private OutputStreamWriter respWriter; @@ -322,25 +325,25 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { boolean multiValued = schemaField.multiValued(); FieldType fieldType = schemaField.getType(); - if (fieldType instanceof TrieIntField) { + if (fieldType instanceof IntValueFieldType) { if (multiValued) { writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true); } else { writers[i] = new IntFieldWriter(field); } - } else if (fieldType instanceof TrieLongField) { + } else if (fieldType instanceof LongValueFieldType) { if (multiValued) { writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true); } else { writers[i] = new LongFieldWriter(field); } - } else if (fieldType instanceof TrieFloatField) { + } else if (fieldType instanceof FloatValueFieldType) { if (multiValued) { writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true); } else { writers[i] = new FloatFieldWriter(field); } - } else if (fieldType instanceof TrieDoubleField) { + } else if (fieldType instanceof DoubleValueFieldType) { if (multiValued) { writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true); } else { @@ -352,7 +355,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { } else { writers[i] = new StringFieldWriter(field, fieldType); } - } else if (fieldType instanceof TrieDateField) { + } else if (fieldType instanceof DateValueFieldType) { if (multiValued) { writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false); } else { @@ -385,25 +388,25 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { throw new IOException(field+" must have DocValues to use this feature."); } - if(ft instanceof TrieIntField) { + if(ft instanceof IntValueFieldType) { if(reverse) { sortValues[i] = new IntValue(field, new IntDesc()); } else { sortValues[i] = new IntValue(field, new IntAsc()); } - } else if(ft instanceof TrieFloatField) { + } else if(ft instanceof FloatValueFieldType) { if(reverse) { sortValues[i] = new FloatValue(field, new FloatDesc()); } else { sortValues[i] = new FloatValue(field, new FloatAsc()); } - } else if(ft instanceof TrieDoubleField) { + } else if(ft instanceof DoubleValueFieldType) { if(reverse) { sortValues[i] = new DoubleValue(field, new DoubleDesc()); } else { sortValues[i] = new DoubleValue(field, new DoubleAsc()); } - } else if(ft instanceof TrieLongField) { + } else if(ft instanceof LongValueFieldType) { if(reverse) { sortValues[i] = new LongValue(field, new LongDesc()); } else { @@ -417,7 +420,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { } else { sortValues[i] = new StringValue(vals, field, new IntAsc()); } - } else if (ft instanceof TrieDateField) { + } else if (ft instanceof DateValueFieldType) { if (reverse) { sortValues[i] = new LongValue(field, new LongDesc()); } else { @@ -1352,6 +1355,23 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { return true; } } + + static LongFunction bitsToValue(FieldType fieldType) { + switch (fieldType.getNumberType()) { + case LONG: + return (bits)-> bits; + case DATE: + return (bits)-> new Date(bits); + case INTEGER: + return (bits)-> (int)bits; + case FLOAT: + return (bits)-> NumericUtils.sortableIntToFloat((int)bits); + case DOUBLE: + return (bits)-> NumericUtils.sortableLongToDouble(bits); + default: + throw new AssertionError("Unsupported NumberType: " + fieldType.getNumberType()); + } + } class MultiFieldWriter extends FieldWriter { private String field; @@ -1359,29 +1379,48 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { private SchemaField schemaField; private boolean numeric; private CharsRefBuilder cref = new CharsRefBuilder(); + private final LongFunction bitsToValue; public MultiFieldWriter(String field, FieldType fieldType, SchemaField schemaField, boolean numeric) { this.field = field; this.fieldType = fieldType; this.schemaField = schemaField; this.numeric = numeric; + if (this.fieldType.isPointField()) { + bitsToValue = bitsToValue(fieldType); + } else { + bitsToValue = null; + } } public boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException { - SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field); - if (vals.advance(docId) != docId) return false; - out.put(this.field, - (IteratorWriter) w -> { - long o; - while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) { - BytesRef ref = vals.lookupOrd(o); - fieldType.indexedToReadable(ref, cref); - IndexableField f = fieldType.createField(schemaField, cref.toString()); - if (f == null) w.add(cref.toString()); - else w.add(fieldType.toObject(f)); - } - }); - return true; + if (this.fieldType.isPointField()) { + SortedNumericDocValues vals = DocValues.getSortedNumeric(reader, this.field); + if (!vals.advanceExact(docId)) return false; + out.put(this.field, + (IteratorWriter) w -> { + for (int i = 0; i < vals.docValueCount(); i++) { + w.add(bitsToValue.apply(vals.nextValue())); + } + }); + return true; + } else { + SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field); + if (vals.advance(docId) != docId) return false; + out.put(this.field, + (IteratorWriter) w -> { + long o; + while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) { + BytesRef ref = vals.lookupOrd(o); + fieldType.indexedToReadable(ref, cref); + IndexableField f = fieldType.createField(schemaField, cref.toString()); + if (f == null) w.add(cref.toString()); + else w.add(fieldType.toObject(f)); + } + }); + return true; + } + } } diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml index ecf0daf6542..6ec93fa9c71 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml @@ -26,6 +26,14 @@ seconds part (.999) is optional. --> + + + + + + + + @@ -33,20 +41,63 @@ - - - - - + + + + + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id diff --git a/solr/core/src/test/org/apache/solr/response/TestExportWriter.java b/solr/core/src/test/org/apache/solr/response/TestExportWriter.java index 5a303e9893d..c55678702b1 100644 --- a/solr/core/src/test/org/apache/solr/response/TestExportWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestExportWriter.java @@ -16,17 +16,40 @@ */ package org.apache.solr.response; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.util.Utils; -import org.junit.*; -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import org.apache.lucene.index.LeafReader; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.util.SuppressForbidden; +import org.apache.solr.common.util.Utils; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.RefCounted; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; -@SuppressCodecs({"Lucene3x", "Lucene40","Lucene41","Lucene42","Lucene45"}) public class TestExportWriter extends SolrTestCaseJ4 { + @BeforeClass public static void beforeClass() throws Exception { System.setProperty("export.test", "true"); initCore("solrconfig-sortingresponse.xml","schema-sortingresponse.xml"); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + assertU(delQ("*:*")); + assertU(commit()); createIndex(); } @@ -47,7 +70,12 @@ public class TestExportWriter extends SolrTestCaseJ4 { "longdv_m", "343332", "stringdv_m", "manchester \"city\"", "stringdv_m", "liverpool", - "stringdv_m", "Everton")); + "stringdv_m", "Everton", + "datedv", "2017-06-16T07:00:00Z", + "datedv_m", "2017-06-16T01:00:00Z", + "datedv_m", "2017-06-16T02:00:00Z", + "datedv_m", "2017-06-16T03:00:00Z", + "datedv_m", "2017-06-16T04:00:00Z")); assertU(adoc("id","7", "floatdv","2.1", @@ -80,7 +108,11 @@ public class TestExportWriter extends SolrTestCaseJ4 { "longdv_m", "343332", "stringdv_m", "manchester \"city\"", "stringdv_m", "liverpool", - "stringdv_m", "everton")); + "stringdv_m", "everton", + "int_is_t", "1", + "int_is_t", "1", + "int_is_t", "1", + "int_is_t", "1")); assertU(commit()); assertU(adoc("id","8", "floatdv","2.1", @@ -98,7 +130,14 @@ public class TestExportWriter extends SolrTestCaseJ4 { "longdv_m", "343332", "stringdv_m", "manchester \"city\"", "stringdv_m", "liverpool", - "stringdv_m", "everton")); + "stringdv_m", "everton", + "datedv", "2017-01-01T00:00:00Z", + "datedv_m", "2017-01-01T01:00:00Z", + "datedv_m", "2017-01-01T02:00:00Z", + "int_is_p", "1", + "int_is_p", "1", + "int_is_p", "1", + "int_is_p", "1")); assertU(commit()); @@ -192,4 +231,152 @@ public class TestExportWriter extends SolrTestCaseJ4 { // Interesting you don't even need to specify a "q" parameter. } + + @Test + public void testDates() throws Exception { + String s = h.query(req("q", "id:1", "qt", "/export", "fl", "datedv", "sort", "datedv asc")); + assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"datedv\":\"2017-06-16T07:00:00Z\"}]}}"); + s = h.query(req("q", "id:1", "qt", "/export", "fl", "datedv_m", "sort", "datedv asc")); + assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"datedv_m\":[\"2017-06-16T01:00:00Z\",\"2017-06-16T02:00:00Z\",\"2017-06-16T03:00:00Z\",\"2017-06-16T04:00:00Z\"]}]}}"); + } + + @Test + public void testDuplicates() throws Exception { + RefCounted ref = null; + try { + ref = h.getCore().getSearcher(); + LeafReader reader = ref.get().getSlowAtomicReader(); + // MultiValued Trie fields use SortedSet + assertNotNull(reader.getSortedSetDocValues("int_is_t")); + assertNull(reader.getSortedNumericDocValues("int_is_t")); + // MultiValued Point fields use SortedNumerics + assertNull(reader.getSortedSetDocValues("int_is_p")); + assertNotNull(reader.getSortedNumericDocValues("int_is_p")); + } finally { + if (ref != null) ref.decref(); + } + String s = h.query(req("q", "id:3", "qt", "/export", "fl", "int_is_t", "sort", "intdv asc")); + assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"int_is_t\":[1]}]}}"); + s = h.query(req("q", "id:8", "qt", "/export", "fl", "int_is_p", "sort", "intdv asc")); + assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"int_is_p\":[1,1,1,1]}]}}"); + } + + /** + * This test doesn't validate the correctness of results, it just compares the response of the same request + * when asking for Trie fields vs Point fields. Can be removed once Trie fields are no longer supported + */ + @Test + @SuppressForbidden(reason="using new Date(time) to create random dates") + public void testRandomNumerics() throws Exception { + assertU(delQ("*:*")); + assertU(commit()); + List trieFields = new ArrayList(); + List pointFields = new ArrayList(); + for (String mv:new String[]{"s", ""}) { + for (String indexed:new String[]{"_ni", ""}) { + for (String type:new String[]{"i", "l", "f", "d", "dt"}) { + String field = "number_" + type + mv + indexed; + SchemaField sf = h.getCore().getLatestSchema().getField(field + "_t"); + assertTrue(sf.hasDocValues()); + assertTrue(sf.getType().getNumberType() != null); + assertFalse(sf.getType().isPointField()); + + sf = h.getCore().getLatestSchema().getField(field + "_p"); + assertTrue(sf.hasDocValues()); + assertTrue(sf.getType().getNumberType() != null); + assertTrue(sf.getType().isPointField()); + + trieFields.add(field + "_t"); + pointFields.add(field + "_p"); + } + } + } + for (int i = 0; i < atLeast(100); i++) { + if (random().nextInt(20) == 0) { + //have some empty docs + assertU(adoc("id", String.valueOf(i))); + continue; + } + + if (random().nextInt(20) == 0 && i > 0) { + //delete some docs + assertU(delI(String.valueOf(i - 1))); + } + + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", String.valueOf(i)); + addInt(doc, random().nextInt(), false); + addLong(doc, random().nextLong(), false); + addFloat(doc, random().nextFloat() * 3000 * (random().nextBoolean()?1:-1), false); + addDouble(doc, random().nextDouble() * 3000 * (random().nextBoolean()?1:-1), false); + addDate(doc, new Date(), false); + + // MV need to be unique in order to be the same in Trie vs Points + Set ints = new HashSet<>(); + Set longs = new HashSet<>(); + Set floats = new HashSet<>(); + Set doubles = new HashSet<>(); + Set dates = new HashSet<>(); + for (int j=0; j < random().nextInt(20); j++) { + ints.add(random().nextInt()); + longs.add(random().nextLong()); + floats.add(random().nextFloat() * 3000 * (random().nextBoolean()?1:-1)); + doubles.add(random().nextDouble() * 3000 * (random().nextBoolean()?1:-1)); + dates.add(new Date(System.currentTimeMillis() + random().nextInt())); + } + ints.stream().forEach((val)->addInt(doc, val, true)); + longs.stream().forEach((val)->addLong(doc, val, true)); + floats.stream().forEach((val)->addFloat(doc, val, true)); + doubles.stream().forEach((val)->addDouble(doc, val, true)); + dates.stream().forEach((val)->addDate(doc, val, true)); + + assertU(adoc(doc)); + if (random().nextInt(20) == 0) { + assertU(commit()); + } + } + assertU(commit()); + doTestQuery("id:1", trieFields, pointFields); + doTestQuery("*:*", trieFields, pointFields); + doTestQuery("id:[0 TO 2]", trieFields, pointFields);// "id" field is really a string, this is not a numeric range query + doTestQuery("id:[0 TO 9]", trieFields, pointFields); + doTestQuery("id:DOES_NOT_EXIST", trieFields, pointFields); + } + + private void doTestQuery(String query, List trieFields, List pointFields) throws Exception { + String trieFieldsFl = String.join(",", trieFields); + String pointFieldsFl = String.join(",", pointFields); + String sort = pickRandom((String)pickRandom(trieFields.toArray()), (String)pickRandom(pointFields.toArray())).replace("s_", "_") + pickRandom(" asc", " desc"); + String resultPoints = h.query(req("q", query, "qt", "/export", "fl", pointFieldsFl, "sort", sort)); + String resultTries = h.query(req("q", query, "qt", "/export", "fl", trieFieldsFl, "sort", sort)); + assertJsonEquals(resultPoints.replaceAll("_p", ""), resultTries.replaceAll("_t", "")); + } + + private void addFloat(SolrInputDocument doc, float value, boolean mv) { + addField(doc, "f", String.valueOf(value), mv); + } + + private void addDouble(SolrInputDocument doc, double value, boolean mv) { + addField(doc, "d", String.valueOf(value), mv); + } + + private void addLong(SolrInputDocument doc, long value, boolean mv) { + addField(doc, "l", String.valueOf(value), mv); + } + + private void addInt(SolrInputDocument doc, int value, boolean mv) { + addField(doc, "i", String.valueOf(value), mv); + } + + private void addDate(SolrInputDocument doc, Date value, boolean mv) { + addField(doc, "dt", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT).format(value), mv); + } + + private void addField(SolrInputDocument doc, String type, String value, boolean mv) { + doc.addField("number_" + type + (mv?"s":"") + "_t", value); + doc.addField("number_" + type + (mv?"s":"") + "_p", value); + doc.addField("number_" + type + (mv?"s":"") + "_ni_t", value); + doc.addField("number_" + type + (mv?"s":"") + "_ni_p", value); + } + } From 94220a01e14f53e0632bfbc1678661ad9c67320a Mon Sep 17 00:00:00 2001 From: Mike Drob Date: Mon, 12 Jun 2017 15:15:54 -0700 Subject: [PATCH 064/131] SOLR-8392: Remove instanceof checks on return value of SolrParam::get --- solr/CHANGES.txt | 2 ++ .../org/apache/solr/handler/SchemaHandler.java | 6 +----- .../org/apache/solr/rest/BaseSolrResource.java | 6 ++---- .../solr/common/params/MapSolrParams.java | 17 ++++------------- 4 files changed, 9 insertions(+), 22 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 9135cfa567b..51d2d5335fa 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -256,6 +256,8 @@ Other Changes * SOLR-10800: Factor out HttpShardHandler.transformReplicasToShardUrls from HttpShardHandler.prepDistributed. (Domenico Fabio Marino, Christine Poerschke) +* SOLR-8392: Remove instanceof checks on return value of SolrParam::get (Mike Drob, Varun Thacker) + ================== 6.7.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java index e3e292b6554..626f6557b8d 100644 --- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java @@ -138,11 +138,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware, break; } case "/schema/zkversion": { - int refreshIfBelowVersion = -1; - Object refreshParam = req.getParams().get("refreshIfBelowVersion"); - if (refreshParam != null) - refreshIfBelowVersion = (refreshParam instanceof Number) ? ((Number) refreshParam).intValue() - : Integer.parseInt(refreshParam.toString()); + int refreshIfBelowVersion = req.getParams().getInt("refreshIfBelowVersion", -1); int zkVersion = -1; IndexSchema schema = req.getSchema(); if (schema instanceof ManagedIndexSchema) { diff --git a/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java b/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java index 5a9310d84e9..cc1eeb2f936 100644 --- a/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java +++ b/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java @@ -125,11 +125,9 @@ public abstract class BaseSolrResource extends ServerResource { SolrCore.preDecorateResponse(solrRequest, solrResponse); // client application can set a timeout for update requests - Object updateTimeoutSecsParam = getSolrRequest().getParams().get(UPDATE_TIMEOUT_SECS); + String updateTimeoutSecsParam = getSolrRequest().getParams().get(UPDATE_TIMEOUT_SECS); if (updateTimeoutSecsParam != null) - updateTimeoutSecs = (updateTimeoutSecsParam instanceof Number) - ? ((Number) updateTimeoutSecsParam).intValue() - : Integer.parseInt(updateTimeoutSecsParam.toString()); + updateTimeoutSecs = Integer.parseInt(updateTimeoutSecsParam); } } diff --git a/solr/solrj/src/java/org/apache/solr/common/params/MapSolrParams.java b/solr/solrj/src/java/org/apache/solr/common/params/MapSolrParams.java index 5454fcac277..53eff87e1f2 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/MapSolrParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/MapSolrParams.java @@ -20,7 +20,7 @@ import java.util.Iterator; import java.util.Map; /** - * + * {@link SolrParams} implementation that can be built from and is backed by a {@link Map}. */ public class MapSolrParams extends SolrParams { protected final Map map; @@ -31,22 +31,13 @@ public class MapSolrParams extends SolrParams { @Override public String get(String name) { - Object o = map.get(name); - if(o == null) return null; - if (o instanceof String) return (String) o; - if (o instanceof String[]) { - String[] strings = (String[]) o; - if(strings.length == 0) return null; - return strings[0]; - } - return String.valueOf(o); + return map.get(name); } @Override public String[] getParams(String name) { - Object val = map.get(name); - if (val instanceof String[]) return (String[]) val; - return val==null ? null : new String[]{String.valueOf(val)}; + String val = map.get(name); + return val==null ? null : new String[]{val}; } @Override From 833a6f3ffbed729869dafeebe5a3c4aa14c113f8 Mon Sep 17 00:00:00 2001 From: Tomas Fernandez Lobbe Date: Mon, 12 Jun 2017 15:46:13 -0700 Subject: [PATCH 065/131] SOLR-10851: SolrClients should clarify expectations for solrServerUrl parameter --- solr/CHANGES.txt | 3 + solr/solr-ref-guide/src/using-solrj.adoc | 8 ++ .../client/solrj/impl/CloudSolrClient.java | 10 ++- .../impl/ConcurrentUpdateSolrClient.java | 18 ++++- .../client/solrj/impl/HttpSolrClient.java | 80 ++++++++++++++----- .../client/solrj/impl/LBHttpSolrClient.java | 38 ++++++++- 6 files changed, 133 insertions(+), 24 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 51d2d5335fa..153ed8207d6 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -406,6 +406,9 @@ Other Changes * SOLR-10761: Switch trie numeric/date fields to points in data-driven-enabled example and test schemas. (Steve Rowe) +* SOLR-10851: SolrClients should clarify expectations for solrServerUrl parameter (Jason Gerlowski + via Tomás Fernández Löbbe) + ================== 6.6.1 ================== Bug Fixes diff --git a/solr/solr-ref-guide/src/using-solrj.adoc b/solr/solr-ref-guide/src/using-solrj.adoc index fb92bd9453e..b619aedbaf7 100644 --- a/solr/solr-ref-guide/src/using-solrj.adoc +++ b/solr/solr-ref-guide/src/using-solrj.adoc @@ -69,6 +69,14 @@ You can sidestep a lot of the messing around with the JAR files by using Maven i If you are worried about the SolrJ libraries expanding the size of your client application, you can use a code obfuscator like http://proguard.sourceforge.net/[ProGuard] to remove APIs that you are not using. +[[UsingSolrJ-SpecifyingSolrUrl]] +== Specifying Solr Base URLs + +Most `SolrClient` implementations (with the notable exception of `CloudSolrClient`) require users to specify one or more Solr base URLs, which the client then uses to send HTTP requests to Solr. The path users include on the base URL they provide has an effect on the behavior of the created client from that point on. + +. A URL with a path pointing to a specific core or collection (e.g. `http://hostname:8983/solr/core1`). When a core or collection is specified in the base URL, subsequent requests made with that client are not required to re-specify the affected collection. However, the client is limited to sending requests to that core/collection, and can not send requests to any others. +. A URL with a generic path pointing to the root Solr path (e.g. `http://hostname:8983/solr`). When no core or collection is specified in the base URL, requests can be made to any core/collection, but the affected core/collection must be specified on all requests. + [[UsingSolrJ-SettingXMLResponseParser]] == Setting XMLResponseParser diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index 11f6b2644fe..10ea590f4a1 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -250,8 +250,8 @@ public class CloudSolrClient extends SolrClient { * chroot is required, use null. * @param solrUrls * A list of Solr URLs to configure the underlying {@link HttpClusterStateProvider}, which will - * use of the these URLs to fetch the list of live nodes for this Solr cluster. Provide only - * one of solrUrls or zkHosts. + * use of the these URLs to fetch the list of live nodes for this Solr cluster. URL's must point to the + * root Solr path ("/solr"). Provide only one of solrUrls or zkHosts. * @param httpClient * the {@link HttpClient} instance to be used for all requests. The provided httpClient should use a * multi-threaded connection manager. If null, a default HttpClient will be used. @@ -1403,6 +1403,9 @@ public class CloudSolrClient extends SolrClient { * * Method may be called multiple times. One of the provided values will be used to fetch * the list of live Solr nodes that the underlying {@link HttpClusterStateProvider} would be maintaining. + * + * Provided Solr URL is expected to point to the root Solr path ("http://hostname:8983/solr"); it should not + * include any collections, cores, or other path components. */ public Builder withSolrUrl(String solrUrl) { this.solrUrls.add(solrUrl); @@ -1413,6 +1416,9 @@ public class CloudSolrClient extends SolrClient { * Provide a list of Solr URL to be used when configuring {@link CloudSolrClient} instances. * One of the provided values will be used to fetch the list of live Solr * nodes that the underlying {@link HttpClusterStateProvider} would be maintaining. + * + * Provided Solr URLs are expected to point to the root Solr path ("http://hostname:8983/solr"); they should not + * include any collections, cores, or other path components. */ public Builder withSolrUrl(Collection solrUrls) { this.solrUrls.addAll(solrUrls); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java index d6675f284ea..f169a605217 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java @@ -753,7 +753,23 @@ public class ConcurrentUpdateSolrClient extends SolrClient { /** * Create a Builder object, based on the provided Solr URL. * - * @param baseSolrUrl the base URL of the Solr server that will be targeted by any created clients. + * Two different paths can be specified as a part of this URL: + * + * 1) A path pointing directly at a particular core + * 
    +     *   SolrClient client = new ConcurrentUpdateSolrClient.Builder("http://my-solr-server:8983/solr/core1").build();
+     *   QueryResponse resp = client.query(new SolrQuery("*:*"));
+     *
    + * Note that when a core is provided in the base URL, queries and other requests can be made without mentioning the + * core explicitly. However, the client can only send requests to that core. + * + * 2) The path of the root Solr path ("/solr") + * 
    +     *   SolrClient client = new ConcurrentUpdateSolrClient.Builder("http://my-solr-server:8983/solr").build();
+     *   QueryResponse resp = client.query("core1", new SolrQuery("*:*"));
+     *
    + * In this case the client is more flexible and can be used to send requests to any cores. This flexibility though + * requires that the core be specified on all requests. */ public Builder(String baseSolrUrl) { this.baseSolrUrl = baseSolrUrl; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java index c1e95763ea1..1ba08bfcae1 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java @@ -83,24 +83,6 @@ import org.slf4j.MDC; /** * A SolrClient implementation that talks directly to a Solr server via HTTP - * - * There are two ways to use an HttpSolrClient: - * - * 1) Pass a URL to the constructor that points directly at a particular core - * 
    - *   SolrClient client = new HttpSolrClient("http://my-solr-server:8983/solr/core1");
- *   QueryResponse resp = client.query(new SolrQuery("*:*"));
- *
    - * In this case, you can query the given core directly, but you cannot query any other - * cores or issue CoreAdmin requests with this client. - * - * 2) Pass the base URL of the node to the constructor - * 
    - *   SolrClient client = new HttpSolrClient("http://my-solr-server:8983/solr");
- *   QueryResponse resp = client.query("core1", new SolrQuery("*:*"));
- *
    - * In this case, you must pass the name of the required core for all queries and updates, - * but you may use the same client for all cores, and for CoreAdmin requests. */ public class HttpSolrClient extends SolrClient { @@ -646,6 +628,27 @@ public class HttpSolrClient extends SolrClient { return baseUrl; } + /** + * Change the base-url used when sending requests to Solr. + * + * Two different paths can be specified as a part of this URL: + * + * 1) A path pointing directly at a particular core + * 
    +   *   httpSolrClient.setBaseURL("http://my-solr-server:8983/solr/core1");
+   *   QueryResponse resp = httpSolrClient.query(new SolrQuery("*:*"));
+   *
    + * Note that when a core is provided in the base URL, queries and other requests can be made without mentioning the + * core explicitly. However, the client can only send requests to that core. + * + * 2) The path of the root Solr path ("/solr") + * 
    +   *   httpSolrClient.setBaseURL("http://my-solr-server:8983/solr");
+   *   QueryResponse resp = httpSolrClient.query("core1", new SolrQuery("*:*"));
+   *
    + * In this case the client is more flexible and can be used to send requests to any cores. The cost of this is that + * the core must be specified on each request. + */ public void setBaseURL(String baseURL) { this.baseUrl = baseURL; } @@ -762,6 +765,27 @@ public class HttpSolrClient extends SolrClient { this.responseParser = new BinaryResponseParser(); } + /** + * Specify the base-url for the created client to use when sending requests to Solr. + * + * Two different paths can be specified as a part of this URL: + * + * 1) A path pointing directly at a particular core + * 
    +     *   SolrClient client = builder.withBaseSolrUrl("http://my-solr-server:8983/solr/core1").build();
+     *   QueryResponse resp = client.query(new SolrQuery("*:*"));
+     *
    + * Note that when a core is provided in the base URL, queries and other requests can be made without mentioning the + * core explicitly. However, the client can only send requests to that core. + * + * 2) The path of the root Solr path ("/solr") + * 
    +     *   SolrClient client = builder.withBaseSolrUrl("http://my-solr-server:8983/solr").build();
+     *   QueryResponse resp = client.query("core1", new SolrQuery("*:*"));
+     *
    + * In this case the client is more flexible and can be used to send requests to any cores. This flexibility though + * requires that the core is specified on all requests. + */ public Builder withBaseSolrUrl(String baseSolrUrl) { this.baseSolrUrl = baseSolrUrl; return this; @@ -770,9 +794,25 @@ public class HttpSolrClient extends SolrClient { /** * Create a Builder object, based on the provided Solr URL. * - * By default, compression is not enabled in created HttpSolrClient objects. + * Two different paths can be specified as a part of this URL: * - * @param baseSolrUrl the base URL of the Solr server that will be targeted by any created clients. + * 1) A path pointing directly at a particular core + * 
    +     *   SolrClient client = new HttpSolrClient.Builder("http://my-solr-server:8983/solr/core1").build();
+     *   QueryResponse resp = client.query(new SolrQuery("*:*"));
+     *
    + * Note that when a core is provided in the base URL, queries and other requests can be made without mentioning the + * core explicitly. However, the client can only send requests to that core. + * + * 2) The path of the root Solr path ("/solr") + * 
    +     *   SolrClient client = new HttpSolrClient.Builder("http://my-solr-server:8983/solr").build();
+     *   QueryResponse resp = client.query("core1", new SolrQuery("*:*"));
+     *
    + * In this case the client is more flexible and can be used to send requests to any cores. This flexibility though + * requires that the core be specified on all requests. + * + * By default, compression is not enabled on created HttpSolrClient objects. */ public Builder(String baseSolrUrl) { this.baseSolrUrl = baseSolrUrl; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java index 7706bf6c930..21816bb04a9 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java @@ -867,16 +867,52 @@ public class LBHttpSolrClient extends SolrClient { * Provide a Solr endpoint to be used when configuring {@link LBHttpSolrClient} instances. * * Method may be called multiple times. All provided values will be used. + * + * Two different paths can be specified as a part of the URL: + * + * 1) A path pointing directly at a particular core + * 
    +     *   SolrClient client = builder.withBaseSolrUrl("http://my-solr-server:8983/solr/core1").build();
+     *   QueryResponse resp = client.query(new SolrQuery("*:*"));
+     *
    + * Note that when a core is provided in the base URL, queries and other requests can be made without mentioning the + * core explicitly. However, the client can only send requests to that core. + * + * 2) The path of the root Solr path ("/solr") + * 
    +     *   SolrClient client = builder.withBaseSolrUrl("http://my-solr-server:8983/solr").build();
+     *   QueryResponse resp = client.query("core1", new SolrQuery("*:*"));
+     *
    + * In this case the client is more flexible and can be used to send requests to any cores. This flexibility though + * requires that the core is specified on all requests. */ public Builder withBaseSolrUrl(String baseSolrUrl) { this.baseSolrUrls.add(baseSolrUrl); return this; } - + /** * Provide Solr endpoints to be used when configuring {@link LBHttpSolrClient} instances. * * Method may be called multiple times. All provided values will be used. + * + * Two different paths can be specified as a part of each URL: + * + * 1) A path pointing directly at a particular core + * 
    +     *   SolrClient client = builder.withBaseSolrUrls("http://my-solr-server:8983/solr/core1").build();
+     *   QueryResponse resp = client.query(new SolrQuery("*:*"));
+     *
    + * Note that when a core is provided in the base URL, queries and other requests can be made without mentioning the + * core explicitly. However, the client can only send requests to that core. + * + * 2) The path of the root Solr path ("/solr") + * 
    +     *   SolrClient client = builder.withBaseSolrUrls("http://my-solr-server:8983/solr").build();
+     *   QueryResponse resp = client.query("core1", new SolrQuery("*:*"));
+     *
    + * In this case the client is more flexible and can be used to send requests to any cores. This flexibility though + * requires that the core is specified on all requests. */ public Builder withBaseSolrUrls(String... solrUrls) { for (String baseSolrUrl : solrUrls) { From 6d97305c635683afab1d1bbd07024f1c8f173bc0 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Tue, 13 Jun 2017 19:23:38 +0930 Subject: [PATCH 066/131] randomize v1 and v2 --- .../core/src/test/org/apache/solr/handler/TestReqParamsAPI.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java b/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java index de4a27afcca..db04486a5f5 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java @@ -50,7 +50,7 @@ public class TestReqParamsAPI extends SolrCloudTestCase { private void setupHarnesses() { for (final JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) { RestTestHarness harness = new RestTestHarness(() -> jettySolrRunner.getBaseUrl().toString() + "/" + COLL_NAME); - if (true) { + if (random().nextBoolean()) { harness.setServerProvider(() -> jettySolrRunner.getBaseUrl().toString() + "/____v2/c/" + COLL_NAME); } restTestHarnesses.add(harness); From b73dffd928af2a34afdf1554a62d417e024b593c Mon Sep 17 00:00:00 2001 From: Cao Manh Dat Date: Tue, 13 Jun 2017 19:38:43 +0700 Subject: [PATCH 067/131] SOLR-10818: Adding test for v2 backup-collection request --- .../org/apache/solr/handler/V2ApiIntegrationTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java index 66f5dbe7a57..881e68b6cbd 100644 --- a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java @@ -18,6 +18,7 @@ package org.apache.solr.handler; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -37,8 +38,6 @@ import org.junit.BeforeClass; import org.junit.Test; public class V2ApiIntegrationTest extends SolrCloudTestCase { - private List restTestHarnesses = new ArrayList<>(); - private static String COLL_NAME = "collection1"; @BeforeClass @@ -99,6 +98,11 @@ public class V2ApiIntegrationTest extends SolrCloudTestCase { assertEquals("/c/collection1/get", Utils.getObjectByPath(result, true, "/spec[0]/url/paths[0]")); result = resAsMap(client, new V2Request.Builder("/collections/"+COLL_NAME+"/get/_introspect").build()); assertEquals("/collections/collection1/get", Utils.getObjectByPath(result, true, "/spec[0]/url/paths[0]")); + String tempDir = createTempDir().toFile().getPath(); + client.request(new V2Request.Builder("/c") + .withMethod(SolrRequest.METHOD.POST) + .withPayload("{backup-collection:{name: backup_test, collection: "+COLL_NAME+" , location: '"+tempDir+"' }}") + .build()); } private Map resAsMap(CloudSolrClient client, V2Request request) throws SolrServerException, IOException { From 232eff0893bccb93d01042f26a00e50870be2f29 Mon Sep 17 00:00:00 2001 From: Andrzej Bialecki Date: Tue, 13 Jun 2017 17:36:51 +0200 Subject: [PATCH 068/131] SOLR-10704 Wait until all leader replicas are recovered before deleting the originals. --- solr/CHANGES.txt | 3 + .../org/apache/solr/cloud/ReplaceNodeCmd.java | 96 ++++++++++++++++++- .../apache/solr/cloud/ReplaceNodeTest.java | 9 +- 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 153ed8207d6..d2a26c09f88 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -364,6 +364,9 @@ Bug Fixes * SOLR-10835: Add support for point fields in Export Handler (Tomás Fernández Löbbe) +* SOLR-10704: REPLACENODE may cause data loss when replicationFactor is 1. (ab, shalin) + + Optimizations ---------------------- * SOLR-10634: JSON Facet API: When a field/terms facet will retrieve all buckets (i.e. limit:-1) diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java index e4240be0188..5adbe8c9577 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java @@ -20,15 +20,18 @@ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.CollectionStateWatcher; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; @@ -60,6 +63,7 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { String source = message.getStr("source"); String target = message.getStr("target"); String async = message.getStr("async"); + int timeout = message.getInt("timeout", 10 * 60); // 10 minutes boolean parallel = message.getBool("parallel", false); ClusterState clusterState = zkStateReader.getClusterState(); @@ -70,13 +74,34 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Target Node: " + target + " is not live"); } List sourceReplicas = getReplicasOfNode(source, clusterState); - + // how many leaders are we moving? for these replicas we have to make sure that either: + // * another existing replica can become a leader, or + // * we wait until the newly created replica completes recovery (and can become the new leader) + int numLeaders = 0; + for (ZkNodeProps props : sourceReplicas) { + if (props.getBool(ZkStateReader.LEADER_PROP, false)) { + numLeaders++; + } + } + // map of collectionName_coreNodeName to watchers + Map watchers = new HashMap<>(); List createdReplicas = new ArrayList<>(); AtomicBoolean anyOneFailed = new AtomicBoolean(false); CountDownLatch countDownLatch = new CountDownLatch(sourceReplicas.size()); + CountDownLatch replicasToRecover = new CountDownLatch(numLeaders); + for (ZkNodeProps sourceReplica : sourceReplicas) { + if (sourceReplica.getBool(ZkStateReader.LEADER_PROP, false)) { + String shardName = sourceReplica.getStr(SHARD_ID_PROP); + String replicaName = sourceReplica.getStr(ZkStateReader.REPLICA_PROP); + String collectionName = sourceReplica.getStr(COLLECTION_PROP); + String key = collectionName + "_" + replicaName; + RecoveryWatcher watcher = new RecoveryWatcher(collectionName, shardName, replicaName, replicasToRecover); + watchers.put(key, watcher); + zkStateReader.registerCollectionStateWatcher(collectionName, watcher); + } NamedList nl = new NamedList(); log.info("Going to create replica for collection={} shard={} on node={}", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target); ZkNodeProps msg = sourceReplica.plus("parallel", String.valueOf(parallel)).plus(CoreAdminParams.NODE, target); @@ -106,10 +131,26 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { } } - log.debug("Waiting for replace node action to complete"); - countDownLatch.await(5, TimeUnit.MINUTES); - log.debug("Finished waiting for replace node action to complete"); + log.debug("Waiting for replicas to be added"); + if (!countDownLatch.await(timeout, TimeUnit.SECONDS)) { + log.info("Timed out waiting for replicas to be added"); + anyOneFailed.set(true); + } else { + log.debug("Finished waiting for replicas to be added"); + } + // now wait for leader replicas to recover + log.debug("Waiting for " + numLeaders + " leader replicas to recover"); + if (!replicasToRecover.await(timeout, TimeUnit.SECONDS)) { + log.info("Timed out waiting for " + replicasToRecover.getCount() + " leader replicas to recover"); + anyOneFailed.set(true); + } else { + log.debug("Finished waiting for leader replicas to recover"); + } + // remove the watchers, we're done either way + for (RecoveryWatcher watcher : watchers.values()) { + zkStateReader.removeCollectionStateWatcher(watcher.collectionId, watcher); + } if (anyOneFailed.get()) { log.info("Failed to create some replicas. Cleaning up all replicas on target node"); CountDownLatch cleanupLatch = new CountDownLatch(createdReplicas.size()); @@ -134,6 +175,7 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { } } cleanupLatch.await(5, TimeUnit.MINUTES); + return; } @@ -155,6 +197,7 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { ZkStateReader.CORE_NAME_PROP, replica.getCoreName(), ZkStateReader.REPLICA_PROP, replica.getName(), ZkStateReader.REPLICA_TYPE, replica.getType().name(), + ZkStateReader.LEADER_PROP, String.valueOf(replica.equals(slice.getLeader())), CoreAdminParams.NODE, source); sourceReplicas.add(props); } @@ -163,4 +206,49 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { } return sourceReplicas; } + + // we use this watcher to wait for replicas to recover + private static class RecoveryWatcher implements CollectionStateWatcher { + String collectionId; + String shardId; + String replicaId; + CountDownLatch countDownLatch; + + RecoveryWatcher(String collectionId, String shardId, String replicaId, CountDownLatch countDownLatch) { + this.collectionId = collectionId; + this.shardId = shardId; + this.replicaId = replicaId; + this.countDownLatch = countDownLatch; + } + + @Override + public boolean onStateChanged(Set liveNodes, DocCollection collectionState) { + if (collectionState == null) { // collection has been deleted - don't wait + countDownLatch.countDown(); + return true; + } + Slice slice = collectionState.getSlice(shardId); + if (slice == null) { // shard has been removed - don't wait + countDownLatch.countDown(); + return true; + } + for (Replica replica : slice.getReplicas()) { + // check if another replica exists - doesn't have to be the one we're moving + // as long as it's active and can become a leader, in which case we don't have to wait + // for recovery of specifically the one that we've just added + if (!replica.getName().equals(replicaId)) { + if (replica.getType().equals(Replica.Type.PULL)) { // not eligible for leader election + continue; + } + // check its state + if (replica.isActive(liveNodes)) { // recovered - stop waiting + countDownLatch.countDown(); + return true; + } + } + } + // set the watch again to wait for the new replica to recover + return false; + } + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java index ecfd3eedc21..d7fae927a96 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java @@ -67,11 +67,16 @@ public class ReplaceNodeTest extends SolrCloudTestCase { CollectionAdminRequest.Create create; // NOTE: always using the createCollection that takes in 'int' for all types of replicas, so we never // have to worry about null checking when comparing the Create command with the final Slices - create = pickRandom(CollectionAdminRequest.createCollection(coll, "conf1", 5, 2,0,0), + create = pickRandom( + CollectionAdminRequest.createCollection(coll, "conf1", 5, 2,0,0), CollectionAdminRequest.createCollection(coll, "conf1", 5, 1,1,0), CollectionAdminRequest.createCollection(coll, "conf1", 5, 0,1,1), CollectionAdminRequest.createCollection(coll, "conf1", 5, 1,0,1), - CollectionAdminRequest.createCollection(coll, "conf1", 5, 0,2,0)); + CollectionAdminRequest.createCollection(coll, "conf1", 5, 0,2,0), + // check also replicationFactor 1 + CollectionAdminRequest.createCollection(coll, "conf1", 5, 1,0,0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 0,1,0) + ); create.setCreateNodeSet(StrUtils.join(l, ',')).setMaxShardsPerNode(3); cloudClient.request(create); log.info("excluded_node : {} ", emptyNode); From 78fcbc0623cb0a602f406e7af844437ae98f729c Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Tue, 13 Jun 2017 12:01:13 -0400 Subject: [PATCH 069/131] ref guide: clean up monospace syntax problems --- solr/solr-ref-guide/src/filter-descriptions.adoc | 4 ++-- solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc | 2 +- solr/solr-ref-guide/src/learning-to-rank.adoc | 2 +- .../src/major-changes-from-solr-5-to-solr-6.adoc | 4 ++-- solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc | 4 ++-- solr/solr-ref-guide/src/running-solr-on-hdfs.adoc | 4 ++-- solr/solr-ref-guide/src/solr-control-script-reference.adoc | 2 +- solr/solr-ref-guide/src/taking-solr-to-production.adoc | 2 +- solr/solr-ref-guide/src/the-stats-component.adoc | 2 +- solr/solr-ref-guide/src/upgrading-solr.adoc | 2 +- solr/solr-ref-guide/src/using-solrj.adoc | 2 +- solr/solr-ref-guide/src/velocity-response-writer.adoc | 2 +- solr/solr-ref-guide/src/working-with-dates.adoc | 2 +- solr/solr-ref-guide/src/zookeeper-access-control.adoc | 4 ++-- 14 files changed, 19 insertions(+), 19 deletions(-) diff --git a/solr/solr-ref-guide/src/filter-descriptions.adoc b/solr/solr-ref-guide/src/filter-descriptions.adoc index 09dbe23fc6d..9458297d99d 100644 --- a/solr/solr-ref-guide/src/filter-descriptions.adoc +++ b/solr/solr-ref-guide/src/filter-descriptions.adoc @@ -1419,9 +1419,9 @@ Case-sensitive matching, capitalized words not stopped. Token positions skip sto Like <>, this filter discards, or _stops_ analysis of, tokens that are on the given stop words list. -Suggest Stop Filter differs from Stop Filter in that it will not remove the last token unless it is followed by a token separator. For example, a query "`find the`" would preserve the '`the`' since it was not followed by a space, punctuation etc., and mark it as a `KEYWORD` so that following filters will not change or remove it. +Suggest Stop Filter differs from Stop Filter in that it will not remove the last token unless it is followed by a token separator. For example, a query `"find the"` would preserve the `'the'` since it was not followed by a space, punctuation etc., and mark it as a `KEYWORD` so that following filters will not change or remove it. -By contrast, a query like "`find the popsicle`" would remove "```the```" as a stopword, since it's followed by a space. When using one of the analyzing suggesters, you would normally use the ordinary `StopFilterFactory` in your index analyzer and then SuggestStopFilter in your query analyzer. +By contrast, a query like "`find the popsicle`" would remove '`the`' as a stopword, since it's followed by a space. When using one of the analyzing suggesters, you would normally use the ordinary `StopFilterFactory` in your index analyzer and then SuggestStopFilter in your query analyzer. *Factory class:* `solr.SuggestStopFilterFactory` diff --git a/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc b/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc index cfc1a78b5c7..ce3250355d2 100644 --- a/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc @@ -115,7 +115,7 @@ If the configuration options for the built-in merge policies do not fully suit y ---- -The example above shows Solr's {solr-javadocs}/solr-core/org/apache/solr/index/SortingMergePolicyFactory.html[`SortingMergePolicyFactory`] being configured to sort documents in merged segments by `"timestamp desc"`, and wrapped around a `TieredMergePolicyFactory` configured to use the values `maxMergeAtOnce=10` and `segmentsPerTier=10` via the `inner` prefix defined by `SortingMergePolicyFactory`'s `wrapped.prefix` option. For more information on using `SortingMergePolicyFactory`, see <>. +The example above shows Solr's {solr-javadocs}/solr-core/org/apache/solr/index/SortingMergePolicyFactory.html[`SortingMergePolicyFactory`] being configured to sort documents in merged segments by `"timestamp desc"`, and wrapped around a `TieredMergePolicyFactory` configured to use the values `maxMergeAtOnce=10` and `segmentsPerTier=10` via the `inner` prefix defined by `SortingMergePolicyFactory` 's `wrapped.prefix` option. For more information on using `SortingMergePolicyFactory`, see <>. [[IndexConfiginSolrConfig-mergeScheduler]] === mergeScheduler diff --git a/solr/solr-ref-guide/src/learning-to-rank.adoc b/solr/solr-ref-guide/src/learning-to-rank.adoc index 3eb077eaaeb..64a461bd6e4 100644 --- a/solr/solr-ref-guide/src/learning-to-rank.adoc +++ b/solr/solr-ref-guide/src/learning-to-rank.adoc @@ -509,7 +509,7 @@ To inspect the content of the `commonFeatureStore` feature store: * A model need not use all the features defined in a feature store. * Multiple models can use the same feature store. -To extract features for `currentFeatureStore`'s features: +To extract features for `currentFeatureStore` 's features: `\http://localhost:8983/solr/techproducts/query?q=test&fl=id,score,[features store=currentFeatureStore]` diff --git a/solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adoc b/solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adoc index 9b58cef26c0..6810e4bd3a6 100644 --- a/solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adoc +++ b/solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adoc @@ -76,9 +76,9 @@ Please review the <`>> is now dependent on the `luceneMatchVersion` specified in the `solrconfig.xml`. When `luceneMatchVersion < 6.0`, an instance of `ClassicSimilarityFactory` will be used, otherwise an instance of `SchemaSimlarityFactory` will be used. Most notably this change means that users can take advantage of per Field Type similarity declarations, with out needing to also explicitly declare a global usage of `SchemaSimlarityFactory`. +Solr's default behavior when a Schema does not explicitly define a global <`>> is now dependent on the `luceneMatchVersion` specified in the `solrconfig.xml`. When `luceneMatchVersion < 6.0`, an instance of `ClassicSimilarityFactory` will be used, otherwise an instance of `SchemaSimilarityFactory` will be used. Most notably this change means that users can take advantage of per Field Type similarity declarations, with out needing to also explicitly declare a global usage of `SchemaSimilarityFactory`. -Regardless of whether it is explicitly declared, or used as an implicit global default, `SchemaSimlarityFactory`'s implicit behavior when a Field Types do not declare an explicit `` has also been changed to depend on the the `luceneMatchVersion`. When `luceneMatchVersion < 6.0`, an instance of `ClassicSimilarity` will be used, otherwise an instance of `BM25Simlarity` will be used. A `defaultSimFromFieldType` init option may be specified on the `SchemaSimilarityFactory` declaration to change this behavior. Please review the `SchemaSimlarityFactory` javadocs for more details +Regardless of whether it is explicitly declared, or used as an implicit global default, `SchemaSimilarityFactory` 's implicit behavior when a Field Types do not declare an explicit `` has also been changed to depend on the the `luceneMatchVersion`. When `luceneMatchVersion < 6.0`, an instance of `ClassicSimilarity` will be used, otherwise an instance of `BM25Similarity` will be used. A `defaultSimFromFieldType` init option may be specified on the `SchemaSimilarityFactory` declaration to change this behavior. Please review the `SchemaSimilarityFactory` javadocs for more details == Replica & Shard Delete Command Changes diff --git a/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc b/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc index 76a6c4f0c51..df377819de5 100644 --- a/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc +++ b/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc @@ -200,7 +200,7 @@ curl --user solr:SolrRocks -H 'Content-type:application/json' -d '{ Permissions can be accessed using their index in the list. Use the `/admin/authorization` API to see the existing permissions and their indices. -The following example updates the '`role`' attribute of permission at index '`3`': +The following example updates the `'role'` attribute of permission at index `3`: [source,bash] ---- @@ -210,7 +210,7 @@ curl --user solr:SolrRocks -H 'Content-type:application/json' -d '{ }' http://localhost:8983/solr/admin/authorization ---- -The following example deletes permission at index '`3`': +The following example deletes permission at index `3`: [source,bash] ---- diff --git a/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc b/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc index b3e12a5e1f0..e23431cdfb9 100644 --- a/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc +++ b/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc @@ -38,7 +38,7 @@ For standalone Solr instances, there are a few parameters you should be sure to * You need to use an `HdfsDirectoryFactory` and a data dir of the form `hdfs://host:port/path` * You need to specify an UpdateLog location of the form `hdfs://host:port/path` -* You should specify a lock factory type of '`hdfs`' or none. +* You should specify a lock factory type of `'hdfs'` or none. If you do not modify `solrconfig.xml`, you can instead start Solr on HDFS with the following command: @@ -58,7 +58,7 @@ This example will start Solr in standalone mode, using the defined JVM propertie In SolrCloud mode, it's best to leave the data and update log directories as the defaults Solr comes with and simply specify the `solr.hdfs.home`. All dynamically created collections will create the appropriate directories automatically under the `solr.hdfs.home` root directory. * Set `solr.hdfs.home` in the form `hdfs://host:port/path` -* You should specify a lock factory type of '`hdfs`' or none. +* You should specify a lock factory type of `'hdfs'` or none. [source,bash] ---- diff --git a/solr/solr-ref-guide/src/solr-control-script-reference.adoc b/solr/solr-ref-guide/src/solr-control-script-reference.adoc index 4c915e3fc68..669c30a3211 100644 --- a/solr/solr-ref-guide/src/solr-control-script-reference.adoc +++ b/solr/solr-ref-guide/src/solr-control-script-reference.adoc @@ -605,7 +605,7 @@ Use the `zk cp` command for transferring files and directories between ZooKeeper |=== |Parameter |Description |Example |-r |Optional. Do a recursive copy. The command will fail if the has children unless '-r' is specified. |`-r` -| |The file or path to copy from. If prefixed with `zk:` then the source is presumed to be ZooKeeper. If no prefix or the prefix is 'file:' this is the local drive. At least one of or must be prefixed by '`zk:`' or the command will fail. a| +| |The file or path to copy from. If prefixed with `zk:` then the source is presumed to be ZooKeeper. If no prefix or the prefix is 'file:' this is the local drive. At least one of or must be prefixed by `'zk:'` or the command will fail. a| `zk:/configs/myconfigs/solrconfig.xml` `file:/Users/apache/configs/src` diff --git a/solr/solr-ref-guide/src/taking-solr-to-production.adoc b/solr/solr-ref-guide/src/taking-solr-to-production.adoc index 799c494cae0..97634101529 100644 --- a/solr/solr-ref-guide/src/taking-solr-to-production.adoc +++ b/solr/solr-ref-guide/src/taking-solr-to-production.adoc @@ -226,7 +226,7 @@ bin/solr zk mkroot /solr -z : [NOTE] ==== -If you also want to bootstrap ZooKeeper with existing `solr_home`, you can instead use use `zkcli.sh` / `zkcli.bat`'s `bootstrap` command, which will also create the chroot path if it does not exist. See <> for more info. +If you also want to bootstrap ZooKeeper with existing `solr_home`, you can instead use the `zkcli.sh` / `zkcli.bat` `bootstrap` command, which will also create the chroot path if it does not exist. See <> for more info. ==== diff --git a/solr/solr-ref-guide/src/the-stats-component.adoc b/solr/solr-ref-guide/src/the-stats-component.adoc index 8014ccf5544..96ba88c1fe3 100644 --- a/solr/solr-ref-guide/src/the-stats-component.adoc +++ b/solr/solr-ref-guide/src/the-stats-component.adoc @@ -179,6 +179,6 @@ Here we compute some statistics for the price field. The min, max, mean, 90th, a [[TheStatsComponent-TheStatsComponentandFaceting]] == The Stats Component and Faceting -Sets of `stats.field` parameters can be referenced by '`tag`' when using Pivot Faceting to compute multiple statistics at every level (i.e.: field) in the tree of pivot constraints. +Sets of `stats.field` parameters can be referenced by `'tag'` when using Pivot Faceting to compute multiple statistics at every level (i.e.: field) in the tree of pivot constraints. For more information and a detailed example, please see <>. diff --git a/solr/solr-ref-guide/src/upgrading-solr.adoc b/solr/solr-ref-guide/src/upgrading-solr.adoc index 74142a9481d..e41b93b6bc6 100644 --- a/solr/solr-ref-guide/src/upgrading-solr.adoc +++ b/solr/solr-ref-guide/src/upgrading-solr.adoc @@ -63,7 +63,7 @@ If you are already using Solr 6.5, Solr 6.6 should not present any major problem * Several changes have been made regarding the "<>" used in Solr, in order to provide better default behavior for new users. There are 3 key impacts of these changes on existing users who upgrade: ** `DefaultSimilarityFactory` has been removed. If you currently have `DefaultSimilarityFactory` explicitly referenced in your `schema.xml`, edit your config to use the functionally identical `ClassicSimilarityFactory`. See https://issues.apache.org/jira/browse/SOLR-8239[SOLR-8239] for more details. ** The implicit default Similarity used when no `` is configured in `schema.xml` has been changed to `SchemaSimilarityFactory`. Users who wish to preserve back-compatible behavior should either explicitly configure `ClassicSimilarityFactory`, or ensure that the `luceneMatchVersion` for the collection is less then 6.0. See https://issues.apache.org/jira/browse/SOLR-8270[SOLR-8270] + http://SOLR-8271[SOLR-8271] for details. -** `SchemaSimilarityFactory` has been modified to use `BM25Similarity` as the default for `fieldTypes` that do not explicitly declare a Similarity. The legacy behavior of using `ClassicSimilarity` as the default will occur if the `luceneMatchVersion` for the collection is less then 6.0, or the '`defaultSimFromFieldType`' configuration option may be used to specify any default of your choosing. See https://issues.apache.org/jira/browse/SOLR-8261[SOLR-8261] + https://issues.apache.org/jira/browse/SOLR-8329[SOLR-8329] for more details. +** `SchemaSimilarityFactory` has been modified to use `BM25Similarity` as the default for `fieldTypes` that do not explicitly declare a Similarity. The legacy behavior of using `ClassicSimilarity` as the default will occur if the `luceneMatchVersion` for the collection is less then 6.0, or the `'defaultSimFromFieldType'` configuration option may be used to specify any default of your choosing. See https://issues.apache.org/jira/browse/SOLR-8261[SOLR-8261] + https://issues.apache.org/jira/browse/SOLR-8329[SOLR-8329] for more details. * If your `solrconfig.xml` file doesn't explicitly mention the `schemaFactory` to use then Solr will choose the `ManagedIndexSchemaFactory` by default. Previously it would have chosen `ClassicIndexSchemaFactory`. This means that the Schema APIs (`//schema`) are enabled and the schema is mutable. When Solr starts your `schema.xml` file will be renamed to `managed-schema`. If you want to retain the old behaviour then please ensure that the `solrconfig.xml` explicitly uses the `ClassicIndexSchemaFactory` or your `luceneMatchVersion` in the `solrconfig.xml` is less than 6.0. See the <> section for more details * `SolrIndexSearcher.QueryCommand` and `QueryResult` were moved to their own classes. If you reference them in your code, you should import them under o.a.s.search (or use your IDE's "Organize Imports"). * The '<>' attribute specified in request handler cannot be overridden from request params. See https://issues.apache.org/jira/browse/SOLR-8698[SOLR-8698] for more details. diff --git a/solr/solr-ref-guide/src/using-solrj.adoc b/solr/solr-ref-guide/src/using-solrj.adoc index b619aedbaf7..4788ea2213f 100644 --- a/solr/solr-ref-guide/src/using-solrj.adoc +++ b/solr/solr-ref-guide/src/using-solrj.adoc @@ -135,7 +135,7 @@ SolrDocumentList list = response.getResults(); [[UsingSolrJ-IndexingDocuments]] == Indexing Documents -Other operations are just as simple. To index (add) a document, all you need to do is create a `SolrInputDocument` and pass it along to the `SolrClient`'s `add()` method. This example assumes that the SolrClient object called 'solr' is already created based on the examples shown earlier. +Other operations are just as simple. To index (add) a document, all you need to do is create a `SolrInputDocument` and pass it along to the `SolrClient` 's `add()` method. This example assumes that the SolrClient object called 'solr' is already created based on the examples shown earlier. [source,java] ---- diff --git a/solr/solr-ref-guide/src/velocity-response-writer.adoc b/solr/solr-ref-guide/src/velocity-response-writer.adoc index fcfe5603d4c..b9bc3bdce5d 100644 --- a/solr/solr-ref-guide/src/velocity-response-writer.adoc +++ b/solr/solr-ref-guide/src/velocity-response-writer.adoc @@ -57,7 +57,7 @@ The "params" resource loader allows templates to be specified in Solr request pa `\http://localhost:8983/solr/gettingstarted/select?q=\*:*&wt=velocity&v.template=custom&v.template.custom=CUSTOM%3A%20%23core_name` -where `v.template=custom` says to render a template called "custom" and `v.template.custom`'s value is the actual custom template. This is disabled by default; it'd be a niche, unusual, use case to need this enabled. +where `v.template=custom` says to render a template called "custom" and `v.template.custom` 's value is the actual custom template. This is disabled by default; it'd be a niche, unusual, use case to need this enabled. |false |solr.resource.loader.enabled |The "solr" resource loader is the only template loader registered by default. Templates are served from resources visible to the SolrResourceLoader under a `velocity/` subdirectory. The VelocityResponseWriter itself has some built-in templates (in its JAR file, under velocity/) that are available automatically through this loader. These built-in templates can be overridden when the same template name is in conf/velocity/ or by using the `template.base.dir` option. |true diff --git a/solr/solr-ref-guide/src/working-with-dates.adoc b/solr/solr-ref-guide/src/working-with-dates.adoc index 7d30b6605cd..31d0f1f6da8 100644 --- a/solr/solr-ref-guide/src/working-with-dates.adoc +++ b/solr/solr-ref-guide/src/working-with-dates.adoc @@ -43,7 +43,7 @@ You can optionally include fractional seconds if you wish, although any precisio * `1972-05-20T17:33:18.77Z` * `1972-05-20T17:33:18.7Z` -There must be a leading '`-`' for dates prior to year 0000, and Solr will format dates with a leading '`+`' for years after 9999. Year 0000 is considered year 1 BC; there is no such thing as year 0 AD or BC. +There must be a leading `'-'` for dates prior to year 0000, and Solr will format dates with a leading `'+'` for years after 9999. Year 0000 is considered year 1 BC; there is no such thing as year 0 AD or BC. .Query escaping may be required [WARNING] diff --git a/solr/solr-ref-guide/src/zookeeper-access-control.adoc b/solr/solr-ref-guide/src/zookeeper-access-control.adoc index d3d6b710e64..919ccb33ca0 100644 --- a/solr/solr-ref-guide/src/zookeeper-access-control.adoc +++ b/solr/solr-ref-guide/src/zookeeper-access-control.adoc @@ -58,7 +58,7 @@ Solr nodes, clients and tools (e.g. ZkCLI) always use a java class called {solr- [[ZooKeeperAccessControl-ControllingCredentials]] === Controlling Credentials -You control which credentials provider will be used by configuring the `zkCredentialsProvider` property in `solr.xml`'s `` section to the name of a class (on the classpath) implementing the {solr-javadocs}/solr-solrj/org/apache/solr/common/cloud/ZkCredentialsProvider[`ZkCredentialsProvider`] interface. `server/solr/solr.xml` in the Solr distribution defines the `zkCredentialsProvider` such that it will take on the value of the same-named `zkCredentialsProvider` system property if it is defined (e.g. by uncommenting the `SOLR_ZK_CREDS_AND_ACLS` environment variable definition in `solr.in.sh/.cmd` - see below), or if not, default to the `DefaultZkCredentialsProvider` implementation. +You control which credentials provider will be used by configuring the `zkCredentialsProvider` property in `solr.xml` 's `` section to the name of a class (on the classpath) implementing the {solr-javadocs}/solr-solrj/org/apache/solr/common/cloud/ZkCredentialsProvider[`ZkCredentialsProvider`] interface. `server/solr/solr.xml` in the Solr distribution defines the `zkCredentialsProvider` such that it will take on the value of the same-named `zkCredentialsProvider` system property if it is defined (e.g. by uncommenting the `SOLR_ZK_CREDS_AND_ACLS` environment variable definition in `solr.in.sh/.cmd` - see below), or if not, default to the `DefaultZkCredentialsProvider` implementation. ==== Out of the Box Credential Implementations @@ -72,7 +72,7 @@ You can always make you own implementation, but Solr comes with two implementati [[ZooKeeperAccessControl-ControllingACLs]] === Controlling ACLs -You control which ACLs will be added by configuring `zkACLProvider` property in `solr.xml`'s `` section to the name of a class (on the classpath) implementing the {solr-javadocs}//solr-solrj/org/apache/solr/common/cloud/ZkACLProvider[`ZkACLProvider`] interface. `server/solr/solr.xml` in the Solr distribution defines the `zkACLProvider` such that it will take on the value of the same-named `zkACLProvider` system property if it is defined (e.g. by uncommenting the `SOLR_ZK_CREDS_AND_ACLS` environment variable definition in `solr.in.sh/.cmd` - see below), or if not, default to the `DefaultZkACLProvider` implementation. +You control which ACLs will be added by configuring `zkACLProvider` property in `solr.xml` 's `` section to the name of a class (on the classpath) implementing the {solr-javadocs}//solr-solrj/org/apache/solr/common/cloud/ZkACLProvider[`ZkACLProvider`] interface. `server/solr/solr.xml` in the Solr distribution defines the `zkACLProvider` such that it will take on the value of the same-named `zkACLProvider` system property if it is defined (e.g. by uncommenting the `SOLR_ZK_CREDS_AND_ACLS` environment variable definition in `solr.in.sh/.cmd` - see below), or if not, default to the `DefaultZkACLProvider` implementation. [[ZooKeeperAccessControl-OutoftheBoxImplementations]] ==== Out of the Box ACL Implementations From cdccbfb92f30cc70a3fee4c1b3655b1aaf1acb7a Mon Sep 17 00:00:00 2001 From: Andrzej Bialecki Date: Tue, 13 Jun 2017 18:53:04 +0200 Subject: [PATCH 070/131] SOLR-10704 Update the ref guide. --- solr/solr-ref-guide/src/collections-api.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc index 33dcb946f3b..9e71365c9c5 100644 --- a/solr/solr-ref-guide/src/collections-api.adoc +++ b/solr/solr-ref-guide/src/collections-api.adoc @@ -1887,7 +1887,7 @@ Deletes all replicas of all collections in that node. Please note that the node [[CollectionsAPI-replacenode]] == REPLACENODE: Move All Replicas in a Node to Another -This command recreates replicas in the source node to the target node. After each replica is copied, the replicas in the source node are deleted. +This command recreates replicas in the source node to the target node. After each replica is copied, the replicas in the source node are deleted. For source replicas that are also shard leaders the operation will wait for "timeout" seconds to make sure there's an active replica that can become a leader (either an existing replica becoming a leader or the new replica completing the recovery and becoming a leader). `/admin/collections?action=REPLACENODE&source=_source-node_&target=_target-node_` @@ -1905,6 +1905,7 @@ This command recreates replicas in the source node to the target node. After eac |target |string |Yes |The target node |parallel |boolean |No |default=false. if this flag is set to true, all replicas are created inseparatee threads. Keep in mind that this can lead to very high network and disk I/O if the replicas have very large indices. |async |string |No |Request ID to track this action which will be <>. +|timeout |int |No |default is 300. Timeout in seconds to wait until new replicas are created, and until leader replicas are fully recovered. |=== [IMPORTANT] From e50332507a6ea3a9589b8919a803200ac144933e Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Tue, 13 Jun 2017 09:53:41 -0700 Subject: [PATCH 071/131] fix precommit by removing unused imports --- .../src/test/org/apache/solr/handler/V2ApiIntegrationTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java index 881e68b6cbd..fa6d4e37729 100644 --- a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java @@ -18,9 +18,7 @@ package org.apache.solr.handler; -import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -33,7 +31,6 @@ import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; -import org.apache.solr.util.RestTestHarness; import org.junit.BeforeClass; import org.junit.Test; From 6396cb759f8c799f381b0730636fa412761030ce Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Tue, 13 Jun 2017 10:17:32 -0700 Subject: [PATCH 072/131] SOLR-10830: Solr now correctly enforces that the '_root_' field has the same fieldType as the uniqueKey field --- solr/CHANGES.txt | 4 + .../apache/solr/schema/DoublePointField.java | 10 +-- .../apache/solr/schema/FloatPointField.java | 10 +-- .../org/apache/solr/schema/IndexSchema.java | 31 ++++++- .../org/apache/solr/schema/IntPointField.java | 10 +-- .../apache/solr/schema/LongPointField.java | 10 +-- .../apache/solr/schema/NumericFieldType.java | 88 ++++++++++++++++--- .../org/apache/solr/schema/PointField.java | 7 +- .../org/apache/solr/schema/TrieField.java | 60 ++++++------- .../apache/solr/update/AddUpdateCommand.java | 8 +- .../solr/update/DirectUpdateHandler2.java | 3 +- .../apache/solr/update/DocumentBuilder.java | 8 +- ...chema-uniquekey-diff-type-dynamic-root.xml | 36 ++++++++ .../bad-schema-uniquekey-diff-type-root.xml | 35 ++++++++ .../solr/collection1/conf/schema.xml | 16 +++- .../solr/collection1/conf/schema11.xml | 4 + .../solr/collection1/conf/schema12.xml | 23 ++++- .../configsets/cloud-hdfs/conf/schema.xml | 2 +- .../conf/managed-schema | 2 +- .../cloud-managed-upgrade/conf/schema.xml | 2 +- .../cloud-managed/conf/managed-schema | 2 +- .../cloud-minimal-jmx/conf/schema.xml | 2 +- .../configsets/cloud-minimal/conf/schema.xml | 2 +- .../configsets/cloud-subdirs/conf/schema.xml | 2 +- .../exitable-directory/conf/schema.xml | 2 +- .../apache/solr/BasicFunctionalityTest.java | 77 +++++++++++++++- .../solr/schema/BadIndexSchemaTest.java | 15 ++++ .../apache/solr/search/TestQueryTypes.java | 46 +++++++++- .../solr/search/TestSolrQueryParser.java | 36 ++++++++ 29 files changed, 467 insertions(+), 86 deletions(-) create mode 100644 solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-diff-type-dynamic-root.xml create mode 100644 solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-diff-type-root.xml diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d2a26c09f88..c1c7293f969 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -156,6 +156,10 @@ Bug Fixes * SOLR-10223: Allow running examples as root on Linux with -force option (janhoy) +* SOLR-10830: Solr now correctly enforces that the '_root_' field has the same fieldType as the + uniqueKey field. With out this enforcement, child document updating was unreliable. (hossman) + + Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/schema/DoublePointField.java b/solr/core/src/java/org/apache/solr/schema/DoublePointField.java index 05a1ce7d124..d2cf6ed36db 100644 --- a/solr/core/src/java/org/apache/solr/schema/DoublePointField.java +++ b/solr/core/src/java/org/apache/solr/schema/DoublePointField.java @@ -61,7 +61,7 @@ public class DoublePointField extends PointField implements DoubleValueFieldType if (min == null) { actualMin = Double.NEGATIVE_INFINITY; } else { - actualMin = Double.parseDouble(min); + actualMin = parseDoubleFromUser(field.getName(), min); if (!minInclusive) { actualMin = DoublePoint.nextUp(actualMin); } @@ -69,7 +69,7 @@ public class DoublePointField extends PointField implements DoubleValueFieldType if (max == null) { actualMax = Double.POSITIVE_INFINITY; } else { - actualMax = Double.parseDouble(max); + actualMax = parseDoubleFromUser(field.getName(), max); if (!maxInclusive) { actualMax = DoublePoint.nextDown(actualMax); } @@ -100,7 +100,7 @@ public class DoublePointField extends PointField implements DoubleValueFieldType @Override protected Query getExactQuery(SchemaField field, String externalVal) { - return DoublePoint.newExactQuery(field.getName(), Double.parseDouble(externalVal)); + return DoublePoint.newExactQuery(field.getName(), parseDoubleFromUser(field.getName(), externalVal)); } @Override @@ -112,7 +112,7 @@ public class DoublePointField extends PointField implements DoubleValueFieldType double[] values = new double[externalVal.size()]; int i = 0; for (String val:externalVal) { - values[i] = Double.parseDouble(val); + values[i] = parseDoubleFromUser(field.getName(), val); i++; } return DoublePoint.newSetQuery(field.getName(), values); @@ -127,7 +127,7 @@ public class DoublePointField extends PointField implements DoubleValueFieldType public void readableToIndexed(CharSequence val, BytesRefBuilder result) { result.grow(Double.BYTES); result.setLength(Double.BYTES); - DoublePoint.encodeDimension(Double.parseDouble(val.toString()), result.bytes(), 0); + DoublePoint.encodeDimension(parseDoubleFromUser(null, val.toString()), result.bytes(), 0); } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/FloatPointField.java b/solr/core/src/java/org/apache/solr/schema/FloatPointField.java index fe9c7533802..e1a97410323 100644 --- a/solr/core/src/java/org/apache/solr/schema/FloatPointField.java +++ b/solr/core/src/java/org/apache/solr/schema/FloatPointField.java @@ -61,7 +61,7 @@ public class FloatPointField extends PointField implements FloatValueFieldType { if (min == null) { actualMin = Float.NEGATIVE_INFINITY; } else { - actualMin = Float.parseFloat(min); + actualMin = parseFloatFromUser(field.getName(), min); if (!minInclusive) { actualMin = FloatPoint.nextUp(actualMin); } @@ -69,7 +69,7 @@ public class FloatPointField extends PointField implements FloatValueFieldType { if (max == null) { actualMax = Float.POSITIVE_INFINITY; } else { - actualMax = Float.parseFloat(max); + actualMax = parseFloatFromUser(field.getName(), max); if (!maxInclusive) { actualMax = FloatPoint.nextDown(actualMax); } @@ -100,7 +100,7 @@ public class FloatPointField extends PointField implements FloatValueFieldType { @Override protected Query getExactQuery(SchemaField field, String externalVal) { - return FloatPoint.newExactQuery(field.getName(), Float.parseFloat(externalVal)); + return FloatPoint.newExactQuery(field.getName(), parseFloatFromUser(field.getName(), externalVal)); } @Override @@ -112,7 +112,7 @@ public class FloatPointField extends PointField implements FloatValueFieldType { float[] values = new float[externalVal.size()]; int i = 0; for (String val:externalVal) { - values[i] = Float.parseFloat(val); + values[i] = parseFloatFromUser(field.getName(), val); i++; } return FloatPoint.newSetQuery(field.getName(), values); @@ -127,7 +127,7 @@ public class FloatPointField extends PointField implements FloatValueFieldType { public void readableToIndexed(CharSequence val, BytesRefBuilder result) { result.grow(Float.BYTES); result.setLength(Float.BYTES); - FloatPoint.encodeDimension(Float.parseFloat(val.toString()), result.bytes(), 0); + FloatPoint.encodeDimension(parseFloatFromUser(null, val.toString()), result.bytes(), 0); } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index 463df3ee681..36efbcf8a25 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -113,6 +113,7 @@ public class IndexSchema { public static final String SOURCE = "source"; public static final String TYPE = "type"; public static final String TYPES = "types"; + public static final String ROOT_FIELD_NAME = "_root_"; public static final String UNIQUE_KEY = "uniqueKey"; public static final String VERSION = "version"; @@ -517,6 +518,20 @@ public class IndexSchema { log.warn("no " + UNIQUE_KEY + " specified in schema."); } else { uniqueKeyField=getIndexedField(node.getNodeValue().trim()); + uniqueKeyFieldName=uniqueKeyField.getName(); + uniqueKeyFieldType=uniqueKeyField.getType(); + + // we fail on init if the ROOT field is *explicitly* defined as incompatible with uniqueKey + // we don't want ot fail if there happens to be a dynamicField matching ROOT, (ie: "*") + // because the user may not care about child docs at all. The run time code + // related to child docs can catch that if it happens + if (fields.containsKey(ROOT_FIELD_NAME) && ! isUsableForChildDocs()) { + String msg = ROOT_FIELD_NAME + " field must be defined using the exact same fieldType as the " + + UNIQUE_KEY + " field ("+uniqueKeyFieldName+") uses: " + uniqueKeyFieldType.getTypeName(); + log.error(msg); + throw new SolrException(ErrorCode.SERVER_ERROR, msg); + } + if (null != uniqueKeyField.getDefaultValue()) { String msg = UNIQUE_KEY + " field ("+uniqueKeyFieldName+ ") can not be configured with a default value ("+ @@ -542,9 +557,6 @@ public class IndexSchema { throw new SolrException(ErrorCode.SERVER_ERROR, msg); } - uniqueKeyFieldName=uniqueKeyField.getName(); - uniqueKeyFieldType=uniqueKeyField.getType(); - // Unless the uniqueKeyField is marked 'required=false' then make sure it exists if( Boolean.FALSE != explicitRequiredProp.get( uniqueKeyFieldName ) ) { uniqueKeyField.required = true; @@ -1914,4 +1926,17 @@ public class IndexSchema { + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE); return expression; } + + /** + * Helper method that returns true if the {@link #ROOT_FIELD_NAME} uses the exact + * same 'type' as the {@link #getUniqueKeyField()} + * + * @lucene.internal + */ + public boolean isUsableForChildDocs() { + FieldType rootType = getFieldType(ROOT_FIELD_NAME); + return (null != uniqueKeyFieldType && + null != rootType && + rootType.getTypeName().equals(uniqueKeyFieldType.getTypeName())); + } } diff --git a/solr/core/src/java/org/apache/solr/schema/IntPointField.java b/solr/core/src/java/org/apache/solr/schema/IntPointField.java index f47f45079e9..7d366127c0c 100644 --- a/solr/core/src/java/org/apache/solr/schema/IntPointField.java +++ b/solr/core/src/java/org/apache/solr/schema/IntPointField.java @@ -64,7 +64,7 @@ public class IntPointField extends PointField implements IntValueFieldType { if (min == null) { actualMin = Integer.MIN_VALUE; } else { - actualMin = Integer.parseInt(min); + actualMin = parseIntFromUser(field.getName(), min); if (!minInclusive) { actualMin++; } @@ -72,7 +72,7 @@ public class IntPointField extends PointField implements IntValueFieldType { if (max == null) { actualMax = Integer.MAX_VALUE; } else { - actualMax = Integer.parseInt(max); + actualMax = parseIntFromUser(field.getName(), max); if (!maxInclusive) { actualMax--; } @@ -97,7 +97,7 @@ public class IntPointField extends PointField implements IntValueFieldType { @Override protected Query getExactQuery(SchemaField field, String externalVal) { - return IntPoint.newExactQuery(field.getName(), Integer.parseInt(externalVal)); + return IntPoint.newExactQuery(field.getName(), parseIntFromUser(field.getName(), externalVal)); } @Override @@ -109,7 +109,7 @@ public class IntPointField extends PointField implements IntValueFieldType { int[] values = new int[externalVal.size()]; int i = 0; for (String val:externalVal) { - values[i] = Integer.parseInt(val); + values[i] = parseIntFromUser(field.getName(), val); i++; } return IntPoint.newSetQuery(field.getName(), values); @@ -124,7 +124,7 @@ public class IntPointField extends PointField implements IntValueFieldType { public void readableToIndexed(CharSequence val, BytesRefBuilder result) { result.grow(Integer.BYTES); result.setLength(Integer.BYTES); - IntPoint.encodeDimension(Integer.parseInt(val.toString()), result.bytes(), 0); + IntPoint.encodeDimension(parseIntFromUser(null, val.toString()), result.bytes(), 0); } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/LongPointField.java b/solr/core/src/java/org/apache/solr/schema/LongPointField.java index bef6c472346..b2d8a3a03b4 100644 --- a/solr/core/src/java/org/apache/solr/schema/LongPointField.java +++ b/solr/core/src/java/org/apache/solr/schema/LongPointField.java @@ -63,7 +63,7 @@ public class LongPointField extends PointField implements LongValueFieldType { if (min == null) { actualMin = Long.MIN_VALUE; } else { - actualMin = Long.parseLong(min); + actualMin = parseLongFromUser(field.getName(), min); if (!minInclusive) { actualMin++; } @@ -71,7 +71,7 @@ public class LongPointField extends PointField implements LongValueFieldType { if (max == null) { actualMax = Long.MAX_VALUE; } else { - actualMax = Long.parseLong(max); + actualMax = parseLongFromUser(field.getName(), max); if (!maxInclusive) { actualMax--; } @@ -96,7 +96,7 @@ public class LongPointField extends PointField implements LongValueFieldType { @Override protected Query getExactQuery(SchemaField field, String externalVal) { - return LongPoint.newExactQuery(field.getName(), Long.parseLong(externalVal)); + return LongPoint.newExactQuery(field.getName(), parseLongFromUser(field.getName(), externalVal)); } @Override @@ -108,7 +108,7 @@ public class LongPointField extends PointField implements LongValueFieldType { long[] values = new long[externalVal.size()]; int i = 0; for (String val:externalVal) { - values[i] = Long.parseLong(val); + values[i] = parseLongFromUser(field.getName(), val); i++; } return LongPoint.newSetQuery(field.getName(), values); @@ -123,7 +123,7 @@ public class LongPointField extends PointField implements LongValueFieldType { public void readableToIndexed(CharSequence val, BytesRefBuilder result) { result.grow(Long.BYTES); result.setLength(Long.BYTES); - LongPoint.encodeDimension(Long.parseLong(val.toString()), result.bytes(), 0); + LongPoint.encodeDimension(parseLongFromUser(null, val.toString()), result.bytes(), 0); } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/NumericFieldType.java b/solr/core/src/java/org/apache/solr/schema/NumericFieldType.java index 6cda9ca6438..cf17aaf7480 100644 --- a/solr/core/src/java/org/apache/solr/schema/NumericFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/NumericFieldType.java @@ -56,8 +56,8 @@ public abstract class NumericFieldType extends PrimitiveFieldType { switch (getNumberType()) { case INTEGER: return numericDocValuesRangeQuery(field.getName(), - min == null ? null : (long) Integer.parseInt(min), - max == null ? null : (long) Integer.parseInt(max), + min == null ? null : (long) parseIntFromUser(field.getName(), min), + max == null ? null : (long) parseIntFromUser(field.getName(), max), minInclusive, maxInclusive, field.multiValued()); case FLOAT: if (field.multiValued()) { @@ -67,8 +67,8 @@ public abstract class NumericFieldType extends PrimitiveFieldType { } case LONG: return numericDocValuesRangeQuery(field.getName(), - min == null ? null : Long.parseLong(min), - max == null ? null : Long.parseLong(max), + min == null ? null : parseLongFromUser(field.getName(), min), + max == null ? null : parseLongFromUser(field.getName(),max), minInclusive, maxInclusive, field.multiValued()); case DOUBLE: if (field.multiValued()) { @@ -90,8 +90,8 @@ public abstract class NumericFieldType extends PrimitiveFieldType { Query query; String fieldName = sf.getName(); - Number minVal = min == null ? null : getNumberType() == NumberType.FLOAT ? Float.parseFloat(min): Double.parseDouble(min); - Number maxVal = max == null ? null : getNumberType() == NumberType.FLOAT ? Float.parseFloat(max): Double.parseDouble(max); + Number minVal = min == null ? null : getNumberType() == NumberType.FLOAT ? parseFloatFromUser(sf.getName(), min): parseDoubleFromUser(sf.getName(), min); + Number maxVal = max == null ? null : getNumberType() == NumberType.FLOAT ? parseFloatFromUser(sf.getName(), max): parseDoubleFromUser(sf.getName(), max); Long minBits = min == null ? null : getNumberType() == NumberType.FLOAT ? (long) Float.floatToIntBits(minVal.floatValue()): Double.doubleToLongBits(minVal.doubleValue()); @@ -124,14 +124,14 @@ public abstract class NumericFieldType extends PrimitiveFieldType { } protected Query getRangeQueryForMultiValuedDoubleDocValues(SchemaField sf, String min, String max, boolean minInclusive, boolean maxInclusive) { - Long minBits = min == null ? NumericUtils.doubleToSortableLong(Double.NEGATIVE_INFINITY): NumericUtils.doubleToSortableLong(Double.parseDouble(min)); - Long maxBits = max == null ? NumericUtils.doubleToSortableLong(Double.POSITIVE_INFINITY): NumericUtils.doubleToSortableLong(Double.parseDouble(max)); + Long minBits = min == null ? NumericUtils.doubleToSortableLong(Double.NEGATIVE_INFINITY): NumericUtils.doubleToSortableLong(parseDoubleFromUser(sf.getName(), min)); + Long maxBits = max == null ? NumericUtils.doubleToSortableLong(Double.POSITIVE_INFINITY): NumericUtils.doubleToSortableLong(parseDoubleFromUser(sf.getName(), max)); return numericDocValuesRangeQuery(sf.getName(), minBits, maxBits, minInclusive, maxInclusive, true); } protected Query getRangeQueryForMultiValuedFloatDocValues(SchemaField sf, String min, String max, boolean minInclusive, boolean maxInclusive) { - Long minBits = (long)(min == null ? NumericUtils.floatToSortableInt(Float.NEGATIVE_INFINITY): NumericUtils.floatToSortableInt(Float.parseFloat(min))); - Long maxBits = (long)(max == null ? NumericUtils.floatToSortableInt(Float.POSITIVE_INFINITY): NumericUtils.floatToSortableInt(Float.parseFloat(max))); + Long minBits = (long)(min == null ? NumericUtils.floatToSortableInt(Float.NEGATIVE_INFINITY): NumericUtils.floatToSortableInt(parseFloatFromUser(sf.getName(), min))); + Long maxBits = (long)(max == null ? NumericUtils.floatToSortableInt(Float.POSITIVE_INFINITY): NumericUtils.floatToSortableInt(parseFloatFromUser(sf.getName(), max))); return numericDocValuesRangeQuery(sf.getName(), minBits, maxBits, minInclusive, maxInclusive, true); } @@ -169,4 +169,72 @@ public abstract class NumericFieldType extends PrimitiveFieldType { return NumericDocValuesField.newRangeQuery(field, actualLowerValue, actualUpperValue); } } + + /** + * Wrapper for {@link Long#parseLong(String)} that throws a BAD_REQUEST error if the input is not valid + * @param fieldName used in any exception, may be null + * @param val string to parse, NPE if null + */ + static long parseLongFromUser(String fieldName, String val) { + if (val == null) { + throw new NullPointerException("Invalid input" + (null == fieldName ? "" : " for field " + fieldName)); + } + try { + return Long.parseLong(val); + } catch (NumberFormatException e) { + String msg = "Invalid Number: " + val + (null == fieldName ? "" : " for field " + fieldName); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, msg); + } + } + + /** + * Wrapper for {@link Integer#parseInt(String)} that throws a BAD_REQUEST error if the input is not valid + * @param fieldName used in any exception, may be null + * @param val string to parse, NPE if null + */ + static int parseIntFromUser(String fieldName, String val) { + if (val == null) { + throw new NullPointerException("Invalid input" + (null == fieldName ? "" : " for field " + fieldName)); + } + try { + return Integer.parseInt(val); + } catch (NumberFormatException e) { + String msg = "Invalid Number: " + val + (null == fieldName ? "" : " for field " + fieldName); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, msg); + } + } + + /** + * Wrapper for {@link Double#parseDouble(String)} that throws a BAD_REQUEST error if the input is not valid + * @param fieldName used in any exception, may be null + * @param val string to parse, NPE if null + */ + static double parseDoubleFromUser(String fieldName, String val) { + if (val == null) { + throw new NullPointerException("Invalid input" + (null == fieldName ? "" : " for field " + fieldName)); + } + try { + return Double.parseDouble(val); + } catch (NumberFormatException e) { + String msg = "Invalid Number: " + val + (null == fieldName ? "" : " for field " + fieldName); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, msg); + } + } + + /** + * Wrapper for {@link Float#parseFloat(String)} that throws a BAD_REQUEST error if the input is not valid + * @param fieldName used in any exception, may be null + * @param val string to parse, NPE if null + */ + static float parseFloatFromUser(String fieldName, String val) { + if (val == null) { + throw new NullPointerException("Invalid input" + (null == fieldName ? "" : " for field " + fieldName)); + } + try { + return Float.parseFloat(val); + } catch (NumberFormatException e) { + String msg = "Invalid Number: " + val + (null == fieldName ? "" : " for field " + fieldName); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, msg); + } + } } diff --git a/solr/core/src/java/org/apache/solr/schema/PointField.java b/solr/core/src/java/org/apache/solr/schema/PointField.java index cad3c7e9706..98105af2ec8 100644 --- a/solr/core/src/java/org/apache/solr/schema/PointField.java +++ b/solr/core/src/java/org/apache/solr/schema/PointField.java @@ -127,7 +127,7 @@ public abstract class PointField extends NumericFieldType { return new IndexOrDocValuesQuery(pointsQuery, dvQuery); } else { return getExactQuery(field, externalVal); - } + } } protected abstract Query getExactQuery(SchemaField field, String externalVal); @@ -191,6 +191,11 @@ public abstract class PointField extends NumericFieldType { protected abstract String indexedToReadable(BytesRef indexedForm); + @Override + public Query getPrefixQuery(QParser parser, SchemaField sf, String termStr) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can't run prefix queries on numeric fields"); + } + protected boolean isFieldUsed(SchemaField field) { boolean indexed = field.indexed(); boolean stored = field.stored(); diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java index f90877cbfdd..998583b449e 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java @@ -344,7 +344,7 @@ public class TrieField extends NumericFieldType { } int ps = precisionStep; Query query; - + if (field.hasDocValues() && !field.indexed()) { return getDocValuesRangeQuery(parser, field, min, max, minInclusive, maxInclusive); } @@ -352,26 +352,26 @@ public class TrieField extends NumericFieldType { switch (type) { case INTEGER: query = LegacyNumericRangeQuery.newIntRange(field.getName(), ps, - min == null ? null : Integer.parseInt(min), - max == null ? null : Integer.parseInt(max), + min == null ? null : parseIntFromUser(field.getName(), min), + max == null ? null : parseIntFromUser(field.getName(), max), minInclusive, maxInclusive); break; case FLOAT: query = LegacyNumericRangeQuery.newFloatRange(field.getName(), ps, - min == null ? null : Float.parseFloat(min), - max == null ? null : Float.parseFloat(max), + min == null ? null : parseFloatFromUser(field.getName(), min), + max == null ? null : parseFloatFromUser(field.getName(), max), minInclusive, maxInclusive); break; case LONG: query = LegacyNumericRangeQuery.newLongRange(field.getName(), ps, - min == null ? null : Long.parseLong(min), - max == null ? null : Long.parseLong(max), + min == null ? null : parseLongFromUser(field.getName(), min), + max == null ? null : parseLongFromUser(field.getName(), max), minInclusive, maxInclusive); break; case DOUBLE: query = LegacyNumericRangeQuery.newDoubleRange(field.getName(), ps, - min == null ? null : Double.parseDouble(min), - max == null ? null : Double.parseDouble(max), + min == null ? null : parseDoubleFromUser(field.getName(), min), + max == null ? null : parseDoubleFromUser(field.getName(), max), minInclusive, maxInclusive); break; case DATE: @@ -383,7 +383,6 @@ public class TrieField extends NumericFieldType { default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field"); } - return query; } @@ -413,29 +412,24 @@ public class TrieField extends NumericFieldType { @Override public void readableToIndexed(CharSequence val, BytesRefBuilder result) { String s = val.toString(); - try { - switch (type) { - case INTEGER: - LegacyNumericUtils.intToPrefixCoded(Integer.parseInt(s), 0, result); - break; - case FLOAT: - LegacyNumericUtils.intToPrefixCoded(NumericUtils.floatToSortableInt(Float.parseFloat(s)), 0, result); - break; - case LONG: - LegacyNumericUtils.longToPrefixCoded(Long.parseLong(s), 0, result); - break; - case DOUBLE: - LegacyNumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(Double.parseDouble(s)), 0, result); - break; - case DATE: - LegacyNumericUtils.longToPrefixCoded(DateMathParser.parseMath(null, s).getTime(), 0, result); - break; - default: - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type); - } - } catch (NumberFormatException nfe) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Invalid Number: " + val); + switch (type) { + case INTEGER: + LegacyNumericUtils.intToPrefixCoded(parseIntFromUser(null, s), 0, result); + break; + case FLOAT: + LegacyNumericUtils.intToPrefixCoded(NumericUtils.floatToSortableInt(parseFloatFromUser(null, s)), 0, result); + break; + case LONG: + LegacyNumericUtils.longToPrefixCoded(parseLongFromUser(null, s), 0, result); + break; + case DOUBLE: + LegacyNumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(parseDoubleFromUser(null, s)), 0, result); + break; + case DATE: + LegacyNumericUtils.longToPrefixCoded(DateMathParser.parseMath(null, s).getTime(), 0, result); + break; + default: + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type); } } diff --git a/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java index 596ddd3d418..cb1af9a29f2 100644 --- a/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java @@ -191,7 +191,7 @@ public class AddUpdateCommand extends UpdateCommand implements Iterable flatten(SolrInputDocument root) { List unwrappedDocs = new ArrayList<>(); recUnwrapp(unwrappedDocs, root); + if (1 < unwrappedDocs.size() && ! req.getSchema().isUsableForChildDocs()) { + throw new SolrException + (SolrException.ErrorCode.BAD_REQUEST, "Unable to index docs with children: the schema must " + + "include definitions for both a uniqueKey field and the '" + IndexSchema.ROOT_FIELD_NAME + + "' field, using the exact same fieldType"); + } return unwrappedDocs; } diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java index e4811091c73..3efb748fd02 100644 --- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -55,6 +55,7 @@ import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.FunctionRangeQuery; import org.apache.solr.search.QParser; @@ -394,7 +395,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState } private Term getIdTerm(AddUpdateCommand cmd) { - return new Term(cmd.isBlock() ? "_root_" : idField.getName(), cmd.getIndexedId()); + return new Term(cmd.isBlock() ? IndexSchema.ROOT_FIELD_NAME : idField.getName(), cmd.getIndexedId()); } private void updateDeleteTrackers(DeleteUpdateCommand cmd) { diff --git a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java index b97af3bcb05..58638ae63c4 100644 --- a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java +++ b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java @@ -176,8 +176,8 @@ public class DocumentBuilder { // check if the copy field is a multivalued or not if (!destinationField.multiValued() && destHasValues) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "ERROR: "+getID(doc, schema)+"multiple values encountered for non multiValued copy field " + - destinationField.getName() + ": " + v); + "Multiple values encountered for non multiValued copy field " + + destinationField.getName() + ": " + v); } used = true; @@ -198,7 +198,9 @@ public class DocumentBuilder { } } catch( SolrException ex ) { - throw ex; + throw new SolrException(SolrException.ErrorCode.getErrorCode(ex.code()), + "ERROR: "+getID(doc, schema)+"Error adding field '" + + field.getName() + "'='" +field.getValue()+"' msg=" + ex.getMessage(), ex ); } catch( Exception ex ) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-diff-type-dynamic-root.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-diff-type-dynamic-root.xml new file mode 100644 index 00000000000..4f06bd1270c --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-diff-type-dynamic-root.xml @@ -0,0 +1,36 @@ + + + + + + + + + id + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-diff-type-root.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-diff-type-root.xml new file mode 100644 index 00000000000..377762d0443 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-diff-type-root.xml @@ -0,0 +1,35 @@ + + + + + + + + + id + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml index 05145f9464f..f0ddae44c46 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml @@ -734,11 +734,25 @@ useDocValuesAsStored="true"/> - + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema11.xml b/solr/core/src/test-files/solr/collection1/conf/schema11.xml index caa24cc2f25..819b6d1b9eb 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema11.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema11.xml @@ -413,6 +413,10 @@ valued. --> + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema12.xml b/solr/core/src/test-files/solr/collection1/conf/schema12.xml index 214fc26c65b..22363952b1a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema12.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema12.xml @@ -604,16 +604,33 @@ + + + + + + + + + + + + + + + + + + + + - - - diff --git a/solr/core/src/test-files/solr/configsets/cloud-hdfs/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-hdfs/conf/schema.xml index aab5e811110..7b8b690d395 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-hdfs/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-hdfs/conf/schema.xml @@ -22,7 +22,7 @@ - + id diff --git a/solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/managed-schema b/solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/managed-schema index e70e02b36f8..39e142e2417 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/managed-schema +++ b/solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/managed-schema @@ -35,7 +35,7 @@ - + id diff --git a/solr/core/src/test-files/solr/configsets/cloud-managed-upgrade/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-managed-upgrade/conf/schema.xml index b9f09f9a50b..1d97a2ac5eb 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-managed-upgrade/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-managed-upgrade/conf/schema.xml @@ -21,7 +21,7 @@ - + id diff --git a/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema b/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema index b9f09f9a50b..1d97a2ac5eb 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema +++ b/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema @@ -21,7 +21,7 @@ - + id diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal-jmx/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal-jmx/conf/schema.xml index aab5e811110..7b8b690d395 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-minimal-jmx/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal-jmx/conf/schema.xml @@ -22,7 +22,7 @@ - + id diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml index aab5e811110..7b8b690d395 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml @@ -22,7 +22,7 @@ - + id diff --git a/solr/core/src/test-files/solr/configsets/cloud-subdirs/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-subdirs/conf/schema.xml index aab5e811110..7b8b690d395 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-subdirs/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-subdirs/conf/schema.xml @@ -22,7 +22,7 @@ - + id diff --git a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml index aab5e811110..7b8b690d395 100644 --- a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml @@ -22,7 +22,7 @@ - + id diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java index 02ae888bd7d..b1747fd9afb 100644 --- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java +++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java @@ -341,6 +341,59 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { ); } + @Test + public void testClientErrorOnMalformedDate() throws Exception { + final String BAD_VALUE = "NOT_A_DATE"; + ignoreException(BAD_VALUE); + + final List FIELDS = new LinkedList<>(); + for (String type : new String[] { + "tdt", "tdt1", "tdtdv", "tdtdv1", + "dt_dv", "dt_dvo", "dt", "dt1", "dt_os" + }) { + FIELDS.add("malformed_" + type); + } + + // test that malformed numerics cause client error not server error + for (String field : FIELDS) { + try { + h.update(add( doc("id","100", field, BAD_VALUE))); + fail("Didn't encounter an error trying to add a bad date: " + field); + } catch (SolrException e) { + String msg = e.toString(); + assertTrue("not an (update) client error on field: " + field +" : "+ msg, + 400 <= e.code() && e.code() < 500); + assertTrue("(update) client error does not mention bad value: " + msg, + msg.contains(BAD_VALUE)); + assertTrue("client error does not mention document id: " + msg, + msg.contains("[doc=100]")); + } + SchemaField sf = h.getCore().getLatestSchema().getField(field); + if (!sf.hasDocValues() && !sf.indexed()) { + continue; + } + try { + h.query(req("q",field + ":" + BAD_VALUE)); + fail("Didn't encounter an error trying to query a bad date: " + field); + } catch (SolrException e) { + String msg = e.toString(); + assertTrue("not a (search) client error on field: " + field +" : "+ msg, + 400 <= e.code() && e.code() < 500); + assertTrue("(search) client error does not mention bad value: " + msg, + msg.contains(BAD_VALUE)); + } + try { + h.query(req("q",field + ":[NOW TO " + BAD_VALUE + "]")); + fail("Didn't encounter an error trying to query a bad date: " + field); + } catch (SolrException e) { + String msg = e.toString(); + assertTrue("not a (search) client error on field: " + field +" : "+ msg, + 400 <= e.code() && e.code() < 500); + assertTrue("(search) client error does not mention bad value: " + msg, + msg.contains(BAD_VALUE)); + } + } + } @Test public void testClientErrorOnMalformedNumbers() throws Exception { @@ -349,7 +402,13 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { ignoreException(BAD_VALUE); final List FIELDS = new LinkedList<>(); - for (String type : new String[] { "ti", "tf", "td", "tl" }) { + for (String type : new String[] { + "ti", "tf", "td", "tl", + "i", "f", "d", "l", + "i_dv", "f_dv", "d_dv", "l_dv", + "i_dvo", "f_dvo", "d_dvo", "l_dvo", + "i_os", "f_os", "d_os", "l_os" + }) { FIELDS.add("malformed_" + type); } @@ -364,6 +423,12 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { 400 <= e.code() && e.code() < 500); assertTrue("(update) client error does not mention bad value: " + msg, msg.contains(BAD_VALUE)); + assertTrue("client error does not mention document id", + msg.contains("[doc=100]")); + } + SchemaField sf = h.getCore().getLatestSchema().getField(field); + if (!sf.hasDocValues() && !sf.indexed()) { + continue; } try { h.query(req("q",field + ":" + BAD_VALUE)); @@ -375,6 +440,16 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { assertTrue("(search) client error does not mention bad value: " + msg, msg.contains(BAD_VALUE)); } + try { + h.query(req("q",field + ":[10 TO " + BAD_VALUE + "]")); + fail("Didn't encounter an error trying to query a non-number: " + field); + } catch (SolrException e) { + String msg = e.toString(); + assertTrue("not a (search) client error on field: " + field +" : "+ msg, + 400 <= e.code() && e.code() < 500); + assertTrue("(search) client error does not mention bad value: " + msg, + msg.contains(BAD_VALUE)); + } } } diff --git a/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java b/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java index 5545d959026..f635436fa83 100644 --- a/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java +++ b/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java @@ -101,6 +101,21 @@ public class BadIndexSchemaTest extends AbstractBadConfigTestBase { public void testDocValuesUnsupported() throws Exception { doTest("bad-schema-unsupported-docValues.xml", "does not support doc values"); } + + public void testRootTypeMissmatchWithUniqueKey() throws Exception { + doTest("bad-schema-uniquekey-diff-type-root.xml", + "using the exact same fieldType as the uniqueKey field (id) uses: string1"); + } + + public void testRootTypeDynamicMissmatchWithUniqueKey() throws Exception { + // in this case, the core should load fine -- but we should get an error adding docs + try { + initCore("solrconfig.xml","bad-schema-uniquekey-diff-type-dynamic-root.xml"); + assertFailedU("Unable to index docs with children", adoc(sdocWithChildren("1","-1"))); + } finally { + deleteCore(); + } + } public void testSweetSpotSimBadConfig() throws Exception { doTest("bad-schema-sweetspot-both-tf.xml", "Can not mix"); diff --git a/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java b/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java index 3f57ee5da0c..f282c3f2171 100644 --- a/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java +++ b/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java @@ -20,6 +20,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; import org.apache.solr.util.AbstractSolrTestCase; import org.junit.BeforeClass; +import org.junit.Test; public class TestQueryTypes extends AbstractSolrTestCase { @@ -209,7 +210,6 @@ public class TestQueryTypes extends AbstractSolrTestCase { ,"//result[@numFound='2']" ); - // // test escapes in quoted strings // @@ -436,5 +436,49 @@ public class TestQueryTypes extends AbstractSolrTestCase { assertQ("Test text field with no analysis doesn't NPE with wildcards (SOLR-4318)", req("q", "text_no_analyzer:should*"), "//result[@numFound='1']"); + + } + + @Test + public void testNumericBadRequests() { + String[] suffixes = new String[50]; + int fieldNum = 0; + for (String type:new String[]{"i", "l", "f", "d", "dt"}) { + for (String s:new String[]{"", "s"}) { + //Trie + suffixes[fieldNum++] = "t" + type + s; + suffixes[fieldNum++] = "t" + type + s + "_dv"; + suffixes[fieldNum++] = "t" + type + s + "_ni_dv"; + + //Points + suffixes[fieldNum++] = type + s + "_p"; + suffixes[fieldNum++] = type + s + "_ni_p"; + } + } + assertEquals(fieldNum,suffixes.length); + + String badNumber = "NOT_A_NUMBER"; + for (String suffix:suffixes) { + // Numeric bad requests + assertQEx("Expecting exception for suffix: " + suffix, badNumber, req("q","{!term f=foo_" + suffix + "}" + badNumber), SolrException.ErrorCode.BAD_REQUEST); + assertQEx("Expecting exception for suffix: " + suffix, badNumber, req("q","{!terms f=foo_" + suffix + "}1 2 3 4 5 " + badNumber), SolrException.ErrorCode.BAD_REQUEST); + assertQEx("Expecting exception for suffix: " + suffix, badNumber, req("q","{!lucene}foo_" + suffix + ":" + badNumber), SolrException.ErrorCode.BAD_REQUEST); + assertQEx("Expecting exception for suffix: " + suffix, badNumber, req("q","{!field f=foo_" + suffix + "}" + badNumber), SolrException.ErrorCode.BAD_REQUEST); + assertQEx("Expecting exception for suffix: " + suffix, badNumber, req("q","{!maxscore}foo_" + suffix + ":" + badNumber), SolrException.ErrorCode.BAD_REQUEST); + assertQEx("Expecting exception for suffix: " + suffix, badNumber, + req("q","{!xmlparser}"), SolrException.ErrorCode.BAD_REQUEST); + if (suffix.contains("_p")) { + // prefix queries work in Trie fields + assertQEx("Expecting exception for suffix: " + suffix, "Can't run prefix queries on numeric fields", + req("q","{!prefix f=foo_" + suffix + "}NOT_A_NUMBER"), SolrException.ErrorCode.BAD_REQUEST); + assertQEx("Expecting exception for suffix: " + suffix, "Can't run prefix queries on numeric fields", + req("q","{!lucene}foo_" + suffix + ":123*"), SolrException.ErrorCode.BAD_REQUEST); + } + + // Skipping: func, boost, raw, nested, frange, spatial*, join, surround, switch, parent, child, collapsing, + // complexphrase, rerank, export, mlt, hash, graph, graphTerms, igain, tlogit, sigificantTerms, payload* + // Maybe add: raw, join, parent, child, collapsing, graphTerms, igain, sigificantTerms, simple + } + } } diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java index 1a2e5720822..4b82e6243d3 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java +++ b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.metrics.MetricsMap; import org.apache.solr.common.params.ModifiableSolrParams; @@ -1038,4 +1039,39 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 { , "/response/numFound==1" ); } + + @Test + public void testBadRequestInSetQuery() throws SyntaxError { + SolrQueryRequest req = req(); + QParser qParser; + String[] fieldSuffix = new String[] { + "ti", "tf", "td", "tl", + "i", "f", "d", "l", + "is", "fs", "ds", "ls", + "i_dv", "f_dv", "d_dv", "l_dv", + "is_dv", "fs_dv", "ds_dv", "ls_dv", + "i_dvo", "f_dvo", "d_dvo", "l_dvo", + }; + + for (String suffix:fieldSuffix) { + //Good queries + qParser = QParser.getParser("foo_" + suffix + ":(1 2 3 4 5 6 7 8 9 10 20 19 18 17 16 15 14 13 12 25)", req); + qParser.setIsFilter(true); + qParser.getQuery(); + } + + for (String suffix:fieldSuffix) { + qParser = QParser.getParser("foo_" + suffix + ":(1 2 3 4 5 6 7 8 9 10 20 19 18 17 16 15 14 13 12 NOT_A_NUMBER)", req); + qParser.setIsFilter(true); // this may change in the future + try { + qParser.getQuery(); + fail("Expecting exception"); + } catch (SolrException e) { + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + assertTrue("Unexpected exception: " + e.getMessage(), e.getMessage().contains("Invalid Number: NOT_A_NUMBER")); + } + } + + + } } \ No newline at end of file From 061a768d873b9c7c75b635beee3effef6e66bb2f Mon Sep 17 00:00:00 2001 From: Tomas Fernandez Lobbe Date: Tue, 13 Jun 2017 10:57:45 -0700 Subject: [PATCH 073/131] SOLR-10833: Updated CHANGES to include SOLR-10833 --- solr/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index c1c7293f969..7468fc3827d 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -370,6 +370,9 @@ Bug Fixes * SOLR-10704: REPLACENODE may cause data loss when replicationFactor is 1. (ab, shalin) +* SOLR-10833: Point numeric fields should throw SolrException(BAD_REQUEST) for malformed numbers in queries. + Trie numeric fields should throw SolrException(BAD_REQUEST) for malformed docValues range queries. + (hossman, Tomás Fernández Löbbe) Optimizations ---------------------- From 9a0d9e83f69e9e10e18621f7bebe08db90b2f3d4 Mon Sep 17 00:00:00 2001 From: Mark Miller Date: Tue, 13 Jun 2017 15:55:51 -0400 Subject: [PATCH 074/131] SOLR-9910: Add solr/solr.cmd parameter to append jetty parameters to the start script. --- solr/CHANGES.txt | 3 +++ solr/bin/solr | 17 ++++++++++++++--- solr/bin/solr.cmd | 22 ++++++++++++++++++++-- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 7468fc3827d..7810c816de9 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -343,6 +343,9 @@ New Features * SOLR-10660: Add reverse Stream Evaluator (Joel Bernstein) +* SOLR-9910: Add solr/solr.cmd parameter to append jetty parameters to the start script. + (Mano Kovacs via Mark Miller) + Bug Fixes ---------------------- * SOLR-10723 JSON Facet API: resize() implemented incorrectly for CountSlotAcc, HllAgg.NumericAcc diff --git a/solr/bin/solr b/solr/bin/solr index b7bf142de8d..6b2b80636be 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -340,6 +340,11 @@ function print_usage() { echo " you could pass: -a \"-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=18983\"" echo " In most cases, you should wrap the additional parameters in double quotes." echo "" + echo " -j Additional parameters to pass to Jetty when starting Solr." + echo " For example, to add configuration folder that jetty should read" + echo " you could pass: -j \"--include-jetty-dir=/etc/jetty/custom/server/\"" + echo " In most cases, you should wrap the additional parameters in double quotes." + echo "" echo " -noprompt Don't prompt for input; accept all defaults when running examples that accept user input" echo "" echo " -v and -q Verbose (-v) or quiet (-q) logging. Sets default log level to DEBUG or WARN instead of INFO" @@ -1437,6 +1442,11 @@ if [ $# -gt 0 ]; then PASS_TO_RUN_EXAMPLE+=" -a \"$ADDITIONAL_CMD_OPTS\"" shift 2 ;; + -j|-jettyconfig) + ADDITIONAL_JETTY_CONFIG="$2" + PASS_TO_RUN_EXAMPLE+=" -j \"$ADDITIONAL_JETTY_CONFIG\"" + shift 2 + ;; -k|-key) STOP_KEY="$2" shift 2 @@ -1803,6 +1813,7 @@ function launch_solr() { stop_port="$STOP_PORT" SOLR_ADDL_ARGS="$2" + SOLR_JETTY_ADDL_CONFIG="$3" # define default GC_TUNE if [ -z ${GC_TUNE+x} ]; then @@ -1911,12 +1922,12 @@ function launch_solr() { esac if [ "$run_in_foreground" == "true" ]; then - exec "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -jar start.jar "${SOLR_JETTY_CONFIG[@]}" + exec "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -jar start.jar "${SOLR_JETTY_CONFIG[@]}" $SOLR_JETTY_ADDL_CONFIG else # run Solr in the background nohup "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -Dsolr.log.muteconsole \ "-XX:OnOutOfMemoryError=$SOLR_TIP/bin/oom_solr.sh $SOLR_PORT $SOLR_LOGS_DIR" \ - -jar start.jar "${SOLR_JETTY_CONFIG[@]}" \ + -jar start.jar "${SOLR_JETTY_CONFIG[@]}" $SOLR_JETTY_ADDL_CONFIG \ 1>"$SOLR_LOGS_DIR/solr-$SOLR_PORT-console.log" 2>&1 & echo $! > "$SOLR_PID_DIR/solr-$SOLR_PORT.pid" # check if /proc/sys/kernel/random/entropy_avail exists then check output of cat /proc/sys/kernel/random/entropy_avail to see if less than 300 @@ -1960,6 +1971,6 @@ function launch_solr() { fi } -launch_solr "$FG" "$ADDITIONAL_CMD_OPTS" +launch_solr "$FG" "$ADDITIONAL_CMD_OPTS" "$ADDITIONAL_JETTY_CONFIG" exit $? diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 820b89f0328..0afea02f5ac 100644 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -323,6 +323,11 @@ goto done @echo you could pass: -a "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=18983" @echo In most cases, you should wrap the additional parameters in double quotes. @echo. +@echo -j opts Additional parameters to pass to Jetty when starting Solr. +@echo For example, to add configuration folder that jetty should read +@echo you could pass: -j "--include-jetty-dir=/etc/jetty/custom/server/" +@echo In most cases, you should wrap the additional parameters in double quotes. +@echo. @echo -noprompt Don't prompt for input; accept all defaults when running examples that accept user input @echo. @echo -v and -q Verbose (-v) or quiet (-q) logging. Sets default log level to DEBUG or WARN instead of INFO @@ -610,6 +615,8 @@ IF "%1"=="-z" goto set_zookeeper IF "%1"=="-zkhost" goto set_zookeeper IF "%1"=="-a" goto set_addl_opts IF "%1"=="-addlopts" goto set_addl_opts +IF "%1"=="-j" goto set_addl_jetty_config +IF "%1"=="-jettyconfig" goto set_addl_jetty_config IF "%1"=="-noprompt" goto set_noprompt IF "%1"=="-k" goto set_stop_key IF "%1"=="-key" goto set_stop_key @@ -816,6 +823,13 @@ SHIFT SHIFT goto parse_args +:set_addl_jetty_config +set "arg=%~2" +set "SOLR_JETTY_ADDL_CONFIG=%~2" +SHIFT +SHIFT +goto parse_args + :set_passthru set "PASSTHRU=%~1=%~2" IF NOT "%SOLR_OPTS%"=="" ( @@ -1112,6 +1126,10 @@ IF "%verbose%"=="1" ( CALL :safe_echo " SOLR_ADDL_ARGS = %SOLR_ADDL_ARGS%" ) + IF NOT "%SOLR_JETTY_ADDL_CONFIG%"=="" ( + CALL :safe_echo " SOLR_JETTY_ADDL_CONFIG = %SOLR_JETTY_ADDL_CONFIG%" + ) + IF "%ENABLE_REMOTE_JMX_OPTS%"=="true" ( @echo RMI_PORT = !RMI_PORT! @echo REMOTE_JMX_OPTS = %REMOTE_JMX_OPTS% @@ -1176,7 +1194,7 @@ IF "%FG%"=="1" ( -Dlog4j.configuration="%LOG4J_CONFIG%" -DSTOP.PORT=!STOP_PORT! -DSTOP.KEY=%STOP_KEY% ^ -Dsolr.solr.home="%SOLR_HOME%" -Dsolr.install.dir="%SOLR_TIP%" ^ -Djetty.host=%SOLR_JETTY_HOST% -Djetty.port=%SOLR_PORT% -Djetty.home="%SOLR_SERVER_DIR%" ^ - -Djava.io.tmpdir="%SOLR_SERVER_DIR%\tmp" -jar start.jar "%SOLR_JETTY_CONFIG%" + -Djava.io.tmpdir="%SOLR_SERVER_DIR%\tmp" -jar start.jar "%SOLR_JETTY_CONFIG%" "%SOLR_JETTY_ADDL_CONFIG%" ) ELSE ( START /B "Solr-%SOLR_PORT%" /D "%SOLR_SERVER_DIR%" ^ "%JAVA%" %SERVEROPT% %SOLR_JAVA_MEM% %START_OPTS% %GCLOG_OPT% ^ @@ -1184,7 +1202,7 @@ IF "%FG%"=="1" ( -Dsolr.log.muteconsole ^ -Dsolr.solr.home="%SOLR_HOME%" -Dsolr.install.dir="%SOLR_TIP%" ^ -Djetty.host=%SOLR_JETTY_HOST% -Djetty.port=%SOLR_PORT% -Djetty.home="%SOLR_SERVER_DIR%" ^ - -Djava.io.tmpdir="%SOLR_SERVER_DIR%\tmp" -jar start.jar "%SOLR_JETTY_CONFIG%" > "!SOLR_LOGS_DIR!\solr-%SOLR_PORT%-console.log" + -Djava.io.tmpdir="%SOLR_SERVER_DIR%\tmp" -jar start.jar "%SOLR_JETTY_CONFIG%" "%SOLR_JETTY_ADDL_CONFIG%" > "!SOLR_LOGS_DIR!\solr-%SOLR_PORT%-console.log" echo %SOLR_PORT%>"%SOLR_TIP%"\bin\solr-%SOLR_PORT%.port REM now wait to see Solr come online ... From d1db5f7af9edb2125583c6c63fa322380ee57cf7 Mon Sep 17 00:00:00 2001 From: Mike Drob Date: Tue, 13 Jun 2017 14:18:52 -0700 Subject: [PATCH 075/131] Revert "SOLR-8392: Remove instanceof checks on return value of SolrParam::get" This reverts commit 94220a01e14f53e0632bfbc1678661ad9c67320a. --- solr/CHANGES.txt | 2 -- .../org/apache/solr/handler/SchemaHandler.java | 6 +++++- .../org/apache/solr/rest/BaseSolrResource.java | 6 ++++-- .../solr/common/params/MapSolrParams.java | 17 +++++++++++++---- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 7810c816de9..a07b68f1b00 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -260,8 +260,6 @@ Other Changes * SOLR-10800: Factor out HttpShardHandler.transformReplicasToShardUrls from HttpShardHandler.prepDistributed. (Domenico Fabio Marino, Christine Poerschke) -* SOLR-8392: Remove instanceof checks on return value of SolrParam::get (Mike Drob, Varun Thacker) - ================== 6.7.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java index 626f6557b8d..e3e292b6554 100644 --- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java @@ -138,7 +138,11 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware, break; } case "/schema/zkversion": { - int refreshIfBelowVersion = req.getParams().getInt("refreshIfBelowVersion", -1); + int refreshIfBelowVersion = -1; + Object refreshParam = req.getParams().get("refreshIfBelowVersion"); + if (refreshParam != null) + refreshIfBelowVersion = (refreshParam instanceof Number) ? ((Number) refreshParam).intValue() + : Integer.parseInt(refreshParam.toString()); int zkVersion = -1; IndexSchema schema = req.getSchema(); if (schema instanceof ManagedIndexSchema) { diff --git a/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java b/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java index cc1eeb2f936..5a9310d84e9 100644 --- a/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java +++ b/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java @@ -125,9 +125,11 @@ public abstract class BaseSolrResource extends ServerResource { SolrCore.preDecorateResponse(solrRequest, solrResponse); // client application can set a timeout for update requests - String updateTimeoutSecsParam = getSolrRequest().getParams().get(UPDATE_TIMEOUT_SECS); + Object updateTimeoutSecsParam = getSolrRequest().getParams().get(UPDATE_TIMEOUT_SECS); if (updateTimeoutSecsParam != null) - updateTimeoutSecs = Integer.parseInt(updateTimeoutSecsParam); + updateTimeoutSecs = (updateTimeoutSecsParam instanceof Number) + ? ((Number) updateTimeoutSecsParam).intValue() + : Integer.parseInt(updateTimeoutSecsParam.toString()); } } diff --git a/solr/solrj/src/java/org/apache/solr/common/params/MapSolrParams.java b/solr/solrj/src/java/org/apache/solr/common/params/MapSolrParams.java index 53eff87e1f2..5454fcac277 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/MapSolrParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/MapSolrParams.java @@ -20,7 +20,7 @@ import java.util.Iterator; import java.util.Map; /** - * {@link SolrParams} implementation that can be built from and is backed by a {@link Map}. + * */ public class MapSolrParams extends SolrParams { protected final Map map; @@ -31,13 +31,22 @@ public class MapSolrParams extends SolrParams { @Override public String get(String name) { - return map.get(name); + Object o = map.get(name); + if(o == null) return null; + if (o instanceof String) return (String) o; + if (o instanceof String[]) { + String[] strings = (String[]) o; + if(strings.length == 0) return null; + return strings[0]; + } + return String.valueOf(o); } @Override public String[] getParams(String name) { - String val = map.get(name); - return val==null ? null : new String[]{val}; + Object val = map.get(name); + if (val instanceof String[]) return (String[]) val; + return val==null ? null : new String[]{String.valueOf(val)}; } @Override From a18a4ce2450b33b0f03dc9882557fa48040f54c8 Mon Sep 17 00:00:00 2001 From: yonik Date: Tue, 13 Jun 2017 17:31:30 -0400 Subject: [PATCH 076/131] SOLR-7452: convert bucket values in FacetStream from Integer to Long for calcite, make bucket labels in JSON Facet API consistent for facet refinement --- .../search/facet/FacetFieldProcessor.java | 1 + .../apache/solr/search/facet/FacetRange.java | 6 +++- .../search/facet/TestJsonFacetRefinement.java | 36 ++++++++++++------- .../solr/search/facet/TestJsonFacets.java | 6 ++++ .../client/solrj/io/stream/FacetStream.java | 3 ++ 5 files changed, 39 insertions(+), 13 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java index 7b08e14ac3a..143cfd64c5f 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java @@ -680,6 +680,7 @@ abstract class FacetFieldProcessor extends FacetProcessor { private SimpleOrderedMap refineBucket(Object bucketVal, boolean skip, Map facetInfo) throws IOException { SimpleOrderedMap bucket = new SimpleOrderedMap<>(); FieldType ft = sf.getType(); + bucketVal = ft.toNativeType(bucketVal); // refinement info passed in as JSON will cause int->long and float->double bucket.add("val", bucketVal); // String internal = ft.toInternal( tobj.toString() ); // TODO - we need a better way to get from object to query... diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java index 398fa63fc82..f627348c39d 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java @@ -533,6 +533,10 @@ class FacetRangeProcessor extends FacetProcessor { public IntCalc(final SchemaField f) { super(f); } @Override + public Comparable bitsToValue(long bits) { + return (int)bits; + } + @Override protected Integer parseStr(String rawval) { return Integer.valueOf(rawval); } @@ -700,7 +704,7 @@ class FacetRangeProcessor extends FacetProcessor { SimpleOrderedMap bucket = new SimpleOrderedMap<>(); FieldType ft = sf.getType(); - bucket.add("val", bucketVal); + bucket.add("val", range.low); // use "low" instead of bucketVal because it will be the right type (we may have been passed back long instead of int for example) // String internal = ft.toInternal( tobj.toString() ); // TODO - we need a better way to get from object to query... Query domainQ = sf.getType().getRangeQuery(null, sf, range.low == null ? null : calc.formatValue(range.low), range.high==null ? null : calc.formatValue(range.high), range.includeLower, range.includeUpper); diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java index 7cf14280c66..94b753f8f04 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java @@ -236,18 +236,18 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS { @Test public void testBasicRefinement() throws Exception { - ModifiableSolrParams p = params("cat_s", "cat_s", "xy_s", "xy_s", "num_d", "num_d", "qw_s", "qw_s", "er_s","er_s"); + ModifiableSolrParams p = params("cat_s", "cat_s", "cat_i", "cat_i", "xy_s", "xy_s", "num_d", "num_d", "qw_s", "qw_s", "er_s","er_s"); doBasicRefinement( p ); p.set("terms","method:dvhash,"); doBasicRefinement( p ); - // multi-valued strings - p = params("cat_s", "cat_ss", "xy_s", "xy_ss", "num_d", "num_d", "qw_s", "qw_ss", "er_s","er_ss"); + // multi-valued + p = params("cat_s", "cat_ss", "cat_i", "cat_is", "xy_s", "xy_ss", "num_d", "num_d", "qw_s", "qw_ss", "er_s","er_ss"); doBasicRefinement( p ); // single valued docvalues - p = params("cat_s", "cat_sd", "xy_s", "xy_sd", "num_d", "num_dd", "qw_s", "qw_sd", "er_s","er_sd"); + p = params("cat_s", "cat_sd", "cat_i", "cat_id", "xy_s", "xy_sd", "num_d", "num_dd", "qw_s", "qw_sd", "er_s","er_sd"); doBasicRefinement( p ); } @@ -262,21 +262,22 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS { client.deleteByQuery("*:*", null); String cat_s = p.get("cat_s"); + String cat_i = p.get("cat_i"); // just like cat_s, but a number String xy_s = p.get("xy_s"); String qw_s = p.get("qw_s"); String er_s = p.get("er_s"); // this field is designed to test numBuckets refinement... the first phase will only have a single bucket returned for the top count bucket of cat_s String num_d = p.get("num_d"); - clients.get(0).add( sdoc("id", "01", "all_s","all", cat_s, "A", xy_s, "X" ,num_d, -1, qw_s, "Q", er_s,"E") ); // A wins count tie - clients.get(0).add( sdoc("id", "02", "all_s","all", cat_s, "B", xy_s, "Y", num_d, 3 ) ); + clients.get(0).add( sdoc("id", "01", "all_s","all", cat_s, "A", cat_i,1, xy_s, "X" ,num_d, -1, qw_s, "Q", er_s,"E") ); // A wins count tie + clients.get(0).add( sdoc("id", "02", "all_s","all", cat_s, "B", cat_i,2, xy_s, "Y", num_d, 3 ) ); - clients.get(1).add( sdoc("id", "11", "all_s","all", cat_s, "B", xy_s, "X", num_d, -5 , er_s,"E") ); // B highest count - clients.get(1).add( sdoc("id", "12", "all_s","all", cat_s, "B", xy_s, "Y", num_d, -11, qw_s, "W" ) ); - clients.get(1).add( sdoc("id", "13", "all_s","all", cat_s, "A", xy_s, "X", num_d, 7 , er_s,"R") ); // "R" will only be picked up via refinement when parent facet is cat_s + clients.get(1).add( sdoc("id", "11", "all_s","all", cat_s, "B", cat_i,2, xy_s, "X", num_d, -5 , er_s,"E") ); // B highest count + clients.get(1).add( sdoc("id", "12", "all_s","all", cat_s, "B", cat_i,2, xy_s, "Y", num_d, -11, qw_s, "W" ) ); + clients.get(1).add( sdoc("id", "13", "all_s","all", cat_s, "A", cat_i,1, xy_s, "X", num_d, 7 , er_s,"R") ); // "R" will only be picked up via refinement when parent facet is cat_s - clients.get(2).add( sdoc("id", "21", "all_s","all", cat_s, "A", xy_s, "X", num_d, 17, qw_s, "W", er_s,"E") ); // A highest count - clients.get(2).add( sdoc("id", "22", "all_s","all", cat_s, "A", xy_s, "Y", num_d, -19 ) ); - clients.get(2).add( sdoc("id", "23", "all_s","all", cat_s, "B", xy_s, "X", num_d, 11 ) ); + clients.get(2).add( sdoc("id", "21", "all_s","all", cat_s, "A", cat_i,1, xy_s, "X", num_d, 17, qw_s, "W", er_s,"E") ); // A highest count + clients.get(2).add( sdoc("id", "22", "all_s","all", cat_s, "A", cat_i,1, xy_s, "Y", num_d, -19 ) ); + clients.get(2).add( sdoc("id", "23", "all_s","all", cat_s, "B", cat_i,2, xy_s, "X", num_d, 11 ) ); client.commit(); @@ -315,6 +316,17 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS { "}" ); + // same as above, but with an integer field instead of a string + client.testJQ(params(p, "q", "*:*", + "json.facet", "{" + + "cat0:{${terms} type:terms, field:${cat_i}, sort:'count desc', limit:1, overrequest:0, refine:true}" + + "}" + ) + , "facets=={ count:8" + + ", cat0:{ buckets:[ {val:1,count:4} ] }" + // w/o overrequest, we need refining to get the correct count. + "}" + ); + // basic refining test through/under a query facet client.testJQ(params(p, "q", "*:*", "json.facet", "{" + diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java index 03cc480b1b0..2f73b50dba1 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java @@ -507,6 +507,12 @@ public class TestJsonFacets extends SolrTestCaseHS { if (terms_method != null) { terms=terms+terms_method; } + String refine_method = p.get("refine_method"); + if (refine_method == null && random().nextBoolean()) { + refine_method = "refine:true,"; + } + if (refine_method != null) terms = terms + refine_method; + p.set("terms", terms); // "${terms}" should be put at the beginning of generic terms facets. // It may specify "method=..." or "limit:-1", so should not be used if the facet explicitly specifies. diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java index c5bd56bcb97..fb53e8464b0 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java @@ -477,6 +477,9 @@ public class FacetStream extends TupleStream implements Expressible { for(int b=0; b Date: Tue, 13 Jun 2017 22:44:04 -0400 Subject: [PATCH 077/131] LUCENE-7876 avoid leafReader.fields --- lucene/CHANGES.txt | 4 ++ .../index/TestBackwardsCompatibility.java | 2 +- .../lucene/index/BufferedUpdatesStream.java | 8 ++-- .../apache/lucene/search/TermInSetQuery.java | 4 +- .../lucene50/TestBlockPostingsFormat.java | 2 +- .../org/apache/lucene/index/Test2BDocs.java | 2 +- .../test/org/apache/lucene/index/TestDoc.java | 10 ++-- .../org/apache/lucene/index/TestFlex.java | 13 +++-- .../lucene/index/TestIndexReaderClose.java | 8 +--- .../apache/lucene/index/TestIndexWriter.java | 4 +- .../lucene/index/TestIndexWriterUnicode.java | 2 +- .../org/apache/lucene/index/TestPayloads.java | 4 +- .../lucene/index/TestPerSegmentDeletes.java | 4 +- .../lucene/index/TestPostingsOffsets.java | 2 +- .../lucene/index/TestSegmentTermDocs.java | 2 +- .../lucene/index/TestSegmentTermEnum.java | 10 ++-- .../lucene/index/TestStressAdvance.java | 11 +++-- .../apache/lucene/index/TestTermsEnum.java | 23 ++++++--- .../lucene/search/TestMultiPhraseQuery.java | 2 +- .../lucene/search/TestPhrasePrefixQuery.java | 2 +- .../search/TestSameScoresWithThreads.java | 2 +- .../lucene/index/memory/TestMemoryIndex.java | 2 +- .../memory/TestMemoryIndexAgainstRAMDir.java | 5 +- .../lucene/queries/CommonTermsQuery.java | 4 +- .../function/valuesource/TFValueSource.java | 6 +-- .../valuesource/TermFreqValueSource.java | 6 +-- .../TestIDVersionPostingsFormat.java | 4 +- .../ThreadedIndexingAndSearchingTestCase.java | 12 +++-- .../handler/admin/LukeRequestHandler.java | 10 ++-- .../component/QueryElevationComponent.java | 5 +- .../handler/component/TermsComponent.java | 48 +++++++++++-------- .../highlight/DefaultSolrHighlighter.java | 2 +- .../org/apache/solr/request/SimpleFacets.java | 4 +- .../org/apache/solr/search/DocSetUtil.java | 4 +- .../java/org/apache/solr/search/Filter.java | 2 +- .../solr/search/FloatPayloadValueSource.java | 4 +- .../solr/search/GraphTermsQParserPlugin.java | 16 +++---- .../apache/solr/search/JoinQParserPlugin.java | 12 ++--- .../FacetFieldProcessorByEnumTermsStream.java | 4 +- .../apache/solr/uninverting/DocTermOrds.java | 4 +- .../apache/solr/update/SolrIndexSplitter.java | 4 +- 41 files changed, 141 insertions(+), 138 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 05dc0f83d1d..2c49c9606ff 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -145,6 +145,10 @@ Other from methods that don't declare them ("sneaky throw" hack). (Robert Muir, Uwe Schindler, Dawid Weiss) +* LUCENE-7876: Avoid calls to LeafReader.fields() and MultiFields.getFields() + that are trivially replaced by LeafReader.terms() and MultiFields.getTerms() + (David Smiley) + Improvements * LUCENE-7841: Normalize ґ to г in Ukrainian analyzer. (Andriy Rysin via Dawid Weiss) diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index 71e82aeb8d4..1250a30752f 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -1192,7 +1192,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase { for (String name : oldNames) { Directory dir = oldIndexDirs.get(name); IndexReader r = DirectoryReader.open(dir); - TermsEnum terms = MultiFields.getFields(r).terms("content").iterator(); + TermsEnum terms = MultiFields.getTerms(r, "content").iterator(); BytesRef t = terms.next(); assertNotNull(t); diff --git a/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java b/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java index 51933ac76d9..9955626d594 100644 --- a/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java +++ b/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java @@ -501,9 +501,8 @@ class BufferedUpdatesStream implements Accountable { queue = new SegmentQueue(numReaders); long segTermCount = 0; - for(int i=0;i updates, SegmentState segState, DocValuesFieldUpdates.Container dvUpdatesContainer) throws IOException { - Fields fields = segState.reader.fields(); // TODO: we can process the updates per DV field, from last to first so that // if multiple terms affect same document for the same field, we add an update @@ -651,7 +649,7 @@ class BufferedUpdatesStream implements Accountable { // if we change the code to process updates in terms order, enable this assert // assert currentField == null || currentField.compareTo(term.field()) < 0; currentField = term.field(); - Terms terms = fields.terms(currentField); + Terms terms = segState.reader.terms(currentField); if (terms != null) { termsEnum = terms.iterator(); } else { diff --git a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java index 08fe3c3a485..9b64d379174 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java @@ -26,7 +26,6 @@ import java.util.Objects; import java.util.Set; import java.util.SortedSet; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -224,8 +223,7 @@ public class TermInSetQuery extends Query implements Accountable { private WeightOrDocIdSet rewrite(LeafReaderContext context) throws IOException { final LeafReader reader = context.reader(); - final Fields fields = reader.fields(); - Terms terms = fields.terms(field); + Terms terms = reader.terms(field); if (terms == null) { return null; } diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat.java index 1a2ab627c72..f2ed86cf271 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat.java @@ -55,7 +55,7 @@ public class TestBlockPostingsFormat extends BasePostingsFormatTestCase { DirectoryReader r = DirectoryReader.open(w); assertEquals(1, r.leaves().size()); - FieldReader field = (FieldReader) r.leaves().get(0).reader().fields().terms("field"); + FieldReader field = (FieldReader) r.leaves().get(0).reader().terms("field"); // We should see exactly two blocks: one root block (prefix empty string) and one block for z* terms (prefix z): Stats stats = field.getStats(); assertEquals(0, stats.floorBlockCount); diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BDocs.java b/lucene/core/src/test/org/apache/lucene/index/Test2BDocs.java index 4fab45a1a47..dbbe2717927 100644 --- a/lucene/core/src/test/org/apache/lucene/index/Test2BDocs.java +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BDocs.java @@ -91,7 +91,7 @@ public class Test2BDocs extends LuceneTestCase { LeafReader reader = context.reader(); int lim = context.reader().maxDoc(); - Terms terms = reader.fields().terms("f1"); + Terms terms = reader.terms("f1"); for (int i=0; i<10000; i++) { TermsEnum te = terms.iterator(); assertTrue( te.seekExact(term) ); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java index a20dece4e22..37761d3f681 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java @@ -249,14 +249,16 @@ public class TestDoc extends LuceneTestCase { for (int i = 0; i < reader.numDocs(); i++) out.println(reader.document(i)); - Fields fields = reader.fields(); - for (String field : fields) { - Terms terms = fields.terms(field); + for (FieldInfo fieldInfo : reader.getFieldInfos()) { + if (fieldInfo.getIndexOptions() == IndexOptions.NONE) { + continue; + } + Terms terms = reader.terms(fieldInfo.name); assertNotNull(terms); TermsEnum tis = terms.iterator(); while(tis.next() != null) { - out.print(" term=" + field + ":" + tis.term()); + out.print(" term=" + fieldInfo.name + ":" + tis.term()); out.println(" DF=" + tis.docFreq()); PostingsEnum positions = tis.postings(null, PostingsEnum.POSITIONS); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFlex.java b/lucene/core/src/test/org/apache/lucene/index/TestFlex.java index d91301fa6ac..662d034775c 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFlex.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFlex.java @@ -17,10 +17,13 @@ package org.apache.lucene.index; -import org.apache.lucene.store.*; -import org.apache.lucene.analysis.*; -import org.apache.lucene.document.*; -import org.apache.lucene.util.*; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; public class TestFlex extends LuceneTestCase { @@ -70,7 +73,7 @@ public class TestFlex extends LuceneTestCase { w.addDocument(doc); w.forceMerge(1); DirectoryReader r = w.getReader(); - TermsEnum terms = getOnlyLeafReader(r).fields().terms("f").iterator(); + TermsEnum terms = getOnlyLeafReader(r).terms("f").iterator(); assertTrue(terms.next() != null); try { assertEquals(0, terms.ord()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java index b99666e7f40..a046023aaac 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java @@ -80,9 +80,7 @@ public class TestIndexReaderClose extends LuceneTestCase { reader.getReaderCacheHelper().addClosedListener(new FaultyListener()); } - IllegalStateException expected = expectThrows(IllegalStateException.class, () -> { - reader.close(); - }); + IllegalStateException expected = expectThrows(IllegalStateException.class, () -> reader.close()); if (throwOnClose) { assertEquals("BOOM!", expected.getMessage()); @@ -90,9 +88,7 @@ public class TestIndexReaderClose extends LuceneTestCase { assertEquals("GRRRRRRRRRRRR!", expected.getMessage()); } - expectThrows(AlreadyClosedException.class, () -> { - reader.fields(); - }); + expectThrows(AlreadyClosedException.class, () -> reader.terms("someField")); if (random().nextBoolean()) { reader.close(); // call it again diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index 67add26e9c9..6897f06ab08 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -694,7 +694,7 @@ public class TestIndexWriter extends LuceneTestCase { writer.close(); DirectoryReader reader = DirectoryReader.open(dir); LeafReader subreader = getOnlyLeafReader(reader); - TermsEnum te = subreader.fields().terms("").iterator(); + TermsEnum te = subreader.terms("").iterator(); assertEquals(new BytesRef("a"), te.next()); assertEquals(new BytesRef("b"), te.next()); assertEquals(new BytesRef("c"), te.next()); @@ -715,7 +715,7 @@ public class TestIndexWriter extends LuceneTestCase { writer.close(); DirectoryReader reader = DirectoryReader.open(dir); LeafReader subreader = getOnlyLeafReader(reader); - TermsEnum te = subreader.fields().terms("").iterator(); + TermsEnum te = subreader.terms("").iterator(); assertEquals(new BytesRef(""), te.next()); assertEquals(new BytesRef("a"), te.next()); assertEquals(new BytesRef("b"), te.next()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java index f45cf210dba..bb55f8262c3 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java @@ -136,7 +136,7 @@ public class TestIndexWriterUnicode extends LuceneTestCase { } private void checkTermsOrder(IndexReader r, Set allTerms, boolean isTop) throws IOException { - TermsEnum terms = MultiFields.getFields(r).terms("f").iterator(); + TermsEnum terms = MultiFields.getTerms(r, "f").iterator(); BytesRefBuilder last = new BytesRefBuilder(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java b/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java index e893d8786a1..33f044db785 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java @@ -479,7 +479,7 @@ public class TestPayloads extends LuceneTestCase { } writer.close(); IndexReader reader = DirectoryReader.open(dir); - TermsEnum terms = MultiFields.getFields(reader).terms(field).iterator(); + TermsEnum terms = MultiFields.getTerms(reader, field).iterator(); PostingsEnum tp = null; while (terms.next() != null) { String termText = terms.term().utf8ToString(); @@ -602,7 +602,7 @@ public class TestPayloads extends LuceneTestCase { field.setTokenStream(ts); writer.addDocument(doc); DirectoryReader reader = writer.getReader(); - TermsEnum te = MultiFields.getFields(reader).terms("field").iterator(); + TermsEnum te = MultiFields.getTerms(reader, "field").iterator(); assertTrue(te.seekExact(new BytesRef("withPayload"))); PostingsEnum de = te.postings(null, PostingsEnum.PAYLOADS); de.nextDoc(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java index 58ef89082d6..112a1084917 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java @@ -221,9 +221,7 @@ public class TestPerSegmentDeletes extends LuceneTestCase { public int[] toDocsArray(Term term, Bits bits, IndexReader reader) throws IOException { - Fields fields = MultiFields.getFields(reader); - Terms cterms = fields.terms(term.field); - TermsEnum ctermsEnum = cterms.iterator(); + TermsEnum ctermsEnum = MultiFields.getTerms(reader, term.field).iterator(); if (ctermsEnum.seekExact(new BytesRef(term.text()))) { PostingsEnum postingsEnum = TestUtil.docs(random(), ctermsEnum, null, PostingsEnum.NONE); return toArray(postingsEnum); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java index c265d928665..600ce76a4da 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java @@ -291,7 +291,7 @@ public class TestPostingsOffsets extends LuceneTestCase { // TODO: improve this LeafReader sub = ctx.reader(); //System.out.println("\nsub=" + sub); - final TermsEnum termsEnum = sub.fields().terms("content").iterator(); + final TermsEnum termsEnum = sub.terms("content").iterator(); PostingsEnum docs = null; PostingsEnum docsAndPositions = null; PostingsEnum docsAndPositionsAndOffsets = null; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java index f1271b8aa55..279e3acfe0d 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java @@ -57,7 +57,7 @@ public class TestSegmentTermDocs extends LuceneTestCase { SegmentReader reader = new SegmentReader(info, Version.LATEST.major, newIOContext(random())); assertTrue(reader != null); - TermsEnum terms = reader.fields().terms(DocHelper.TEXT_FIELD_2_KEY).iterator(); + TermsEnum terms = reader.terms(DocHelper.TEXT_FIELD_2_KEY).iterator(); terms.seekCeil(new BytesRef("field")); PostingsEnum termDocs = TestUtil.docs(random(), terms, null, PostingsEnum.FREQS); if (termDocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java index 1e85e14c9de..e382bb685d9 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java @@ -19,14 +19,14 @@ package org.apache.lucene.index; import java.io.IOException; -import org.apache.lucene.document.Field; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.TestUtil; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; public class TestSegmentTermEnum extends LuceneTestCase { @@ -80,7 +80,7 @@ public class TestSegmentTermEnum extends LuceneTestCase { addDoc(writer, "aaa bbb"); writer.close(); LeafReader reader = getOnlyLeafReader(DirectoryReader.open(dir)); - TermsEnum terms = reader.fields().terms("content").iterator(); + TermsEnum terms = reader.terms("content").iterator(); assertNotNull(terms.next()); assertEquals("aaa", terms.term().utf8ToString()); assertNotNull(terms.next()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java b/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java index 606a11aff65..b984314aa7b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java @@ -21,10 +21,13 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.lucene.util.*; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.store.*; -import org.apache.lucene.document.*; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; public class TestStressAdvance extends LuceneTestCase { @@ -74,7 +77,7 @@ public class TestStressAdvance extends LuceneTestCase { bDocIDs.add(docID); } } - final TermsEnum te = getOnlyLeafReader(r).fields().terms("field").iterator(); + final TermsEnum te = getOnlyLeafReader(r).terms("field").iterator(); PostingsEnum de = null; for(int iter2=0;iter2<10;iter2++) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java index d2df59ff4fd..fcc043c76bf 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java @@ -18,7 +18,17 @@ package org.apache.lucene.index; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeSet; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -738,7 +748,7 @@ public class TestTermsEnum extends LuceneTestCase { DirectoryReader r = w.getReader(); w.close(); LeafReader sub = getOnlyLeafReader(r); - Terms terms = sub.fields().terms("field"); + Terms terms = sub.terms("field"); Automaton automaton = new RegExp(".*", RegExp.NONE).toAutomaton(); CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false); TermsEnum te = terms.intersect(ca, null); @@ -792,7 +802,7 @@ public class TestTermsEnum extends LuceneTestCase { DirectoryReader r = w.getReader(); w.close(); LeafReader sub = getOnlyLeafReader(r); - Terms terms = sub.fields().terms("field"); + Terms terms = sub.terms("field"); Automaton automaton = new RegExp(".*d", RegExp.NONE).toAutomaton(); CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false); @@ -846,7 +856,7 @@ public class TestTermsEnum extends LuceneTestCase { DirectoryReader r = w.getReader(); w.close(); LeafReader sub = getOnlyLeafReader(r); - Terms terms = sub.fields().terms("field"); + Terms terms = sub.terms("field"); Automaton automaton = new RegExp(".*", RegExp.NONE).toAutomaton(); // accept ALL CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false); @@ -986,7 +996,7 @@ public class TestTermsEnum extends LuceneTestCase { w.addDocument(doc); IndexReader r = w.getReader(); assertEquals(1, r.leaves().size()); - TermsEnum te = r.leaves().get(0).reader().fields().terms("field").iterator(); + TermsEnum te = r.leaves().get(0).reader().terms("field").iterator(); for(int i=0;i<=termCount;i++) { assertTrue("term '" + termsList.get(i).utf8ToString() + "' should exist but doesn't", te.seekExact(termsList.get(i))); } @@ -1007,9 +1017,8 @@ public class TestTermsEnum extends LuceneTestCase { doc.add(newStringField("field", "foobar", Field.Store.NO)); w.addDocument(doc); IndexReader r = w.getReader(); - Fields fields = MultiFields.getFields(r); + Terms terms = MultiFields.getTerms(r, "field"); CompiledAutomaton automaton = new CompiledAutomaton(new RegExp("do_not_match_anything").toAutomaton()); - Terms terms = fields.terms("field"); String message = expectThrows(IllegalArgumentException.class, () -> {terms.intersect(automaton, null);}).getMessage(); assertEquals("please use CompiledAutomaton.getTermsEnum instead", message); r.close(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java index 262cbf339d4..409299ff559 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java @@ -71,7 +71,7 @@ public class TestMultiPhraseQuery extends LuceneTestCase { // this TermEnum gives "piccadilly", "pie" and "pizza". String prefix = "pi"; - TermsEnum te = MultiFields.getFields(reader).terms("body").iterator(); + TermsEnum te = MultiFields.getTerms(reader,"body").iterator(); te.seekCeil(new BytesRef(prefix)); do { String s = te.term().utf8ToString(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java index 81511509377..114190c7206 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java @@ -73,7 +73,7 @@ public class TestPhrasePrefixQuery extends LuceneTestCase { // this TermEnum gives "piccadilly", "pie" and "pizza". String prefix = "pi"; - TermsEnum te = MultiFields.getFields(reader).terms("body").iterator(); + TermsEnum te = MultiFields.getTerms(reader, "body").iterator(); te.seekCeil(new BytesRef(prefix)); do { String s = te.term().utf8ToString(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java b/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java index 83a11be8b7a..88f92a5a503 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java @@ -61,7 +61,7 @@ public class TestSameScoresWithThreads extends LuceneTestCase { w.close(); final IndexSearcher s = newSearcher(r); - Terms terms = MultiFields.getFields(r).terms("body"); + Terms terms = MultiFields.getTerms(r, "body"); int termCount = 0; TermsEnum termsEnum = terms.iterator(); while(termsEnum.next() != null) { diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java index fa62262ddd4..d4ce8f1475f 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java @@ -131,7 +131,7 @@ public class TestMemoryIndex extends LuceneTestCase { mi.addField("field", "some terms be here", analyzer); IndexSearcher searcher = mi.createSearcher(); LeafReader reader = (LeafReader) searcher.getIndexReader(); - TermsEnum terms = reader.fields().terms("field").iterator(); + TermsEnum terms = reader.terms("field").iterator(); terms.seekExact(0); assertEquals("be", terms.term().utf8ToString()); TestUtil.checkReader(reader); diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java index 8a751b1cdb8..c911183c7d7 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java @@ -53,7 +53,6 @@ import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.*; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; @@ -67,8 +66,8 @@ import org.apache.lucene.search.spans.SpanOrQuery; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.ByteBlockPool.Allocator; import org.apache.lucene.util.ByteBlockPool; +import org.apache.lucene.util.ByteBlockPool.Allocator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LineFileDocs; @@ -171,7 +170,7 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase { private void duellReaders(CompositeReader other, LeafReader memIndexReader) throws IOException { - Fields memFields = memIndexReader.fields(); + Fields memFields = memIndexReader.getTermVectors(0); for (String field : MultiFields.getFields(other)) { Terms memTerms = memFields.terms(field); Terms iwTerms = memIndexReader.terms(field); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java index 857f4d3fdbc..2fdeaa797a9 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java @@ -22,7 +22,6 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; @@ -214,11 +213,10 @@ public class CommonTermsQuery extends Query { Term[] queryTerms) throws IOException { TermsEnum termsEnum = null; for (LeafReaderContext context : leaves) { - final Fields fields = context.reader().fields(); for (int i = 0; i < queryTerms.length; i++) { Term term = queryTerms[i]; TermContext termContext = contextArray[i]; - final Terms terms = fields.terms(term.field()); + final Terms terms = context.reader().terms(term.field()); if (terms == null) { // field does not exist continue; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java index f0e252c6980..baed0ffd410 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java @@ -19,9 +19,8 @@ package org.apache.lucene.queries.function.valuesource; import java.io.IOException; import java.util.Map; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queries.function.FunctionValues; @@ -50,8 +49,7 @@ public class TFValueSource extends TermFreqValueSource { @Override public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException { - Fields fields = readerContext.reader().fields(); - final Terms terms = fields.terms(indexedField); + final Terms terms = readerContext.reader().terms(indexedField); IndexSearcher searcher = (IndexSearcher)context.get("searcher"); final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(true), indexedField); if (similarity == null) { diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java index 789e3e9e6e3..55c54f5da30 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java @@ -19,9 +19,8 @@ package org.apache.lucene.queries.function.valuesource; import java.io.IOException; import java.util.Map; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queries.function.FunctionValues; @@ -48,8 +47,7 @@ public class TermFreqValueSource extends DocFreqValueSource { @Override public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException { - Fields fields = readerContext.reader().fields(); - final Terms terms = fields.terms(indexedField); + final Terms terms = readerContext.reader().terms(indexedField); return new IntDocValues(this) { PostingsEnum docs ; diff --git a/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java index d173762a636..0574e70d26c 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java +++ b/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java @@ -37,12 +37,12 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.ConcurrentMergeScheduler; -import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.index.PerThreadPKLookup; +import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.TieredMergePolicy; @@ -75,7 +75,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase { doc.add(makeIDField("id1", 110)); w.addDocument(doc); IndexReader r = w.getReader(); - IDVersionSegmentTermsEnum termsEnum = (IDVersionSegmentTermsEnum) r.leaves().get(0).reader().fields().terms("id").iterator(); + IDVersionSegmentTermsEnum termsEnum = (IDVersionSegmentTermsEnum) r.leaves().get(0).reader().terms("id").iterator(); assertTrue(termsEnum.seekExact(new BytesRef("id0"), 50)); assertTrue(termsEnum.seekExact(new BytesRef("id0"), 100)); assertFalse(termsEnum.seekExact(new BytesRef("id0"), 101)); diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java index ac94cf7ca14..fa9d51081b3 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java @@ -18,7 +18,14 @@ package org.apache.lucene.index; import java.io.IOException; import java.nio.file.Path; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.WeakHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -349,8 +356,7 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas } if (s.getIndexReader().numDocs() > 0) { smokeTestSearcher(s); - Fields fields = MultiFields.getFields(s.getIndexReader()); - Terms terms = fields.terms("body"); + Terms terms = MultiFields.getTerms(s.getIndexReader(), "body"); if (terms == null) { continue; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java index 9864016a54f..20fb71a6885 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java @@ -16,10 +16,6 @@ */ package org.apache.solr.handler.admin; -import static org.apache.lucene.index.IndexOptions.DOCS; -import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS; -import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; - import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -83,6 +79,10 @@ import org.apache.solr.update.SolrIndexWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.lucene.index.IndexOptions.DOCS; +import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS; +import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; + /** * This handler exposes the internal lucene index. It is inspired by and * modeled on Luke, the Lucene Index Browser by Andrzej Bialecki. @@ -366,7 +366,7 @@ public class LukeRequestHandler extends RequestHandlerBase if (sfield != null && schema.isDynamicField(sfield.getName()) && schema.getDynamicPattern(sfield.getName()) != null) { fieldMap.add("dynamicBase", schema.getDynamicPattern(sfield.getName())); } - Terms terms = reader.fields().terms(fieldName); + Terms terms = reader.terms(fieldName); if (terms == null) { // Not indexed, so we need to report what we can (it made it through the fl param if specified) finfo.add( fieldName, fieldMap ); continue; diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java index 4fe1d716324..3f3dd5c99b8 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java @@ -38,7 +38,6 @@ import com.carrotsearch.hppc.IntIntHashMap; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -665,9 +664,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore protected void doSetNextReader(LeafReaderContext context) throws IOException { //convert the ids to Lucene doc ids, the ordSet and termValues needs to be the same size as the number of elevation docs we have ordSet.clear(); - Fields fields = context.reader().fields(); - if (fields == null) return; - Terms terms = fields.terms(idField); + Terms terms = context.reader().terms(idField); if (terms == null) return; TermsEnum termsEnum = terms.iterator(); BytesRefBuilder term = new BytesRefBuilder(); diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java index 8a735d13f5a..80dfa40932f 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java @@ -15,22 +15,6 @@ * limitations under the License. */ package org.apache.solr.handler.component; -import org.apache.lucene.index.*; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.CharsRefBuilder; -import org.apache.lucene.util.StringHelper; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.*; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.schema.FieldType; -import org.apache.solr.schema.StrField; -import org.apache.solr.request.SimpleFacets.CountPair; -import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.util.BoundedTreeSet; -import org.apache.solr.client.solrj.response.TermsResponse; import java.io.IOException; import java.util.Arrays; @@ -39,6 +23,32 @@ import java.util.List; import java.util.Locale; import java.util.regex.Pattern; +import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.CharsRefBuilder; +import org.apache.lucene.util.StringHelper; +import org.apache.solr.client.solrj.response.TermsResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.TermsParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.request.SimpleFacets.CountPair; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.StrField; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.BoundedTreeSet; + /** * Return TermEnum information, useful for things like auto suggest. * @@ -138,13 +148,12 @@ public class TermsComponent extends SearchComponent { final LeafReader indexReader = rb.req.getSearcher().getSlowAtomicReader(); - Fields lfields = indexReader.fields(); for (String field : fields) { NamedList fieldTerms = new NamedList<>(); termsResult.add(field, fieldTerms); - Terms terms = lfields.terms(field); + Terms terms = indexReader.terms(field); if (terms == null) { // field does not exist continue; @@ -562,10 +571,9 @@ public class TermsComponent extends SearchComponent { Term[] queryTerms) throws IOException { TermsEnum termsEnum = null; for (LeafReaderContext context : topReaderContext.leaves()) { - final Fields fields = context.reader().fields(); for (int i = 0; i < queryTerms.length; i++) { Term term = queryTerms[i]; - final Terms terms = fields.terms(term.field()); + final Terms terms = context.reader().terms(term.field()); if (terms == null) { // field does not exist continue; diff --git a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java index 18d9b446121..1029c7dd7e3 100644 --- a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java @@ -245,7 +245,7 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf try { // It'd be nice to know if payloads are on the tokenStream but the presence of the attribute isn't a good // indicator. - final Terms terms = request.getSearcher().getSlowAtomicReader().fields().terms(fieldName); + final Terms terms = request.getSearcher().getSlowAtomicReader().terms(fieldName); if (terms != null) { defaultPayloads = terms.hasPayloads(); } diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java index 406a4ed4c8c..f39eda47360 100644 --- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java +++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java @@ -37,7 +37,6 @@ import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiPostingsEnum; @@ -935,8 +934,7 @@ public class SimpleFacets { prefixTermBytes = new BytesRef(indexedPrefix); } - Fields fields = r.fields(); - Terms terms = fields==null ? null : fields.terms(field); + Terms terms = r.terms(field); TermsEnum termsEnum = null; SolrIndexSearcher.DocsEnumState deState = null; BytesRef term = null; diff --git a/solr/core/src/java/org/apache/solr/search/DocSetUtil.java b/solr/core/src/java/org/apache/solr/search/DocSetUtil.java index a7c9bef857a..0bc6dac23b1 100644 --- a/solr/core/src/java/org/apache/solr/search/DocSetUtil.java +++ b/solr/core/src/java/org/apache/solr/search/DocSetUtil.java @@ -21,7 +21,6 @@ import java.util.Iterator; import java.util.List; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -169,8 +168,7 @@ public class DocSetUtil { for (LeafReaderContext ctx : leaves) { assert leaves.get(ctx.ord) == ctx; LeafReader r = ctx.reader(); - Fields f = r.fields(); - Terms t = f.terms(field); + Terms t = r.terms(field); if (t == null) continue; // field is missing TermsEnum te = t.iterator(); if (te.seekExact(termVal)) { diff --git a/solr/core/src/java/org/apache/solr/search/Filter.java b/solr/core/src/java/org/apache/solr/search/Filter.java index f4a148c6f0c..e37d573af4d 100644 --- a/solr/core/src/java/org/apache/solr/search/Filter.java +++ b/solr/core/src/java/org/apache/solr/search/Filter.java @@ -69,7 +69,7 @@ public abstract class Filter extends Query { * represent the whole underlying index i.e. if the index has more than * one segment the given reader only represents a single segment. * The provided context is always an atomic context, so you can call - * {@link org.apache.lucene.index.LeafReader#fields()} + * {@link org.apache.lucene.index.LeafReader#terms(String)} * on the context's reader, for example. * * @param acceptDocs diff --git a/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java b/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java index 65b5fa01354..23e07e2c6de 100644 --- a/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java +++ b/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java @@ -20,7 +20,6 @@ package org.apache.solr.search; import java.io.IOException; import java.util.Map; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; @@ -56,8 +55,7 @@ public class FloatPayloadValueSource extends ValueSource { @Override public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException { - Fields fields = readerContext.reader().fields(); - final Terms terms = fields.terms(indexedField); + final Terms terms = readerContext.reader().terms(indexedField); FunctionValues defaultValues = defaultValueSource.getValues(context, readerContext); diff --git a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java index a8a86cb8783..d659265b1e4 100644 --- a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java @@ -18,15 +18,19 @@ package org.apache.solr.search; -import org.apache.lucene.index.Fields; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermContext; -import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.ConstantScoreQuery; @@ -49,11 +53,6 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.FieldType; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - /** * The GraphTermsQuery builds a disjunction query from a list of terms. The terms are first filtered by the maxDocFreq parameter. * This allows graph traversals to skip traversing high frequency nodes which is often desirable from a performance standpoint. @@ -220,8 +219,7 @@ public class GraphTermsQParserPlugin extends QParserPlugin { private WeightOrDocIdSet rewrite(LeafReaderContext context) throws IOException { final LeafReader reader = context.reader(); - final Fields fields = reader.fields(); - Terms terms = fields.terms(field); + Terms terms = reader.terms(field); if(terms == null) { return new WeightOrDocIdSet(new BitDocIdSet(new FixedBitSet(reader.maxDoc()), 0)); } diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index abdea534685..e4e92cfec62 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -23,8 +23,8 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiPostingsEnum; import org.apache.lucene.index.PostingsEnum; @@ -316,11 +316,11 @@ class JoinQuery extends Query { fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size()); } - Fields fromFields = fromSearcher.getSlowAtomicReader().fields(); - Fields toFields = fromSearcher==toSearcher ? fromFields : toSearcher.getSlowAtomicReader().fields(); - if (fromFields == null) return DocSet.EMPTY; - Terms terms = fromFields.terms(fromField); - Terms toTerms = toFields.terms(toField); + + LeafReader fromReader = fromSearcher.getSlowAtomicReader(); + LeafReader toReader = fromSearcher==toSearcher ? fromReader : toSearcher.getSlowAtomicReader(); + Terms terms = fromReader.terms(fromField); + Terms toTerms = toReader.terms(toField); if (terms == null || toTerms==null) return DocSet.EMPTY; String prefixStr = TrieField.getMainValuePrefix(fromSearcher.getSchema().getFieldType(fromField)); BytesRef prefix = prefixStr == null ? null : new BytesRef(prefixStr); diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java index d28e024c7f5..bbc2973c8af 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.Iterator; import java.util.List; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiPostingsEnum; import org.apache.lucene.index.PostingsEnum; @@ -156,8 +155,7 @@ class FacetFieldProcessorByEnumTermsStream extends FacetFieldProcessor implement } } - Fields fields = fcontext.searcher.getSlowAtomicReader().fields(); - Terms terms = fields == null ? null : fields.terms(sf.getName()); + Terms terms = fcontext.searcher.getSlowAtomicReader().terms(sf.getName()); termsEnum = null; deState = null; diff --git a/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java b/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java index fcc6974f480..07801466f68 100644 --- a/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java +++ b/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java @@ -22,7 +22,7 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.lucene.codecs.PostingsFormat; // javadocs +import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; @@ -595,7 +595,7 @@ public class DocTermOrds implements Accountable { public OrdWrappedTermsEnum(LeafReader reader) throws IOException { assert indexedTermsArray != null; assert 0 != indexedTermsArray.length; - termsEnum = reader.fields().terms(field).iterator(); + termsEnum = reader.terms(field).iterator(); } @Override diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java index e9950f2b42c..aadbe7433c8 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.List; import org.apache.lucene.index.CodecReader; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.FilterCodecReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReader; @@ -165,8 +164,7 @@ public class SolrIndexSplitter { } Bits liveDocs = reader.getLiveDocs(); - Fields fields = reader.fields(); - Terms terms = fields==null ? null : fields.terms(field.getName()); + Terms terms = reader.terms(field.getName()); TermsEnum termsEnum = terms==null ? null : terms.iterator(); if (termsEnum == null) return docSets; From 92b17838a346ad55a6a4ab796b8ab8cbbe4ffea2 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Wed, 14 Jun 2017 18:07:40 +0930 Subject: [PATCH 078/131] SOLR-10876: Regression in loading runtime UpdateRequestProcessors like TemplateUpdateProcessorFactory --- solr/CHANGES.txt | 2 + .../handler/dataimport/VariableResolver.java | 4 +- .../java/org/apache/solr/core/PluginBag.java | 32 ++++++--- .../TemplateUpdateProcessorFactory.java | 18 ++--- .../UpdateRequestProcessorChain.java | 15 ++--- .../configsets/cloud-minimal/conf/schema.xml | 1 + .../TemplateUpdateProcessorTest.java | 67 +++++++++++++++++-- .../src/update-request-processors.adoc | 2 +- .../cloud/AbstractFullDistribZkTestBase.java | 2 +- 9 files changed, 106 insertions(+), 37 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a07b68f1b00..4c35991ec42 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -159,6 +159,8 @@ Bug Fixes * SOLR-10830: Solr now correctly enforces that the '_root_' field has the same fieldType as the uniqueKey field. With out this enforcement, child document updating was unreliable. (hossman) +* SOLR-10876: Regression in loading runtime UpdateRequestProcessors like TemplateUpdateProcessorFactory (noble) + Optimizations ---------------------- diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java index 8fced583e2c..090e21bce7c 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java @@ -143,7 +143,7 @@ public class VariableResolver { * @return the string with the placeholders replaced with their values */ public String replaceTokens(String template) { - return TemplateUpdateProcessorFactory.replaceTokens(template, cache, fun); + return TemplateUpdateProcessorFactory.replaceTokens(template, cache, fun, TemplateUpdateProcessorFactory.DOLLAR_BRACES_PLACEHOLDER_PATTERN); } public void addNamespace(String name, Map newMap) { if (newMap != null) { @@ -164,7 +164,7 @@ public class VariableResolver { } public List getVariables(String expr) { - return TemplateUpdateProcessorFactory.getVariables(expr, cache); + return TemplateUpdateProcessorFactory.getVariables(expr, cache, TemplateUpdateProcessorFactory.DOLLAR_BRACES_PLACEHOLDER_PATTERN); } static class CurrentLevel { diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java index e03fc062b7e..b0c68f22e24 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginBag.java +++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java @@ -35,6 +35,9 @@ import java.util.zip.ZipInputStream; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.analysis.util.ResourceLoaderAware; +import org.apache.solr.api.Api; +import org.apache.solr.api.ApiBag; +import org.apache.solr.api.ApiSupport; import org.apache.solr.cloud.CloudUtil; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.StrUtils; @@ -46,15 +49,12 @@ import org.apache.solr.util.SimplePostTool; import org.apache.solr.util.plugin.NamedListInitializedPlugin; import org.apache.solr.util.plugin.PluginInfoInitialized; import org.apache.solr.util.plugin.SolrCoreAware; -import org.apache.solr.api.Api; -import org.apache.solr.api.ApiBag; -import org.apache.solr.api.ApiSupport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static java.util.Collections.singletonMap; -import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.api.ApiBag.HANDLER_NAME; +import static org.apache.solr.common.params.CommonParams.NAME; /** * This manages the lifecycle of a set of plugin of the same type . @@ -125,10 +125,12 @@ public class PluginBag implements AutoCloseable { public PluginHolder createPlugin(PluginInfo info) { if ("true".equals(String.valueOf(info.attributes.get("runtimeLib")))) { log.debug(" {} : '{}' created with runtimeLib=true ", meta.getCleanTag(), info.name); - return new LazyPluginHolder<>(meta, info, core, core.getMemClassLoader()); + return new LazyPluginHolder(meta, info, core, "true".equals(System.getProperty("enable.runtime.lib")) ? + core.getMemClassLoader() : + core.getResourceLoader(), true); } else if ("lazy".equals(info.attributes.get("startup")) && meta.options.contains(SolrConfig.PluginOpts.LAZY)) { log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name); - return new LazyPluginHolder(meta, info, core, core.getResourceLoader()); + return new LazyPluginHolder(meta, info, core, core.getResourceLoader(), false); } else { T inst = core.createInstance(info.className, (Class) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader()); initInstance(inst, info); @@ -371,11 +373,13 @@ public class PluginBag implements AutoCloseable { protected SolrException solrException; private final SolrCore core; protected ResourceLoader resourceLoader; + private final boolean isRuntimeLib; - LazyPluginHolder(SolrConfig.SolrPluginInfo pluginMeta, PluginInfo pluginInfo, SolrCore core, ResourceLoader loader) { + LazyPluginHolder(SolrConfig.SolrPluginInfo pluginMeta, PluginInfo pluginInfo, SolrCore core, ResourceLoader loader, boolean isRuntimeLib) { super(pluginInfo); this.pluginMeta = pluginMeta; + this.isRuntimeLib = isRuntimeLib; this.core = core; this.resourceLoader = loader; if (loader instanceof MemClassLoader) { @@ -413,7 +417,19 @@ public class PluginBag implements AutoCloseable { loader.loadJars(); } Class clazz = (Class) pluginMeta.clazz; - T localInst = core.createInstance(pluginInfo.className, clazz, pluginMeta.getCleanTag(), null, resourceLoader); + T localInst = null; + try { + localInst = core.createInstance(pluginInfo.className, clazz, pluginMeta.getCleanTag(), null, resourceLoader); + } catch (SolrException e) { + if (isRuntimeLib && !(resourceLoader instanceof MemClassLoader)) { + throw new SolrException(SolrException.ErrorCode.getErrorCode(e.code()), + e.getMessage() + ". runtime library loading is not enabled, start Solr with -Denable.runtime.lib=true", + e.getCause()); + } + throw e; + + + } initInstance(localInst, pluginInfo); if (localInst instanceof SolrCoreAware) { SolrResourceLoader.assertAwareCompatibility(SolrCoreAware.class, localInst); diff --git a/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java index c16a0c7aa25..19c331f073d 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java @@ -57,18 +57,18 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory doc.addField(fName, replaceTokens(template, templateCache, s -> { Object v = doc.getFieldValue(s); return v == null ? "" : v; - })); + }, BRACES_PLACEHOLDER_PATTERN)); } } } - public static Resolved getResolved(String template, Cache cache) { + public static Resolved getResolved(String template, Cache cache, Pattern pattern) { Resolved r = cache == null ? null : cache.get(template); if (r == null) { r = new Resolved(); - Matcher m = PLACEHOLDER_PATTERN.matcher(template); + Matcher m = pattern.matcher(template); while (m.find()) { String variable = m.group(1); r.startIndexes.add(m.start(0)); @@ -83,19 +83,19 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory /** * Get a list of variables embedded in the template string. */ - public static List getVariables(String template, Cache cache) { - Resolved r = getResolved(template, cache); + public static List getVariables(String template, Cache cache, Pattern pattern) { + Resolved r = getResolved(template, cache, pattern ); if (r == null) { return Collections.emptyList(); } return new ArrayList<>(r.variables); } - public static String replaceTokens(String template, Cache cache, Function fun) { + public static String replaceTokens(String template, Cache cache, Function fun, Pattern pattern) { if (template == null) { return null; } - Resolved r = getResolved(template, cache); + Resolved r = getResolved(template, cache, pattern); if (r.startIndexes != null) { StringBuilder sb = new StringBuilder(template); for (int i = r.startIndexes.size() - 1; i >= 0; i--) { @@ -115,6 +115,8 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory public List variables = new ArrayList<>(2); } - public static final Pattern PLACEHOLDER_PATTERN = Pattern + public static final Pattern DOLLAR_BRACES_PLACEHOLDER_PATTERN = Pattern .compile("[$][{](.*?)[}]"); + public static final Pattern BRACES_PLACEHOLDER_PATTERN = Pattern + .compile("[{](.*?)[}]"); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java index 05d1a5ae3fc..3db42aa8837 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java +++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java @@ -29,7 +29,6 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; -import org.apache.solr.core.PluginBag; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; @@ -272,16 +271,12 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized if (s.isEmpty()) continue; UpdateRequestProcessorFactory p = core.getUpdateProcessors().get(s); if (p == null) { - try { - PluginInfo pluginInfo = new PluginInfo("updateProcessor", - Utils.makeMap("name", s, - "class", s + "UpdateProcessorFactory", - "runtimeLib", "true")); + PluginInfo pluginInfo = new PluginInfo("updateProcessor", + Utils.makeMap("name", s, + "class", s + "UpdateProcessorFactory", + "runtimeLib", "true")); - PluginBag.PluginHolder pluginHolder = core.getUpdateProcessors().createPlugin(pluginInfo); - core.getUpdateProcessors().put(s, p = pluginHolder.get()); - } catch (SolrException e) { - } + core.getUpdateProcessors().put(s, p = core.getUpdateProcessors().createPlugin(pluginInfo).get()); if (p == null) throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No such processor " + s); } diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml index 7b8b690d395..08a1716deb1 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml @@ -24,5 +24,6 @@ + id diff --git a/solr/core/src/test/org/apache/solr/update/processor/TemplateUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TemplateUpdateProcessorTest.java index 7ee8a34938b..e14521903d6 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/TemplateUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/TemplateUpdateProcessorTest.java @@ -17,22 +17,56 @@ package org.apache.solr.update.processor; -import org.apache.solr.SolrTestCaseJ4; +import java.lang.invoke.MethodHandles; + +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; +import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Utils; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.update.AddUpdateCommand; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.rules.ExpectedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TemplateUpdateProcessorTest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(5) + .addConfig("conf1", configset("cloud-minimal")) + .configure(); + } + + @After + public void after() throws Exception { + cluster.deleteAllCollections(); + cluster.shutdown(); + } + + @org.junit.Rule + public ExpectedException expectedException = ExpectedException.none(); + -public class TemplateUpdateProcessorTest extends SolrTestCaseJ4 { public void testSimple() throws Exception { + ModifiableSolrParams params = new ModifiableSolrParams() + .add("processor", "Template") + .add("Template.field", "id:{firstName}_{lastName}") + .add("Template.field", "another:{lastName}_{firstName}") + .add("Template.field", "missing:{lastName}_{unKnown}"); AddUpdateCommand cmd = new AddUpdateCommand(new LocalSolrQueryRequest(null, - new ModifiableSolrParams() - .add("processor", "Template") - .add("Template.field", "id:${firstName}_${lastName}") - .add("Template.field", "another:${lastName}_${firstName}") - .add("Template.field", "missing:${lastName}_${unKnown}") + params )); cmd.solrDoc = new SolrInputDocument(); @@ -44,5 +78,24 @@ public class TemplateUpdateProcessorTest extends SolrTestCaseJ4 { assertEquals("Cruise_Tom", cmd.solrDoc.getFieldValue("another")); assertEquals("Cruise_", cmd.solrDoc.getFieldValue("missing")); + SolrInputDocument solrDoc = new SolrInputDocument(); + solrDoc.addField("id", "1"); + + params = new ModifiableSolrParams() + .add("processor", "Template") + .add("commit", "true") + .add("Template.field", "x_s:key_{id}"); + params.add("commit", "true"); + UpdateRequest add = new UpdateRequest().add(solrDoc); + add.setParams(params); + NamedList result = cluster.getSolrClient().request(CollectionAdminRequest.createCollection("c", "conf1", 1, 1)); + Utils.toJSONString(result.asMap(4)); + AbstractFullDistribZkTestBase.waitForCollection(cluster.getSolrClient().getZkStateReader(), "c",1); + cluster.getSolrClient().request(add, "c"); + QueryResponse rsp = cluster.getSolrClient().query("c", + new ModifiableSolrParams().add("q","id:1")); + assertEquals( "key_1", rsp.getResults().get(0).getFieldValue("x_s")); + + } } diff --git a/solr/solr-ref-guide/src/update-request-processors.adoc b/solr/solr-ref-guide/src/update-request-processors.adoc index a4f3e0c0b90..d1f5c351e74 100644 --- a/solr/solr-ref-guide/src/update-request-processors.adoc +++ b/solr/solr-ref-guide/src/update-request-processors.adoc @@ -392,7 +392,7 @@ For example: [source,bash] ---- -processor=Template&Template.field=fullName:Mr. ${firstName} ${lastName} +processor=Template&Template.field=fullName:Mr. {firstName} {lastName} ---- The above example would add a new field to the document called `fullName`. The fields `firstName and` `lastName` are supplied from the document fields. If either of them is missing, that part is replaced with an empty string. If those fields are multi-valued, only the first value is used. diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index 5fa4af5812a..cf6f2e1e0f7 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -353,7 +353,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes } - protected static void waitForCollection(ZkStateReader reader, String collection, int slices) throws Exception { + public static void waitForCollection(ZkStateReader reader, String collection, int slices) throws Exception { // wait until shards have started registering... int cnt = 30; while (!reader.getClusterState().hasCollection(collection)) { From c3c895548f6334566c20f2396a33fdc8c031ab89 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Wed, 14 Jun 2017 18:56:43 +0930 Subject: [PATCH 079/131] SOLR-10876: Regression in loading runtime UpdateRequestProcessors like TemplateUpdateProcessorFactory --- solr/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 4c35991ec42..88821bb2ab4 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -92,6 +92,8 @@ Upgrading from Solr 6.x * Custom SolrMetricReporter classes must implement a new, factored out doInit() method and the validate() method must pass for all reporters including reporters that are not enabled, please see SOLR-10671 for details. +* TemplateUpdateRequestProcessorFactory uses {} instead of ${} for template + New Features ---------------------- * SOLR-9857, SOLR-9858: Collect aggregated metrics from nodes and shard leaders in overseer. (ab) From abc393dbfdb805361747ef651393332968851f3d Mon Sep 17 00:00:00 2001 From: David Smiley Date: Thu, 15 Jun 2017 00:59:01 -0400 Subject: [PATCH 080/131] LUCENE-7500: Remove LeafReader.fields in lieu of LeafReader.terms. Optimized MultiFields.getTerms. --- lucene/CHANGES.txt | 5 ++ .../org/apache/lucene/index/CodecReader.java | 11 +-- .../lucene/index/ExitableDirectoryReader.java | 40 ++--------- .../java/org/apache/lucene/index/Fields.java | 10 ++- .../apache/lucene/index/FilterLeafReader.java | 6 +- .../org/apache/lucene/index/LeafReader.java | 15 +--- .../lucene/index/MergeReaderWrapper.java | 7 +- .../org/apache/lucene/index/MultiFields.java | 69 +++++++++++++++++-- .../lucene/index/ParallelLeafReader.java | 33 ++++----- .../lucene/index/SlowCodecReaderWrapper.java | 16 +++-- .../lucene/index/SortingLeafReader.java | 6 +- .../index/TestExitableDirectoryReader.java | 16 +---- .../lucene/index/TestFilterLeafReader.java | 17 +---- .../lucene/index/TestParallelTermEnum.java | 20 ++---- .../lucene/search/TermInSetQueryTest.java | 24 +++---- .../apache/lucene/search/TestTermQuery.java | 43 +++++------- .../highlight/TermVectorLeafReader.java | 8 +-- .../highlight/WeightedSpanTermExtractor.java | 25 ++----- .../search/uhighlight/PhraseHelper.java | 29 +++----- .../TermVectorFilteredLeafReader.java | 22 +----- .../lucene/index/memory/MemoryIndex.java | 8 +-- .../lucene/index/AssertingLeafReader.java | 7 +- .../index/BaseIndexFileFormatTestCase.java | 2 +- .../index/BasePostingsFormatTestCase.java | 15 ++-- .../lucene/index/FieldFilterLeafReader.java | 9 ++- .../org/apache/lucene/search/QueryUtils.java | 26 ++----- .../index/SlowCompositeReaderWrapper.java | 4 +- .../org/apache/solr/search/TestDocSet.java | 5 +- 28 files changed, 214 insertions(+), 284 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 2c49c9606ff..96cfb3a5967 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -69,6 +69,11 @@ API Changes * LUCENE-7850: Removed support for legacy numerics. (Adrien Grand) +* LUCENE-7500: Removed abstract LeafReader.fields(); instead terms(fieldName) + has been made abstract, fomerly was final. Also, MultiFields.getTerms + was optimized to work directly instead of being implemented on getFields. + (David Smiley) + Bug Fixes * LUCENE-7626: IndexWriter will no longer accept broken token offsets diff --git a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java index 460a9d6cf34..9efcf253669 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java @@ -98,12 +98,15 @@ public abstract class CodecReader extends LeafReader implements Accountable { throw new IndexOutOfBoundsException("docID must be >= 0 and < maxDoc=" + maxDoc() + " (got docID=" + docID + ")"); } } - + @Override - public final Fields fields() { - return getPostingsReader(); + public final Terms terms(String field) throws IOException { + //ensureOpen(); no; getPostingsReader calls this + // We could check the FieldInfo IndexOptions but there's no point since + // PostingsReader will simply return null for fields that don't exist or that have no terms index. + return getPostingsReader().terms(field); } - + // returns the FieldInfo that corresponds to the given field and type, or // null if the field does not exist, or not indexed as the requested // DovDocValuesType. diff --git a/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java index a9b7472a5a8..fa1f6bae38e 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java @@ -17,14 +17,13 @@ package org.apache.lucene.index; -import org.apache.lucene.index.FilterLeafReader.FilterFields; +import java.io.IOException; + import org.apache.lucene.index.FilterLeafReader.FilterTerms; import org.apache.lucene.index.FilterLeafReader.FilterTermsEnum; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.CompiledAutomaton; -import java.io.IOException; - /** * The {@link ExitableDirectoryReader} wraps a real index {@link DirectoryReader} and @@ -79,14 +78,12 @@ public class ExitableDirectoryReader extends FilterDirectoryReader { } @Override - public Fields fields() throws IOException { - Fields fields = super.fields(); - if (queryTimeout.isTimeoutEnabled()) { - return new ExitableFields(fields, queryTimeout); - } - else { - return fields; // break out of wrapper as soon as possible + public Terms terms(String field) throws IOException { + Terms terms = in.terms(field); + if (terms == null) { + return null; } + return (queryTimeout.isTimeoutEnabled()) ? new ExitableTerms(terms, queryTimeout) : terms; } // this impl does not change deletes or data so we can delegate the @@ -103,29 +100,6 @@ public class ExitableDirectoryReader extends FilterDirectoryReader { } - /** - * Wrapper class for another Fields implementation that is used by the ExitableFilterAtomicReader. - */ - public static class ExitableFields extends FilterFields { - - private QueryTimeout queryTimeout; - - /** Constructor **/ - public ExitableFields(Fields fields, QueryTimeout queryTimeout) { - super(fields); - this.queryTimeout = queryTimeout; - } - - @Override - public Terms terms(String field) throws IOException { - Terms terms = in.terms(field); - if (terms == null) { - return null; - } - return new ExitableTerms(terms, queryTimeout); - } - } - /** * Wrapper class for another Terms implementation that is used by ExitableFields. */ diff --git a/lucene/core/src/java/org/apache/lucene/index/Fields.java b/lucene/core/src/java/org/apache/lucene/index/Fields.java index c5794b61113..d1bd9022f2b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/Fields.java +++ b/lucene/core/src/java/org/apache/lucene/index/Fields.java @@ -20,9 +20,15 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Iterator; -/** Flex API for access to fields and terms - * @lucene.experimental */ +import org.apache.lucene.codecs.FieldsProducer; +/** + * Provides a {@link Terms} index for fields that have it, and lists which fields do. + * This is primarily an internal/experimental API (see {@link FieldsProducer}), + * although it is also used to expose the set of term vectors per document. + * + * @lucene.experimental + */ public abstract class Fields implements Iterable { /** Sole constructor. (For invocation by subclass diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java index f3d8112ce33..04500384401 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java @@ -345,11 +345,11 @@ public abstract class FilterLeafReader extends LeafReader { protected void doClose() throws IOException { in.close(); } - + @Override - public Fields fields() throws IOException { + public Terms terms(String field) throws IOException { ensureOpen(); - return in.fields(); + return in.terms(field); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java index c738bc58683..faea32de288 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java @@ -18,7 +18,6 @@ package org.apache.lucene.index; import java.io.IOException; -import org.apache.lucene.index.IndexReader.CacheHelper; import org.apache.lucene.util.Bits; /** {@code LeafReader} is an abstract class, providing an interface for accessing an @@ -60,7 +59,7 @@ public abstract class LeafReader extends IndexReader { } /** - * Optional method: Return a {@link CacheHelper} that can be used to cache + * Optional method: Return a {@link IndexReader.CacheHelper} that can be used to cache * based on the content of this leaf regardless of deletions. Two readers * that have the same data but different sets of deleted documents or doc * values updates may be considered equal. Consider using @@ -73,12 +72,6 @@ public abstract class LeafReader extends IndexReader { */ public abstract CacheHelper getCoreCacheHelper(); - /** - * Returns {@link Fields} for this reader. - * This method will not return null. - */ - public abstract Fields fields() throws IOException; - @Override public final int docFreq(Term term) throws IOException { final Terms terms = terms(term.field()); @@ -139,10 +132,8 @@ public abstract class LeafReader extends IndexReader { return terms.getSumTotalTermFreq(); } - /** This may return null if the field does not exist.*/ - public final Terms terms(String field) throws IOException { - return fields().terms(field); - } + /** Returns the {@link Terms} index for this field, or null if it has none. */ + public abstract Terms terms(String field) throws IOException; /** Returns {@link PostingsEnum} for the specified term. * This will return null if either the field or diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java index 3a3573adb5b..565bdd45ff1 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java @@ -70,8 +70,11 @@ class MergeReaderWrapper extends LeafReader { } @Override - public Fields fields() throws IOException { - return fields; + public Terms terms(String field) throws IOException { + ensureOpen(); + // We could check the FieldInfo IndexOptions but there's no point since + // PostingsReader will simply return null for fields that don't exist or that have no terms index. + return fields.terms(field); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java index 1736bace115..1a7b15bd81c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -31,11 +32,12 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.MergedIterator; /** - * Exposes flex API, merged from flex API of sub-segments. + * Provides a single {@link Fields} term index view over an + * {@link IndexReader}. * This is useful when you're interacting with an {@link * IndexReader} implementation that consists of sequential * sub-readers (eg {@link DirectoryReader} or {@link - * MultiReader}). + * MultiReader}) and you must treat it as a {@link LeafReader}. * *

    NOTE: for composite readers, you'll get better * performance by gathering the sub readers using @@ -45,7 +47,6 @@ import org.apache.lucene.util.MergedIterator; * * @lucene.experimental */ - public final class MultiFields extends Fields { private final Fields[] subs; private final ReaderSlice[] subSlices; @@ -64,13 +65,13 @@ public final class MultiFields extends Fields { switch (leaves.size()) { case 1: // already an atomic reader / reader with one leave - return leaves.get(0).reader().fields(); + return new LeafReaderFields(leaves.get(0).reader()); default: final List fields = new ArrayList<>(leaves.size()); final List slices = new ArrayList<>(leaves.size()); for (final LeafReaderContext ctx : leaves) { final LeafReader r = ctx.reader(); - final Fields f = r.fields(); + final Fields f = new LeafReaderFields(r); fields.add(f); slices.add(new ReaderSlice(ctx.docBase, r.maxDoc(), fields.size()-1)); } @@ -115,9 +116,31 @@ public final class MultiFields extends Fields { } } - /** This method may return null if the field does not exist.*/ + /** This method may return null if the field does not exist or if it has no terms. */ public static Terms getTerms(IndexReader r, String field) throws IOException { - return getFields(r).terms(field); + final List leaves = r.leaves(); + if (leaves.size() == 1) { + return leaves.get(0).reader().terms(field); + } + + final List termsPerLeaf = new ArrayList<>(leaves.size()); + final List slicePerLeaf = new ArrayList<>(leaves.size()); + + for (int leafIdx = 0; leafIdx < leaves.size(); leafIdx++) { + LeafReaderContext ctx = leaves.get(leafIdx); + Terms subTerms = ctx.reader().terms(field); + if (subTerms != null) { + termsPerLeaf.add(subTerms); + slicePerLeaf.add(new ReaderSlice(ctx.docBase, r.maxDoc(), leafIdx - 1)); + } + } + + if (termsPerLeaf.size() == 0) { + return null; + } else { + return new MultiTerms(termsPerLeaf.toArray(Terms.EMPTY_ARRAY), + slicePerLeaf.toArray(ReaderSlice.EMPTY_ARRAY)); + } } /** Returns {@link PostingsEnum} for the specified field and @@ -264,5 +287,37 @@ public final class MultiFields extends Fields { } return fields; } + + private static class LeafReaderFields extends Fields { + + private final LeafReader leafReader; + private final List indexedFields; + + LeafReaderFields(LeafReader leafReader) { + this.leafReader = leafReader; + this.indexedFields = new ArrayList<>(); + for (FieldInfo fieldInfo : leafReader.getFieldInfos()) { + if (fieldInfo.getIndexOptions() != IndexOptions.NONE) { + indexedFields.add(fieldInfo.name); + } + } + Collections.sort(indexedFields); + } + + @Override + public Iterator iterator() { + return Collections.unmodifiableList(indexedFields).iterator(); + } + + @Override + public int size() { + return indexedFields.size(); + } + + @Override + public Terms terms(String field) throws IOException { + return leafReader.terms(field); + } + } } diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java index 5fb28b0fb15..74b39622ebe 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java @@ -19,6 +19,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Iterator; import java.util.Map; @@ -50,7 +51,6 @@ import org.apache.lucene.util.Version; */ public class ParallelLeafReader extends LeafReader { private final FieldInfos fieldInfos; - private final ParallelFields fields = new ParallelFields(); private final LeafReader[] parallelReaders, storedFieldsReaders; private final Set completeReaderSet = Collections.newSetFromMap(new IdentityHashMap()); @@ -58,9 +58,10 @@ public class ParallelLeafReader extends LeafReader { private final int maxDoc, numDocs; private final boolean hasDeletions; private final LeafMetaData metaData; - private final SortedMap fieldToReader = new TreeMap<>(); private final SortedMap tvFieldToReader = new TreeMap<>(); - + private final SortedMap fieldToReader = new TreeMap<>();//TODO needn't sort? + private final Map termsFieldToReader = new HashMap<>(); + /** Create a ParallelLeafReader based on the provided * readers; auto-closes the given readers on {@link #close()}. */ public ParallelLeafReader(LeafReader... readers) throws IOException { @@ -130,9 +131,15 @@ public class ParallelLeafReader extends LeafReader { if (!fieldToReader.containsKey(fieldInfo.name)) { builder.add(fieldInfo); fieldToReader.put(fieldInfo.name, reader); + // only add these if the reader responsible for that field name is the current: + // TODO consider populating 1st leaf with vectors even if the field name has been seen on a previous leaf if (fieldInfo.hasVectors()) { tvFieldToReader.put(fieldInfo.name, reader); } + // TODO consider populating 1st leaf with terms even if the field name has been seen on a previous leaf + if (fieldInfo.getIndexOptions() != IndexOptions.NONE) { + termsFieldToReader.put(fieldInfo.name, reader); + } } } } @@ -154,17 +161,6 @@ public class ParallelLeafReader extends LeafReader { fieldInfos = builder.finish(); this.metaData = new LeafMetaData(createdVersionMajor, minVersion, indexSort); - - // build Fields instance - for (final LeafReader reader : this.parallelReaders) { - final Fields readerFields = reader.fields(); - for (String field : readerFields) { - // only add if the reader responsible for that field name is the current: - if (fieldToReader.get(field) == reader) { - this.fields.addField(field, readerFields.terms(field)); - } - } - } // do this finally so any Exceptions occurred before don't affect refcounts: for (LeafReader reader : completeReaderSet) { @@ -230,13 +226,14 @@ public class ParallelLeafReader extends LeafReader { ensureOpen(); return hasDeletions ? parallelReaders[0].getLiveDocs() : null; } - + @Override - public Fields fields() { + public Terms terms(String field) throws IOException { ensureOpen(); - return fields; + LeafReader leafReader = termsFieldToReader.get(field); + return leafReader == null ? null : leafReader.terms(field); } - + @Override public int numDocs() { // Don't call ensureOpen() here (it could affect performance) diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java index b115a856620..53325578035 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java @@ -18,6 +18,8 @@ package org.apache.lucene.index; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import org.apache.lucene.codecs.DocValuesProducer; @@ -284,21 +286,27 @@ public final class SlowCodecReaderWrapper { } private static FieldsProducer readerToFieldsProducer(final LeafReader reader) throws IOException { - final Fields fields = reader.fields(); + ArrayList indexedFields = new ArrayList<>(); + for (FieldInfo fieldInfo : reader.getFieldInfos()) { + if (fieldInfo.getIndexOptions() != IndexOptions.NONE) { + indexedFields.add(fieldInfo.name); + } + } + Collections.sort(indexedFields); return new FieldsProducer() { @Override public Iterator iterator() { - return fields.iterator(); + return indexedFields.iterator(); } @Override public Terms terms(String field) throws IOException { - return fields.terms(field); + return reader.terms(field); } @Override public int size() { - return fields.size(); + return indexedFields.size(); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java index 321b552a066..eb9f7ed8a52 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java @@ -49,6 +49,7 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; */ class SortingLeafReader extends FilterLeafReader { + //TODO remove from here; move to FreqProxTermsWriter or FreqProxFields? static class SortingFields extends FilterFields { private final Sorter.DocMap docMap; @@ -1042,8 +1043,9 @@ class SortingLeafReader extends FilterLeafReader { } @Override - public Fields fields() throws IOException { - return new SortingFields(in.fields(), in.getFieldInfos(), docMap); + public Terms terms(String field) throws IOException { + Terms terms = super.terms(field); + return terms==null ? null : new SortingTerms(terms, in.getFieldInfos().fieldInfo(field).getIndexOptions(), docMap); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java index 3f424f51503..14b200230a6 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java @@ -37,17 +37,6 @@ import org.junit.Ignore; public class TestExitableDirectoryReader extends LuceneTestCase { private static class TestReader extends FilterLeafReader { - private static class TestFields extends FilterFields { - TestFields(Fields in) { - super(in); - } - - @Override - public Terms terms(String field) throws IOException { - return new TestTerms(super.terms(field)); - } - } - private static class TestTerms extends FilterTerms { TestTerms(Terms in) { super(in); @@ -83,8 +72,9 @@ public class TestExitableDirectoryReader extends LuceneTestCase { } @Override - public Fields fields() throws IOException { - return new TestFields(super.fields()); + public Terms terms(String field) throws IOException { + Terms terms = super.terms(field); + return terms==null ? null : new TestTerms(terms); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java index 79862fc25ca..de06f900083 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java @@ -35,18 +35,6 @@ public class TestFilterLeafReader extends LuceneTestCase { private static class TestReader extends FilterLeafReader { - /** Filter that only permits terms containing 'e'.*/ - private static class TestFields extends FilterFields { - TestFields(Fields in) { - super(in); - } - - @Override - public Terms terms(String field) throws IOException { - return new TestTerms(super.terms(field)); - } - } - private static class TestTerms extends FilterTerms { TestTerms(Terms in) { super(in); @@ -103,8 +91,9 @@ public class TestFilterLeafReader extends LuceneTestCase { } @Override - public Fields fields() throws IOException { - return new TestFields(super.fields()); + public Terms terms(String field) throws IOException { + Terms terms = super.terms(field); + return terms==null ? null : new TestTerms(terms); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java index 25d05ca155b..8afad2cc57c 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java @@ -18,7 +18,6 @@ package org.apache.lucene.index; import java.io.IOException; -import java.util.Iterator; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -90,21 +89,10 @@ public class TestParallelTermEnum extends LuceneTestCase { public void test1() throws IOException { ParallelLeafReader pr = new ParallelLeafReader(ir1, ir2); - Fields fields = pr.fields(); - Iterator fe = fields.iterator(); + assertEquals(3, pr.getFieldInfos().size()); - String f = fe.next(); - assertEquals("field1", f); - checkTerms(fields.terms(f), "brown", "fox", "jumps", "quick", "the"); - - f = fe.next(); - assertEquals("field2", f); - checkTerms(fields.terms(f), "brown", "fox", "jumps", "quick", "the"); - - f = fe.next(); - assertEquals("field3", f); - checkTerms(fields.terms(f), "dog", "fox", "jumps", "lazy", "over", "the"); - - assertFalse(fe.hasNext()); + checkTerms(pr.terms("field1"), "brown", "fox", "jumps", "quick", "the"); + checkTerms(pr.terms("field2"), "brown", "fox", "jumps", "quick", "the"); + checkTerms(pr.terms("field3"), "dog", "fox", "jumps", "lazy", "over", "the"); } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java b/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java index 9b8a285937f..e0154ab8ecd 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java +++ b/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java @@ -29,7 +29,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexReader; @@ -219,21 +218,16 @@ public class TermInSetQueryTest extends LuceneTestCase { } @Override - public Fields fields() throws IOException { - return new FilterFields(in.fields()) { + public Terms terms(String field) throws IOException { + Terms terms = super.terms(field); + if (terms == null) { + return null; + } + return new FilterTerms(terms) { @Override - public Terms terms(String field) throws IOException { - final Terms in = this.in.terms(field); - if (in == null) { - return null; - } - return new FilterTerms(in) { - @Override - public TermsEnum iterator() throws IOException { - counter.incrementAndGet(); - return super.iterator(); - } - }; + public TermsEnum iterator() throws IOException { + counter.incrementAndGet(); + return super.iterator(); } }; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java index 41d82d5cc7c..02c12289b98 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java @@ -22,7 +22,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.LeafReader; @@ -123,31 +122,27 @@ public class TestTermQuery extends LuceneTestCase { } @Override - public Fields fields() throws IOException { - return new FilterFields(super.fields()) { + public Terms terms(String field) throws IOException { + Terms terms = super.terms(field); + return terms==null ? null : new FilterTerms(terms) { @Override - public Terms terms(String field) throws IOException { - return new FilterTerms(super.terms(field)) { + public TermsEnum iterator() throws IOException { + return new FilterTermsEnum(super.iterator()) { @Override - public TermsEnum iterator() throws IOException { - return new FilterTermsEnum(super.iterator()) { - @Override - public SeekStatus seekCeil(BytesRef text) throws IOException { - throw new AssertionError("no seek"); - } - @Override - public void seekExact(BytesRef term, TermState state) throws IOException { - throw new AssertionError("no seek"); - } - @Override - public boolean seekExact(BytesRef text) throws IOException { - throw new AssertionError("no seek"); - } - @Override - public void seekExact(long ord) throws IOException { - throw new AssertionError("no seek"); - } - }; + public SeekStatus seekCeil(BytesRef text) throws IOException { + throw new AssertionError("no seek"); + } + @Override + public void seekExact(BytesRef term, TermState state) throws IOException { + throw new AssertionError("no seek"); + } + @Override + public boolean seekExact(BytesRef text) throws IOException { + throw new AssertionError("no seek"); + } + @Override + public void seekExact(long ord) throws IOException { + throw new AssertionError("no seek"); } }; } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java index 3a723cf8113..144209dcceb 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java @@ -25,8 +25,8 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Fields; -import org.apache.lucene.index.LeafMetaData; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.LeafMetaData; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PointValues; @@ -90,8 +90,8 @@ public class TermVectorLeafReader extends LeafReader { } @Override - public Fields fields() throws IOException { - return fields; + public Terms terms(String field) throws IOException { + return fields.terms(field); } @Override @@ -148,7 +148,7 @@ public class TermVectorLeafReader extends LeafReader { if (docID != 0) { return null; } - return fields(); + return fields; } @Override diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java index a898854cb7a..9fbc121b11d 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java @@ -18,7 +18,6 @@ package org.apache.lucene.search.highlight; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -30,7 +29,6 @@ import org.apache.lucene.analysis.CachingTokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; @@ -429,30 +427,15 @@ public class WeightedSpanTermExtractor { DelegatingLeafReader(LeafReader in) { super(in); } - + @Override public FieldInfos getFieldInfos() { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException();//TODO merge them } @Override - public Fields fields() throws IOException { - return new FilterFields(super.fields()) { - @Override - public Terms terms(String field) throws IOException { - return super.terms(DelegatingLeafReader.FIELD_NAME); - } - - @Override - public Iterator iterator() { - return Collections.singletonList(DelegatingLeafReader.FIELD_NAME).iterator(); - } - - @Override - public int size() { - return 1; - } - }; + public Terms terms(String field) throws IOException { + return super.terms(DelegatingLeafReader.FIELD_NAME); } @Override diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java index ebc5d3761e6..0195abb872d 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java @@ -24,7 +24,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -36,7 +35,6 @@ import java.util.function.Predicate; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -529,12 +527,16 @@ public class PhraseHelper { } } + //TODO move up; it's currently inbetween other inner classes that are related /** + * Needed to support the ability to highlight a query irrespective of the field a query refers to + * (aka requireFieldMatch=false). * This reader will just delegate every call to a single field in the wrapped * LeafReader. This way we ensure that all queries going through this reader target the same field. - */ + */ static final class SingleFieldFilterLeafReader extends FilterLeafReader { final String fieldName; + SingleFieldFilterLeafReader(LeafReader in, String fieldName) { super(in); this.fieldName = fieldName; @@ -542,27 +544,12 @@ public class PhraseHelper { @Override public FieldInfos getFieldInfos() { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException();//TODO merge them } @Override - public Fields fields() throws IOException { - return new FilterFields(super.fields()) { - @Override - public Terms terms(String field) throws IOException { - return super.terms(fieldName); - } - - @Override - public Iterator iterator() { - return Collections.singletonList(fieldName).iterator(); - } - - @Override - public int size() { - return 1; - } - }; + public Terms terms(String field) throws IOException { + return super.terms(fieldName); } @Override diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java index 8555cbd81e6..4165af45a9e 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java @@ -18,7 +18,6 @@ package org.apache.lucene.search.uhighlight; import java.io.IOException; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.PostingsEnum; @@ -52,24 +51,9 @@ final class TermVectorFilteredLeafReader extends FilterLeafReader { } @Override - public Fields fields() throws IOException { - return new TermVectorFilteredFields(in.fields(), filterTerms); - } - - private static final class TermVectorFilteredFields extends FilterLeafReader.FilterFields { - // NOTE: super ("in") is baseFields - - private final Terms filterTerms; - - TermVectorFilteredFields(Fields baseFields, Terms filterTerms) { - super(baseFields); - this.filterTerms = filterTerms; - } - - @Override - public Terms terms(String field) throws IOException { - return new TermsFilteredTerms(in.terms(field), filterTerms); - } + public Terms terms(String field) throws IOException { + Terms terms = in.terms(field); + return terms==null ? null : new TermsFilteredTerms(terms, filterTerms); } private static final class TermsFilteredTerms extends FilterLeafReader.FilterTerms { diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java index 92b9359c948..9812a3977cc 100644 --- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java +++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java @@ -1134,7 +1134,7 @@ public class MemoryIndex { */ private final class MemoryIndexReader extends LeafReader { - private Fields memoryFields = new MemoryFields(fields); + private final MemoryFields memoryFields = new MemoryFields(fields); private MemoryIndexReader() { super(); // avoid as much superclass baggage as possible @@ -1236,8 +1236,8 @@ public class MemoryIndex { } @Override - public Fields fields() { - return memoryFields; + public Terms terms(String field) throws IOException { + return memoryFields.terms(field); } private class MemoryFields extends Fields { @@ -1589,7 +1589,7 @@ public class MemoryIndex { @Override public Fields getTermVectors(int docID) { if (docID == 0) { - return fields(); + return memoryFields; } else { return null; } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java index a58765348ca..7c1b672e838 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java @@ -71,10 +71,11 @@ public class AssertingLeafReader extends FilterLeafReader { } @Override - public Fields fields() throws IOException { - return new AssertingFields(super.fields()); + public Terms terms(String field) throws IOException { + Terms terms = super.terms(field); + return terms == null ? null : new AssertingTerms(terms); } - + @Override public Fields getTermVectors(int docID) throws IOException { Fields fields = super.getTermVectors(docID); diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java index e45011428ca..959466a59f2 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java @@ -335,7 +335,7 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase { // PostingsFormat try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(writeState)) { - consumer.write(oneDocReader.fields()); + consumer.write(MultiFields.getFields(oneDocReader)); IOUtils.close(consumer); IOUtils.close(consumer); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java index 1155a7365c3..f69ca55f193 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java @@ -193,10 +193,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest iw.addDocument(doc); DirectoryReader ir = iw.getReader(); LeafReader ar = getOnlyLeafReader(ir); - Fields fields = ar.fields(); - int fieldCount = fields.size(); - // -1 is allowed, if the codec doesn't implement fields.size(): - assertTrue(fieldCount == 1 || fieldCount == -1); + assertEquals(1, ar.getFieldInfos().size()); Terms terms = ar.terms(""); assertNotNull(terms); TermsEnum termsEnum = terms.iterator(); @@ -218,10 +215,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest iw.addDocument(doc); DirectoryReader ir = iw.getReader(); LeafReader ar = getOnlyLeafReader(ir); - Fields fields = ar.fields(); - int fieldCount = fields.size(); - // -1 is allowed, if the codec doesn't implement fields.size(): - assertTrue(fieldCount == 1 || fieldCount == -1); + assertEquals(1, ar.getFieldInfos().size()); Terms terms = ar.terms(""); assertNotNull(terms); TermsEnum termsEnum = terms.iterator(); @@ -296,11 +290,10 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest iw.forceMerge(1); DirectoryReader ir = iw.getReader(); LeafReader ar = getOnlyLeafReader(ir); - Fields fields = ar.fields(); // Ghost busting terms dict impls will have // fields.size() == 0; all others must be == 1: - assertTrue(fields.size() <= 1); - Terms terms = fields.terms("ghostField"); + assertTrue(ar.getFieldInfos().size() <= 1); + Terms terms = ar.terms("ghostField"); if (terms != null) { TermsEnum termsEnum = terms.iterator(); BytesRef term = termsEnum.next(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java index 751833f6fd6..38a3b7a6749 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java @@ -108,11 +108,10 @@ public final class FieldFilterLeafReader extends FilterLeafReader { } @Override - public Fields fields() throws IOException { - final Fields f = super.fields(); - return (f == null) ? null : new FieldFilterFields(f); + public Terms terms(String field) throws IOException { + return hasField(field) ? super.terms(field) : null; } - + @Override public BinaryDocValues getBinaryDocValues(String field) throws IOException { return hasField(field) ? super.getBinaryDocValues(field) : null; @@ -145,7 +144,7 @@ public final class FieldFilterLeafReader extends FilterLeafReader { if (negate) sb.append('!'); return sb.append(fields).append(')').toString(); } - + private class FieldFilterFields extends FilterFields { public FieldFilterFields(Fields in) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java index bd23fd210b3..cb2c984104a 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java @@ -17,17 +17,16 @@ package org.apache.lucene.search; import java.io.IOException; -import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Random; +import junit.framework.Assert; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Fields; -import org.apache.lucene.index.LeafMetaData; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafMetaData; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiReader; @@ -42,8 +41,6 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.Version; -import junit.framework.Assert; - import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertFalse; import static junit.framework.Assert.assertTrue; @@ -172,23 +169,8 @@ public class QueryUtils { return new LeafReader() { @Override - public Fields fields() throws IOException { - return new Fields() { - @Override - public Iterator iterator() { - return Collections.emptyList().iterator(); - } - - @Override - public Terms terms(String field) throws IOException { - return null; - } - - @Override - public int size() { - return 0; - } - }; + public Terms terms(String field) throws IOException { + return null; } @Override diff --git a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java index ab7313fbf5e..ad1e81f3d45 100644 --- a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java +++ b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java @@ -98,9 +98,9 @@ public final class SlowCompositeReaderWrapper extends LeafReader { } @Override - public Fields fields() { + public Terms terms(String field) throws IOException { ensureOpen(); - return fields; + return fields.terms(field); } @Override diff --git a/solr/core/src/test/org/apache/solr/search/TestDocSet.java b/solr/core/src/test/org/apache/solr/search/TestDocSet.java index e5cc6ebde4c..b5923170095 100644 --- a/solr/core/src/test/org/apache/solr/search/TestDocSet.java +++ b/solr/core/src/test/org/apache/solr/search/TestDocSet.java @@ -25,9 +25,9 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Fields; -import org.apache.lucene.index.LeafMetaData; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.LeafMetaData; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiReader; @@ -37,6 +37,7 @@ import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Terms; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSetIterator; @@ -399,7 +400,7 @@ public class TestDocSet extends LuceneTestCase { } @Override - public Fields fields() { + public Terms terms(String field) throws IOException { return null; } From f1e2be64519a9ec815785b59e6187c3e99f7d998 Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Thu, 15 Jun 2017 09:14:24 -0700 Subject: [PATCH 081/131] SOLR-10834: Fixed tests and test configs to stop using numeric uniqueKey fields Squashed commit of the following (from jira/SOLR-10834 branch): commit 8f1043840f38533864b2c713daf066b6c3509147 commit 7b95773bd524cd86aaccc56cc33a003a9aff2004 commit b26bf9d60e2b94e0cdc365d1e2c0a37c33e24198 commit df11992106f8c338503b6e3e9a27ba6ddcfa2953 commit fcf98132410ed247e451bb449a8337a09bd857ce commit 05e8e226de359a6d7bc99219eaec161a32268f17 commit 6dce948294351560948a32b64679b1879657af79 commit 53f97845caaa8adc25862e4017b94f3091063552 commit d5bfb5f57016341fbeaf73b5e4c9ed10dc3816d0 commit d333f7b1eee10893a81532ac2f5a77a46716d90b commit 15983ceec4702dc8c7562250d59cd8231c67d46a commit e18e2e771fb4678cb911a62bbc7c74a873466bf0 commit 134e210bdf601600a9d90dd0720a35cb122896b0 commit ec03260265f8a3bbdfd7f9b015de16a4950a05eb commit 5d430057ed335801a524e1e7666061075ab6d859 commit 1625df3d21b8fa27815d7a7b89a55fc338eeb23b --- solr/CHANGES.txt | 2 + .../collection1/conf/schema-icucollate-dv.xml | 4 +- .../collection1/conf/schema-icucollate.xml | 4 +- .../conf/schema-icucollateoptions.xml | 4 +- .../solr/schema/TestICUCollationField.java | 28 +- .../TestICUCollationFieldDocValues.java | 28 +- .../schema/TestICUCollationFieldOptions.java | 26 +- .../analysisconfs/analysis-err-schema.xml | 4 +- .../collection1/conf/schema-binaryfield.xml | 2 +- .../conf/schema-blockjoinfacetcomponent.xml | 4 +- .../solr/collection1/conf/schema-bm25.xml | 4 +- .../collection1/conf/schema-charfilters.xml | 4 +- .../collection1/conf/schema-collate-dv.xml | 4 +- .../solr/collection1/conf/schema-collate.xml | 4 +- .../conf/schema-copyfield-test.xml | 2 +- .../collection1/conf/schema-custom-field.xml | 6 +- .../collection1/conf/schema-customfield.xml | 3 +- .../solr/collection1/conf/schema-dfi.xml | 4 +- .../solr/collection1/conf/schema-dfr.xml | 4 +- .../conf/schema-distributed-missing-sort.xml | 6 +- .../collection1/conf/schema-docValues.xml | 5 +- .../conf/schema-field-sort-values.xml | 2 +- .../solr/collection1/conf/schema-hash.xml | 2 +- .../solr/collection1/conf/schema-ib.xml | 4 +- .../collection1/conf/schema-lmdirichlet.xml | 4 +- .../conf/schema-lmjelinekmercer.xml | 4 +- .../conf/schema-not-required-unique-key.xml | 2 +- .../collection1/conf/schema-phrasesuggest.xml | 4 +- .../conf/schema-postingshighlight.xml | 4 +- .../collection1/conf/schema-replication1.xml | 2 +- .../collection1/conf/schema-replication2.xml | 2 +- .../conf/schema-required-fields.xml | 2 +- .../solr/collection1/conf/schema-reversed.xml | 2 +- .../conf/schema-sim-default-override.xml | 4 +- .../solr/collection1/conf/schema-sim.xml | 4 +- .../conf/schema-simpleqpplugin.xml | 4 +- .../solr/collection1/conf/schema-sorts.xml | 2 +- .../solr/collection1/conf/schema-sql.xml | 2 +- .../collection1/conf/schema-sweetspot.xml | 4 +- .../conf/schema-synonym-tokenizer.xml | 6 +- .../solr/collection1/conf/schema-tfidf.xml | 4 +- .../conf/schema-tokenizer-test.xml | 5 +- .../solr/collection1/conf/schema-trie.xml | 2 +- .../conf/schema-unifiedhighlight.xml | 4 +- .../solr/collection1/conf/schema.xml | 5 +- .../solr/collection1/conf/schema11.xml | 6 +- .../configsets/configset-2/conf/schema.xml | 4 +- .../test-files/solr/crazy-path-to-schema.xml | 4 +- .../apache/solr/BasicFunctionalityTest.java | 36 +-- .../org/apache/solr/CursorPagingTest.java | 58 ++-- .../apache/solr/DisMaxRequestHandlerTest.java | 22 +- .../src/test/org/apache/solr/SampleTest.java | 4 +- .../apache/solr/TestDistributedGrouping.java | 18 +- .../solr/TestDistributedMissingSort.java | 60 ++-- .../apache/solr/TestDistributedSearch.java | 20 +- .../org/apache/solr/TestTolerantSearch.java | 4 +- .../PathHierarchyTokenizerFactoryTest.java | 32 +- .../apache/solr/analysis/TestCharFilters.java | 8 +- .../solr/cloud/BasicDistributedZk2Test.java | 8 +- .../solr/cloud/BasicDistributedZkTest.java | 10 +- .../org/apache/solr/cloud/BasicZkTest.java | 4 +- .../solr/cloud/DistribCursorPagingTest.java | 10 +- .../cloud/DistribJoinFromCollectionTest.java | 8 +- .../FullThrottleStoppableIndexingThread.java | 2 +- .../apache/solr/cloud/HttpPartitionTest.java | 2 +- .../cloud/SegmentTerminateEarlyTestState.java | 30 +- .../autoscaling/AutoScalingHandlerTest.java | 2 +- .../org/apache/solr/core/HelloStream.java | 2 +- .../org/apache/solr/core/MockInfoBean.java | 2 +- .../org/apache/solr/core/SOLR749Test.java | 12 +- .../apache/solr/core/TestJmxIntegration.java | 2 +- .../snapshots/TestSolrCloudSnapshots.java | 2 +- .../core/snapshots/TestSolrCoreSnapshots.java | 2 +- .../solr/handler/MoreLikeThisHandlerTest.java | 20 +- .../handler/StandardRequestHandlerTest.java | 24 +- .../apache/solr/handler/TestSQLHandler.java | 32 +- .../handler/XsltUpdateRequestHandlerTest.java | 2 +- .../solr/handler/admin/MBeansHandlerTest.java | 2 +- .../admin/PropertiesRequestHandlerTest.java | 2 +- .../DistributedExpandComponentTest.java | 40 +-- .../DistributedMLTComponentTest.java | 12 +- ...stributedQueryComponentCustomSortTest.java | 38 +-- ...istributedQueryElevationComponentTest.java | 4 +- .../QueryElevationComponentTest.java | 90 +++--- .../handler/component/StatsComponentTest.java | 20 +- ...tDistributedStatsComponentCardinality.java | 4 +- .../component/TestExpandComponent.java | 100 +++---- .../apache/solr/request/SimpleFacetsTest.java | 144 ++++----- .../org/apache/solr/request/TestFaceting.java | 14 +- .../transform/TestChildDocTransformer.java | 120 ++++---- .../org/apache/solr/rest/TestRestManager.java | 2 +- .../schema/AbstractCurrencyFieldTest.java | 82 ++--- .../org/apache/solr/schema/CopyFieldTest.java | 12 +- .../solr/schema/CurrencyFieldXmlFileTest.java | 4 +- .../solr/schema/DateRangeFieldTest.java | 2 +- .../org/apache/solr/schema/DocValuesTest.java | 256 ++++++++-------- .../schema/IndexSchemaRuntimeFieldTest.java | 4 +- .../apache/solr/schema/IndexSchemaTest.java | 6 +- .../apache/solr/schema/TestBinaryField.java | 6 +- .../solr/schema/TestCollationField.java | 32 +- .../schema/TestCollationFieldDocValues.java | 32 +- .../solr/schema/TestHalfAndHalfDocValues.java | 6 +- .../apache/solr/schema/TestOmitPositions.java | 4 +- .../apache/solr/search/SpatialFilterTest.java | 2 +- .../search/TestCollapseQParserPlugin.java | 280 +++++++++--------- .../TestComplexPhraseLeadingWildcard.java | 2 +- .../apache/solr/search/TestCustomSort.java | 20 +- .../solr/search/TestFieldSortValues.java | 10 +- .../search/TestFoldingMultitermQuery.java | 2 +- .../search/TestGraphTermsQParserPlugin.java | 34 +-- .../solr/search/TestHashQParserPlugin.java | 14 +- .../apache/solr/search/TestQueryTypes.java | 22 +- .../apache/solr/search/TestRangeQuery.java | 4 +- .../solr/search/TestReRankQParserPlugin.java | 208 ++++++------- .../org/apache/solr/search/TestReload.java | 2 +- .../apache/solr/search/TestSearchPerf.java | 2 +- .../solr/search/TestSolrQueryParser.java | 2 +- .../apache/solr/search/facet/DebugAgg.java | 2 +- .../search/function/SortByFunctionTest.java | 68 ++--- .../search/function/TestFunctionQuery.java | 43 +-- .../TestMinMaxOnMultiValuedField.java | 8 +- .../solr/search/mlt/SimpleMLTQParserTest.java | 86 +++--- .../solr/search/stats/TestDistribIDF.java | 14 +- .../security/MockAuthorizationPlugin.java | 2 +- .../solr/servlet/NoCacheHeaderTest.java | 2 +- .../solr/spelling/suggest/SuggesterTest.java | 19 +- .../suggest/TestAnalyzeInfixSuggestions.java | 2 +- .../suggest/TestFreeTextSuggestions.java | 2 +- .../update/processor/AtomicUpdatesTest.java | 36 +-- .../ClassificationUpdateProcessorTest.java | 2 +- .../org/apache/solr/util/PrimUtilsTest.java | 2 +- .../org/apache/solr/util/UtilsToolTest.java | 2 +- .../apache/solr/util/hll/BitVectorTest.java | 2 +- .../apache/solr/util/hll/ExplicitHLLTest.java | 2 +- .../org/apache/solr/util/hll/FullHLLTest.java | 2 +- .../collection1/conf/schema-replication1.xml | 2 +- .../solr/collection1/conf/schema-sql.xml | 2 +- .../solrj/solr/collection1/conf/schema.xml | 2 +- .../solrj/solr/crazy-path-to-schema.xml | 4 +- 139 files changed, 1326 insertions(+), 1306 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 88821bb2ab4..8db87529101 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -424,6 +424,8 @@ Other Changes * SOLR-10851: SolrClients should clarify expectations for solrServerUrl parameter (Jason Gerlowski via Tomás Fernández Löbbe) +* SOLR-10834: Fixed tests and test configs to stop using numeric uniqueKey fields (hossman) + ================== 6.6.1 ================== Bug Fixes diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate-dv.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate-dv.xml index 1e2029d11e0..63f7330c9f2 100644 --- a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate-dv.xml +++ b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate-dv.xml @@ -20,7 +20,7 @@ - + @@ -37,7 +37,7 @@ - + diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate.xml index 1d121f250ce..9698013aac6 100644 --- a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate.xml +++ b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate.xml @@ -20,7 +20,7 @@ - + @@ -37,7 +37,7 @@ - + diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollateoptions.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollateoptions.xml index cc8e340fe7c..59b8d251bfc 100644 --- a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollateoptions.xml +++ b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollateoptions.xml @@ -20,7 +20,7 @@ - + @@ -47,7 +47,7 @@ locale="en" strength="tertiary" caseFirst="upper"/> - + diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java index 4cff7faeeb9..f164080a7b8 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java @@ -108,8 +108,8 @@ public class TestICUCollationField extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_de:tone", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" + "//result/doc[1]/str[@name='id'][.=4]", + "//result/doc[2]/str[@name='id'][.=7]" ); } @@ -122,8 +122,8 @@ public class TestICUCollationField extends SolrTestCaseJ4 { assertQ("Collated RangeQ: ", req("fl", "id", "q", "sort_de:[tone TO tp]", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" + "//result/doc[1]/str[@name='id'][.=4]", + "//result/doc[2]/str[@name='id'][.=7]" ); } @@ -134,8 +134,8 @@ public class TestICUCollationField extends SolrTestCaseJ4 { assertQ("Collated Sort: ", req("fl", "id", "q", "sort_da:[tz TO töz]", "sort", "sort_da asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=11]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=11]", + "//result/doc[2]/str[@name='id'][.=4]" ); } @@ -147,8 +147,8 @@ public class TestICUCollationField extends SolrTestCaseJ4 { assertQ("Collated Sort: ", req("fl", "id", "q", "sort_ar:[\u0698 TO \u0633\u0633]", "sort", "sort_ar asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=12]", - "//result/doc[2]/int[@name='id'][.=1]" + "//result/doc[1]/str[@name='id'][.=12]", + "//result/doc[2]/str[@name='id'][.=1]" ); } @@ -171,9 +171,9 @@ public class TestICUCollationField extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_tr_canon:\"I Will Use Turkish Casıng\"", "sort", "id asc" ), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=3]", - "//result/doc[3]/int[@name='id'][.=5]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=5]" ); } @@ -183,10 +183,10 @@ public class TestICUCollationField extends SolrTestCaseJ4 { */ public void testCustomCollation() { assertQ("Collated TQ: ", - req("fl", "id", "q", "sort_custom:toene", "sort", "id asc" ), + req("fl", "id", "q", "sort_custom:toene"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=10]" + "//result/doc/str[@name='id'][.=4]", + "//result/doc/str[@name='id'][.=10]" ); } } diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldDocValues.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldDocValues.java index bdddc986508..57b403a52ad 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldDocValues.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldDocValues.java @@ -96,8 +96,8 @@ public class TestICUCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_de:tone", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" + "//result/doc[1]/str[@name='id'][.=4]", + "//result/doc[2]/str[@name='id'][.=7]" ); } @@ -110,8 +110,8 @@ public class TestICUCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated RangeQ: ", req("fl", "id", "q", "sort_de:[tone TO tp]", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" + "//result/doc[1]/str[@name='id'][.=4]", + "//result/doc[2]/str[@name='id'][.=7]" ); } @@ -122,8 +122,8 @@ public class TestICUCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated Sort: ", req("fl", "id", "q", "sort_da:[tz TO töz]", "sort", "sort_da asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=11]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=11]", + "//result/doc[2]/str[@name='id'][.=4]" ); } @@ -135,8 +135,8 @@ public class TestICUCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated Sort: ", req("fl", "id", "q", "sort_ar:[\u0698 TO \u0633\u0633]", "sort", "sort_ar asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=12]", - "//result/doc[2]/int[@name='id'][.=1]" + "//result/doc[1]/str[@name='id'][.=12]", + "//result/doc[2]/str[@name='id'][.=1]" ); } @@ -159,9 +159,9 @@ public class TestICUCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_tr_canon:\"I Will Use Turkish Casıng\"", "sort", "id asc" ), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=3]", - "//result/doc[3]/int[@name='id'][.=5]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=5]" ); } @@ -171,10 +171,10 @@ public class TestICUCollationFieldDocValues extends SolrTestCaseJ4 { */ public void testCustomCollation() { assertQ("Collated TQ: ", - req("fl", "id", "q", "sort_custom:toene", "sort", "id asc" ), + req("fl", "id", "q", "sort_custom:toene"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=10]" + "//result/doc/str[@name='id'][.=4]", + "//result/doc/str[@name='id'][.=10]" ); } } diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java index 7a5dca74e49..0b198b7506f 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java @@ -52,9 +52,9 @@ public class TestICUCollationFieldOptions extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_ignore_punctuation:foobar", "sort", "id asc" ), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=2]", - "//result/doc[3]/int[@name='id'][.=3]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=2]", + "//result/doc[3]/str[@name='id'][.=3]" ); } @@ -66,8 +66,8 @@ public class TestICUCollationFieldOptions extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_ignore_space:\"foo bar\"", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=3]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=3]" ); } @@ -79,8 +79,8 @@ public class TestICUCollationFieldOptions extends SolrTestCaseJ4 { assertQ("Collated sort: ", req("fl", "id", "q", "id:[4 TO 5]", "sort", "sort_numerics asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=5]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=5]", + "//result/doc[2]/str[@name='id'][.=4]" ); } @@ -92,15 +92,15 @@ public class TestICUCollationFieldOptions extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_ignore_accents:resume", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=6]", - "//result/doc[2]/int[@name='id'][.=9]" + "//result/doc[1]/str[@name='id'][.=6]", + "//result/doc[2]/str[@name='id'][.=9]" ); assertQ("Collated TQ: ", req("fl", "id", "q", "sort_ignore_accents:Resume", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=7]", - "//result/doc[2]/int[@name='id'][.=8]" + "//result/doc[1]/str[@name='id'][.=7]", + "//result/doc[2]/str[@name='id'][.=8]" ); } @@ -112,8 +112,8 @@ public class TestICUCollationFieldOptions extends SolrTestCaseJ4 { assertQ("Collated sort: ", req("fl", "id", "q", "id:6 OR id:8", "sort", "sort_uppercase_first asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=8]", - "//result/doc[2]/int[@name='id'][.=6]" + "//result/doc[1]/str[@name='id'][.=8]", + "//result/doc[2]/str[@name='id'][.=6]" ); } } diff --git a/solr/core/src/test-files/solr/analysisconfs/analysis-err-schema.xml b/solr/core/src/test-files/solr/analysisconfs/analysis-err-schema.xml index 168df6b90ba..eedf4b7ec01 100644 --- a/solr/core/src/test-files/solr/analysisconfs/analysis-err-schema.xml +++ b/solr/core/src/test-files/solr/analysisconfs/analysis-err-schema.xml @@ -22,7 +22,7 @@ - + @@ -32,7 +32,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml index eb33896e78e..26a428d599d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml @@ -67,7 +67,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-blockjoinfacetcomponent.xml b/solr/core/src/test-files/solr/collection1/conf/schema-blockjoinfacetcomponent.xml index e262b7305bd..dc23b84c20f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-blockjoinfacetcomponent.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-blockjoinfacetcomponent.xml @@ -22,9 +22,9 @@ - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-bm25.xml b/solr/core/src/test-files/solr/collection1/conf/schema-bm25.xml index 13924b70786..7c31f159362 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-bm25.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-bm25.xml @@ -19,7 +19,7 @@ - + @@ -37,7 +37,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-charfilters.xml b/solr/core/src/test-files/solr/collection1/conf/schema-charfilters.xml index 11e2aa77530..1adb7aaf7e8 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-charfilters.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-charfilters.xml @@ -16,7 +16,7 @@ --> - + @@ -38,7 +38,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-collate-dv.xml b/solr/core/src/test-files/solr/collection1/conf/schema-collate-dv.xml index af74654b9b3..0a2953d590c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-collate-dv.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-collate-dv.xml @@ -19,7 +19,7 @@ - + @@ -37,7 +37,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml b/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml index d5c4d690a52..bfe65b157e1 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml @@ -19,7 +19,7 @@ - + @@ -37,7 +37,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml b/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml index c16bf55d4e6..96adf2dd623 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml @@ -333,7 +333,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml b/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml index 58c40d5cbbf..23999e41b33 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml @@ -26,7 +26,7 @@ - + @@ -34,6 +34,10 @@ stored="true" multiValued="false" docValues="true" required="true"/> + + + + id diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-customfield.xml b/solr/core/src/test-files/solr/collection1/conf/schema-customfield.xml index 4d0cbfd8d5b..83948b7fcae 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-customfield.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-customfield.xml @@ -32,6 +32,7 @@ attribute and any other attributes determine the real type and behavior of the fieldType. --> + - + @@ -39,7 +39,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-dfr.xml b/solr/core/src/test-files/solr/collection1/conf/schema-dfr.xml index 4b27707c87c..30835105570 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-dfr.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-dfr.xml @@ -19,7 +19,7 @@ - + @@ -54,7 +54,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml b/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml index 822d2be8c02..13daf46ca38 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml @@ -45,7 +45,7 @@ - + @@ -77,6 +77,10 @@ + + + + id diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml index 9e4286d226b..0216791d48d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml @@ -53,7 +53,10 @@ - + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-field-sort-values.xml b/solr/core/src/test-files/solr/collection1/conf/schema-field-sort-values.xml index 0cd22b01213..94485f6401e 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-field-sort-values.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-field-sort-values.xml @@ -26,7 +26,7 @@ - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-ib.xml b/solr/core/src/test-files/solr/collection1/conf/schema-ib.xml index e477e2567eb..7e66bb526bf 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-ib.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-ib.xml @@ -19,7 +19,7 @@ - + @@ -42,7 +42,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-lmdirichlet.xml b/solr/core/src/test-files/solr/collection1/conf/schema-lmdirichlet.xml index a51685e1688..3c250988054 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-lmdirichlet.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-lmdirichlet.xml @@ -19,7 +19,7 @@ - + @@ -35,7 +35,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-lmjelinekmercer.xml b/solr/core/src/test-files/solr/collection1/conf/schema-lmjelinekmercer.xml index 7c0fcd3997d..a90fd6552fd 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-lmjelinekmercer.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-lmjelinekmercer.xml @@ -19,7 +19,7 @@ - + @@ -35,7 +35,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml b/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml index e2bbef36e56..d38952f3183 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml @@ -31,7 +31,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml b/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml index 0bc78d6adb5..7059cd399b4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml @@ -19,7 +19,7 @@ - + @@ -48,7 +48,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-postingshighlight.xml b/solr/core/src/test-files/solr/collection1/conf/schema-postingshighlight.xml index 18aa51a28f3..e68ba0f117b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-postingshighlight.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-postingshighlight.xml @@ -19,7 +19,7 @@ - + @@ -37,7 +37,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml b/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml index b647fa2fbde..eeb51bdccd7 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml @@ -30,7 +30,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml b/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml index 8c8c49d6e92..ade7bf54517 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml @@ -30,7 +30,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml b/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml index f394bbb704b..551ac1d9357 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml @@ -316,7 +316,7 @@ - diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-reversed.xml b/solr/core/src/test-files/solr/collection1/conf/schema-reversed.xml index 63887cb34b4..3bb52ebdfc8 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-reversed.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-reversed.xml @@ -66,7 +66,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sim-default-override.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sim-default-override.xml index 100caaf031e..276432441bb 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sim-default-override.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sim-default-override.xml @@ -50,7 +50,7 @@ - + @@ -60,7 +60,7 @@ - + id diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sim.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sim.xml index 68c4d6d963b..b9415711df6 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sim.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sim.xml @@ -19,7 +19,7 @@ - + @@ -46,7 +46,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-simpleqpplugin.xml b/solr/core/src/test-files/solr/collection1/conf/schema-simpleqpplugin.xml index 1183fd059d5..0ed7e10f6e0 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-simpleqpplugin.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-simpleqpplugin.xml @@ -20,7 +20,7 @@ - + @@ -50,7 +50,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml index 8497318726c..1ff68c1b89d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml @@ -25,7 +25,7 @@ NOTE: Tests expect every field in this schema to be sortable. id - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml index efa110c7136..e73c9c9a5ac 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml @@ -436,7 +436,7 @@ --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sweetspot.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sweetspot.xml index ba021a22987..080faabfacc 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sweetspot.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sweetspot.xml @@ -19,7 +19,7 @@ - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-synonym-tokenizer.xml b/solr/core/src/test-files/solr/collection1/conf/schema-synonym-tokenizer.xml index 09792f98a52..8ab64afdbb3 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-synonym-tokenizer.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-synonym-tokenizer.xml @@ -18,7 +18,7 @@ - + @@ -33,8 +33,8 @@ - + id - \ No newline at end of file + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-tfidf.xml b/solr/core/src/test-files/solr/collection1/conf/schema-tfidf.xml index 0bec86f83ad..df05e82d15b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-tfidf.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-tfidf.xml @@ -38,7 +38,7 @@ - - + + id diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml b/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml index af28016ded0..174787ef989 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml @@ -32,7 +32,8 @@ more concise example. attribute and any other attributes determine the real type and behavior of the fieldType. --> - + + @@ -127,7 +128,7 @@ more concise example. - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml b/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml index 8cab0c85ff6..775ec9d2980 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml @@ -273,7 +273,7 @@ --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-unifiedhighlight.xml b/solr/core/src/test-files/solr/collection1/conf/schema-unifiedhighlight.xml index 3e093f342c0..0c9a08357ad 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-unifiedhighlight.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-unifiedhighlight.xml @@ -18,7 +18,7 @@ - + @@ -36,7 +36,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml index f0ddae44c46..685ee687630 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml @@ -497,8 +497,8 @@ - - + + @@ -806,6 +806,7 @@ + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema11.xml b/solr/core/src/test-files/solr/collection1/conf/schema11.xml index 819b6d1b9eb..9584061897b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema11.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema11.xml @@ -319,8 +319,10 @@ valued. --> best performance. --> - - + + + + diff --git a/solr/core/src/test-files/solr/configsets/configset-2/conf/schema.xml b/solr/core/src/test-files/solr/configsets/configset-2/conf/schema.xml index 981e5226d95..73f59c8eeb0 100644 --- a/solr/core/src/test-files/solr/configsets/configset-2/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/configset-2/conf/schema.xml @@ -18,8 +18,8 @@ - - + + id diff --git a/solr/core/src/test-files/solr/crazy-path-to-schema.xml b/solr/core/src/test-files/solr/crazy-path-to-schema.xml index ef9ce3ac893..5b443ea3c52 100644 --- a/solr/core/src/test-files/solr/crazy-path-to-schema.xml +++ b/solr/core/src/test-files/solr/crazy-path-to-schema.xml @@ -23,7 +23,7 @@ - + @@ -37,7 +37,7 @@ - + diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java index b1747fd9afb..2122c90fe57 100644 --- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java +++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java @@ -111,7 +111,7 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { assertQ("query with ignored field", req("bar_ignored:yo id:42") ,"//*[@numFound='1']" - ,"//int[@name='id'][.='42']" + ,"//str[@name='id'][.='42']" ); } @@ -150,13 +150,13 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { assertQ("backslash escaping semicolon", req("id:42 AND val_s:aa\\;bb") ,"//*[@numFound='1']" - ,"//int[@name='id'][.='42']" + ,"//str[@name='id'][.='42']" ); assertQ("quote escaping semicolon", req("id:42 AND val_s:\"aa;bb\"") ,"//*[@numFound='1']" - ,"//int[@name='id'][.='42']" + ,"//str[@name='id'][.='42']" ); assertQ("no escaping semicolon", @@ -275,15 +275,15 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { ); assertQ(req("{!lucene q.op=AND df=text}grape green") ,"//*[@numFound='1']" - ,"//int[@name='id'][.='103']" + ,"//str[@name='id'][.='103']" ); assertQ(req("-_val_:\"{!lucene q.op=AND df=text}grape green\"") ,"//*[@numFound='5']" - ,"//int[@name='id'][.='101']" - ,"//int[@name='id'][.='102']" - ,"//int[@name='id'][.='104']" - ,"//int[@name='id'][.='105']" - ,"//int[@name='id'][.='106']" + ,"//str[@name='id'][.='101']" + ,"//str[@name='id'][.='102']" + ,"//str[@name='id'][.='104']" + ,"//str[@name='id'][.='105']" + ,"//str[@name='id'][.='106']" ); // tests @@ -298,26 +298,26 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { assertU(commit()); assertQ(req("*:*") ,"//*[@numFound='4']" - ,"//int[@name='id'][.='101']" - ,"//int[@name='id'][.='102']" - ,"//int[@name='id'][.='103']" - ,"//int[@name='id'][.='106']" + ,"//str[@name='id'][.='101']" + ,"//str[@name='id'][.='102']" + ,"//str[@name='id'][.='103']" + ,"//str[@name='id'][.='106']" ); assertU(delQ("{!term f=id}106")); assertU(commit()); assertQ(req("*:*") ,"//*[@numFound='3']" - ,"//int[@name='id'][.='101']" - ,"//int[@name='id'][.='102']" - ,"//int[@name='id'][.='103']" + ,"//str[@name='id'][.='101']" + ,"//str[@name='id'][.='102']" + ,"//str[@name='id'][.='103']" ); assertU(delQ("-_val_:\"{!lucene q.op=AND df=text}grape green\"")); assertU(commit()); assertQ(req("*:*") ,"//*[@numFound='1']" - ,"//int[@name='id'][.='103']" + ,"//str[@name='id'][.='103']" ); assertU(delQ("-text:doesnotexist")); @@ -907,7 +907,7 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { assertQ("check math on field query: " + q, req("q", q), "*[count(//doc)=1]", - "//int[@name='id'][.='1']"); + "//str[@name='id'][.='1']"); } // range queries using date math diff --git a/solr/core/src/test/org/apache/solr/CursorPagingTest.java b/solr/core/src/test/org/apache/solr/CursorPagingTest.java index eb1c6bc6fe9..dfaddea6cac 100644 --- a/solr/core/src/test/org/apache/solr/CursorPagingTest.java +++ b/solr/core/src/test/org/apache/solr/CursorPagingTest.java @@ -191,17 +191,17 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==9" ,"/response/start==0" - ,"/response/docs==[{'id':9},{'id':8},{'id':7},{'id':6}]" + ,"/response/docs==[{'id':'9'},{'id':'8'},{'id':'7'},{'id':'6'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==9" ,"/response/start==0" - ,"/response/docs==[{'id':5},{'id':3},{'id':2},{'id':1}]" + ,"/response/docs==[{'id':'5'},{'id':'3'},{'id':'2'},{'id':'1'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==9" ,"/response/start==0" - ,"/response/docs==[{'id':0}]" + ,"/response/docs==[{'id':'0'}]" ); // no more, so no change to cursorMark, and no new docs assertEquals(cursorMark, @@ -223,13 +223,13 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==7" ,"/response/start==0" - ,"/response/docs==[{'id':6},{'id':1},{'id':8},{'id':5}]" + ,"/response/docs==[{'id':'6'},{'id':'1'},{'id':'8'},{'id':'5'}]" ,"/facet_counts/facet_fields/str=={'a':4,'b':3,'c':0}" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==7" ,"/response/start==0" - ,"/response/docs==[{'id':4},{'id':3},{'id':0}]" + ,"/response/docs==[{'id':'4'},{'id':'3'},{'id':'0'}]" ,"/facet_counts/facet_fields/str=={'a':4,'b':3,'c':0}" ); // no more, so no change to cursorMark, and no new docs @@ -253,19 +253,19 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':7},{'id':0},{'id':3}]" + ,"/response/docs==[{'id':'7'},{'id':'0'},{'id':'3'}]" ,"/facet_counts/facet_fields/str=={'a':4,'b':1,'c':3}" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':4},{'id':1},{'id':6}]" + ,"/response/docs==[{'id':'4'},{'id':'1'},{'id':'6'}]" ,"/facet_counts/facet_fields/str=={'a':4,'b':1,'c':3}" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':9},{'id':2}]" + ,"/response/docs==[{'id':'9'},{'id':'2'}]" ,"/facet_counts/facet_fields/str=={'a':4,'b':1,'c':3}" ); // no more, so no change to cursorMark, and no new docs @@ -287,17 +287,17 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':3},{'id':7},{'id':0}]" + ,"/response/docs==[{'id':'3'},{'id':'7'},{'id':'0'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':4},{'id':1},{'id':6}]" + ,"/response/docs==[{'id':'4'},{'id':'1'},{'id':'6'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':9},{'id':2}]" + ,"/response/docs==[{'id':'9'},{'id':'2'}]" ); // no more, so no change to cursorMark, and no new docs assertEquals(cursorMark, @@ -317,17 +317,17 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':7},{'id':0},{'id':4}]" + ,"/response/docs==[{'id':'7'},{'id':'0'},{'id':'4'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':1},{'id':6},{'id':9}]" + ,"/response/docs==[{'id':'1'},{'id':'6'},{'id':'9'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':2},{'id':3}]" + ,"/response/docs==[{'id':'2'},{'id':'3'}]" ); // no more, so no change to cursorMark, and no new docs assertEquals(cursorMark, @@ -346,12 +346,12 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==10" ,"/response/start==0" - ,"/response/docs==[{'id':6},{'id':4},{'id':3},{'id':1},{'id':8},{'id':5}]" + ,"/response/docs==[{'id':'6'},{'id':'4'},{'id':'3'},{'id':'1'},{'id':'8'},{'id':'5'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==10" ,"/response/start==0" - ,"/response/docs==[{'id':0},{'id':9},{'id':7},{'id':2}]" + ,"/response/docs==[{'id':'0'},{'id':'9'},{'id':'7'},{'id':'2'}]" ); // no more, so no change to cursorMark, and no new docs assertEquals(cursorMark, @@ -370,27 +370,27 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==10" ,"/response/start==0" - ,"/response/docs==[{'id':2},{'id':9}]" + ,"/response/docs==[{'id':'2'},{'id':'9'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==10" ,"/response/start==0" - ,"/response/docs==[{'id':7},{'id':4}]" + ,"/response/docs==[{'id':'7'},{'id':'4'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==10" ,"/response/start==0" - ,"/response/docs==[{'id':3},{'id':8}]" + ,"/response/docs==[{'id':'3'},{'id':'8'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==10" ,"/response/start==0" - ,"/response/docs==[{'id':5},{'id':6}]" + ,"/response/docs==[{'id':'5'},{'id':'6'}]" ); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==10" ,"/response/start==0" - ,"/response/docs==[{'id':1},{'id':0}]" + ,"/response/docs==[{'id':'1'},{'id':'0'}]" ); // we've exactly exhausted all the results, but solr had no way of know that // no more, so no change to cursorMark, and no new docs @@ -410,7 +410,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==2" ,"/response/start==0" - ,"/response/docs==[{'id':7},{'id':3}]" + ,"/response/docs==[{'id':'7'},{'id':'3'}]" ); // no more, so no change to cursorMark, and no new docs assertEquals(cursorMark, @@ -455,7 +455,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==10" ,"/response/start==0" - ,"/response/docs==[{'id':1},{'id':3}]" + ,"/response/docs==[{'id':'1'},{'id':'3'}]" ); // delete the last guy we got assertU(delI("3")); @@ -463,7 +463,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==9" ,"/response/start==0" - ,"/response/docs==[{'id':4},{'id':6}]" + ,"/response/docs==[{'id':'4'},{'id':'6'}]" ); // delete the next guy we expect assertU(delI("0")); @@ -471,7 +471,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':5},{'id':8}]" + ,"/response/docs==[{'id':'5'},{'id':'8'}]" ); // update a doc we've already seen so it repeats assertU(adoc("id", "5", "str", "c")); @@ -479,7 +479,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':2},{'id':5}]" + ,"/response/docs==[{'id':'2'},{'id':'5'}]" ); // update the next doc we expect so it's now in the past assertU(adoc("id", "7", "str", "a")); @@ -487,7 +487,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" - ,"/response/docs==[{'id':9}]" + ,"/response/docs==[{'id':'9'}]" ); // no more, so no change to cursorMark, and no new docs assertEquals(cursorMark, @@ -684,7 +684,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark, nextCursorMark); } for (Map doc : docs) { - int id = ((Long)doc.get("id")).intValue(); + int id = Integer.parseInt(doc.get("id").toString()); assertFalse("walk already seen: " + id, ids.exists(id)); ids.put(id); assertFalse("id set bigger then max allowed ("+maxSize+"): " + ids.size(), @@ -773,7 +773,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { cursorMark, nextCursorMark); } for (Map doc : docs) { - int id = ((Long)doc.get("id")).intValue(); + int id = Integer.parseInt(doc.get("id").toString()); assertFalse("walk already seen: " + id, ids.exists(id)); ids.put(id); assertFalse("id set bigger then max allowed ("+maxSize+"): " + ids.size(), diff --git a/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java index 8d1b758c87c..faf67b42185 100644 --- a/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java @@ -85,9 +85,9 @@ public class DisMaxRequestHandlerTest extends SolrTestCaseJ4 { assertQ("basic cross field matching, boost on same field matching", req("cool stuff") ,"//*[@numFound='3']" - ,"//result/doc[1]/int[@name='id'][.='42']" - ,"//result/doc[2]/int[@name='id'][.='666']" - ,"//result/doc[3]/int[@name='id'][.='8675309']" + ,"//result/doc[1]/str[@name='id'][.='42']" + ,"//result/doc[2]/str[@name='id'][.='666']" + ,"//result/doc[3]/str[@name='id'][.='8675309']" ); assertQ("multi qf", @@ -112,9 +112,9 @@ public class DisMaxRequestHandlerTest extends SolrTestCaseJ4 { ,"bq", "subject:hell^400" ) ,"//*[@numFound='3']" - ,"//result/doc[1]/int[@name='id'][.='666']" - ,"//result/doc[2]/int[@name='id'][.='42']" - ,"//result/doc[3]/int[@name='id'][.='8675309']" + ,"//result/doc[1]/str[@name='id'][.='666']" + ,"//result/doc[2]/str[@name='id'][.='42']" + ,"//result/doc[3]/str[@name='id'][.='8675309']" ); assertQ("multi boost query", @@ -126,16 +126,16 @@ public class DisMaxRequestHandlerTest extends SolrTestCaseJ4 { , CommonParams.DEBUG_QUERY, "true" ) ,"//*[@numFound='3']" - ,"//result/doc[1]/int[@name='id'][.='666']" - ,"//result/doc[2]/int[@name='id'][.='8675309']" - ,"//result/doc[3]/int[@name='id'][.='42']" + ,"//result/doc[1]/str[@name='id'][.='666']" + ,"//result/doc[2]/str[@name='id'][.='8675309']" + ,"//result/doc[3]/str[@name='id'][.='42']" ); assertQ("minimum mm is three", req("cool stuff traveling") ,"//*[@numFound='2']" - ,"//result/doc[1]/int[@name='id'][. ='42']" - ,"//result/doc[2]/int[@name='id'][. ='666']" + ,"//result/doc[1]/str[@name='id'][. ='42']" + ,"//result/doc[2]/str[@name='id'][. ='666']" ); assertQ("at 4 mm allows one missing ", diff --git a/solr/core/src/test/org/apache/solr/SampleTest.java b/solr/core/src/test/org/apache/solr/SampleTest.java index cea23ee59e9..244272f7a6e 100644 --- a/solr/core/src/test/org/apache/solr/SampleTest.java +++ b/solr/core/src/test/org/apache/solr/SampleTest.java @@ -61,7 +61,7 @@ public class SampleTest extends SolrTestCaseJ4 { assertQ("couldn't find subject hoss", req("subject:Hoss") ,"//result[@numFound=1]" - ,"//int[@name='id'][.='4055']" + ,"//str[@name='id'][.='4055']" ); } @@ -94,7 +94,7 @@ public class SampleTest extends SolrTestCaseJ4 { assertQ("couldn't find subject hoss", req ,"//result[@numFound=1]" - ,"//int[@name='id'][.='4055']" + ,"//str[@name='id'][.='4055']" ); /* make your own LocalRequestFactory to build a request diff --git a/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java b/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java index ca7f9ee3da5..d187da81000 100644 --- a/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java +++ b/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java @@ -264,23 +264,23 @@ public class TestDistributedGrouping extends BaseDistributedSearchTestCase { // We validate distributed grouping with scoring as first sort. // note: this 'q' matches all docs and returns the 'id' as the score, which is unique and so our results should be deterministic. handle.put("maxScore", SKIP);// TODO see SOLR-6612 - query("q", "{!func}id", "rows", 100, "fl", "score,id," + i1, "group", "true", "group.field", i1, "group.limit", -1, "sort", i1 + " desc", "group.sort", "score desc"); // SOLR-2955 - query("q", "{!func}id", "rows", 100, "fl", "score,id," + i1, "group", "true", "group.field", i1, "group.limit", -1, "sort", "score desc, _docid_ asc, id asc"); - query("q", "{!func}id", "rows", 100, "fl", "score,id," + i1, "group", "true", "group.field", i1, "group.limit", -1); + query("q", "{!func}id_i1", "rows", 100, "fl", "score,id," + i1, "group", "true", "group.field", i1, "group.limit", -1, "sort", i1 + " desc", "group.sort", "score desc"); // SOLR-2955 + query("q", "{!func}id_i1", "rows", 100, "fl", "score,id," + i1, "group", "true", "group.field", i1, "group.limit", -1, "sort", "score desc, _docid_ asc, id asc"); + query("q", "{!func}id_i1", "rows", 100, "fl", "score,id," + i1, "group", "true", "group.field", i1, "group.limit", -1); // some explicit checks of non default sorting, and sort/group.sort with diff clauses - query("q", "{!func}id", "rows", 100, "fl", tlong + ",id," + i1, "group", "true", + query("q", "{!func}id_i1", "rows", 100, "fl", tlong + ",id," + i1, "group", "true", "group.field", i1, "group.limit", -1, "sort", tlong+" asc, id desc"); - query("q", "{!func}id", "rows", 100, "fl", tlong + ",id," + i1, "group", "true", + query("q", "{!func}id_i1", "rows", 100, "fl", tlong + ",id," + i1, "group", "true", "group.field", i1, "group.limit", -1, "sort", "id asc", "group.sort", tlong+" asc, id desc"); - query("q", "{!func}id", "rows", 100, "fl", tlong + ",id," + i1, "group", "true", + query("q", "{!func}id_i1", "rows", 100, "fl", tlong + ",id," + i1, "group", "true", "group.field", i1, "group.limit", -1, "sort", tlong+" asc, id desc", "group.sort", "id asc"); - rsp = query("q", "{!func}id", "fq", oddField+":[* TO *]", + rsp = query("q", "{!func}id_i1", "fq", oddField+":[* TO *]", "rows", 100, "fl", tlong + ",id," + i1, "group", "true", "group.field", i1, "group.limit", -1, "sort", tlong+" asc", @@ -293,8 +293,8 @@ public class TestDistributedGrouping extends BaseDistributedSearchTestCase { assertEquals(rsp.toString(), 232, nl.get("groupValue")); SolrDocumentList docs = (SolrDocumentList) nl.get("doclist"); assertEquals(docs.toString(), 5, docs.getNumFound()); - assertEquals(docs.toString(), 22, docs.get(0).getFirstValue("id")); - assertEquals(docs.toString(), 21, docs.get(4).getFirstValue("id")); + assertEquals(docs.toString(), "22", docs.get(0).getFirstValue("id")); + assertEquals(docs.toString(), "21", docs.get(4).getFirstValue("id")); diff --git a/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java b/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java index 7a91c83afa6..378ad0dcfc5 100644 --- a/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java +++ b/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java @@ -104,13 +104,13 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { // sint1_ml desc sort pos: 7 8 10 11 3 12 6 2 9 1 13 4 5 QueryResponse rsp = query("q","*:*", "sort", sint1_ml + " desc", "rows", "13"); - assertFieldValues(rsp.getResults(), id, 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11); + assertFieldValues(rsp.getResults(), "id_i", 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11); rsp = query("q","*:*", "sort", sint1_ml + " asc", "rows", "13"); - assertFieldValues(rsp.getResults(), id, 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10); + assertFieldValues(rsp.getResults(), "id_i", 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10); - rsp = query("q","*:*", "sort", sint1_ml + " desc," + id + " asc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", sint1_ml + " desc, id_i asc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11, 14, 15, 16, 17, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, @@ -119,8 +119,8 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149); - rsp = query("q","*:*", "sort", sint1_ml + " asc," + id + " desc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", sint1_ml + " asc, id_i desc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10, 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, @@ -135,13 +135,13 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { // long1_ml desc sort pos: 7 8 10 11 3 12 6 2 9 1 13 4 5 rsp = query("q","*:*", "sort", long1_ml + " desc", "rows", "13"); - assertFieldValues(rsp.getResults(), id, 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11); + assertFieldValues(rsp.getResults(), "id_i", 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11); rsp = query("q","*:*", "sort", long1_ml + " asc", "rows", "13"); - assertFieldValues(rsp.getResults(), id, 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10); + assertFieldValues(rsp.getResults(), "id_i", 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10); - rsp = query("q","*:*", "sort", long1_ml + " desc," + id + " asc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", long1_ml + " desc, id_i asc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11, 14, 15, 16, 17, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, @@ -150,8 +150,8 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149); - rsp = query("q","*:*", "sort", long1_ml + " asc," + id + " desc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", long1_ml + " asc, id_i desc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10, 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, @@ -167,13 +167,13 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { // string1_ml desc sort pos: 12 13 10 9 1 3 8 5 6 7 2 4 11 rsp = query("q","*:*", "sort", string1_ml + " desc", "rows", "13"); - assertFieldValues(rsp.getResults(), id, 5, 11, 6, 12, 8, 9, 10, 7, 4, 3, 13, 1, 2); + assertFieldValues(rsp.getResults(), "id_i", 5, 11, 6, 12, 8, 9, 10, 7, 4, 3, 13, 1, 2); rsp = query("q","*:*", "sort", string1_ml + " asc", "rows", "13"); - assertFieldValues(rsp.getResults(), id, 2, 1, 13, 3, 4, 7, 10, 9, 8, 12, 6, 11, 5); + assertFieldValues(rsp.getResults(), "id_i", 2, 1, 13, 3, 4, 7, 10, 9, 8, 12, 6, 11, 5); - rsp = query("q","*:*", "sort", string1_ml + " desc," + id + " asc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", string1_ml + " desc, id_i asc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", 5, 11, 6, 12, 8, 9, 10, 7, 4, 3, 13, 1, 2, // missing field string1_ml="a_s1", ascending id sort 14, 15, 16, 17, @@ -183,8 +183,8 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149); - rsp = query("q","*:*", "sort", string1_ml + " asc," + id + " desc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", string1_ml + " asc, id_i desc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", 2, 1, 13, 3, 4, 7, 10, 9, 8, 12, 6, 11, 5, // missing field string1_ml="a_s1", descending id sort 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, @@ -201,8 +201,8 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { // sint1_mf asc sort pos: 7 6 4 3 11 2 8 12 5 13 1 10 9 // sint1_mf desc sort pos: 7 8 10 11 3 12 6 2 9 1 13 4 5 - QueryResponse rsp = query("q","*:*", "sort", sint1_mf + " desc," + id + " asc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + QueryResponse rsp = query("q","*:*", "sort", sint1_mf + " desc, id_i asc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", 14, 15, 16, 17, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, @@ -211,8 +211,8 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11); - rsp = query("q","*:*", "sort", sint1_mf + " asc," + id + " desc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", sint1_mf + " asc, id_i desc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, @@ -227,8 +227,8 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { // long1_mf asc sort pos: 7 6 4 3 11 2 8 12 5 13 1 10 9 // long1_mf desc sort pos: 7 8 10 11 3 12 6 2 9 1 13 4 5 - rsp = query("q","*:*", "sort", long1_mf + " desc," + id + " asc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", long1_mf + " desc, id_i asc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", 14, 15, 16, 17, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, @@ -237,8 +237,8 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11); - rsp = query("q","*:*", "sort", long1_mf + " asc," + id + " desc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", long1_mf + " asc, id_i desc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, @@ -253,8 +253,8 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { // string1_mf asc sort pos: 2 1 4 5 13 11 6 9 8 7 12 10 3 // string1_mf desc sort pos: 12 13 10 9 1 3 8 5 6 7 2 4 11 - rsp = query("q","*:*", "sort", string1_mf + " desc," + id + " asc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", string1_mf + " desc, id_i asc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", // missing field string1_mf="a_s1_mf", ascending id sort 14, 15, 16, 17, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, @@ -264,8 +264,8 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 5, 11, 6, 12, 8, 9, 10, 7, 4, 3, 13, 1, 2); - rsp = query("q","*:*", "sort", string1_mf + " asc," + id + " desc", "rows", "200"); - assertFieldValues(rsp.getResults(), id, + rsp = query("q","*:*", "sort", string1_mf + " asc, id_i desc", "rows", "200"); + assertFieldValues(rsp.getResults(), "id_i", // missing field string1_mf="a_s1_mf", descending id sort 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, diff --git a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java index fc4884753a0..cde3296443a 100644 --- a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java +++ b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java @@ -242,7 +242,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { "facet.field", tdate_b, "facet.field", tdate_a); assertEquals(2, rsp.getFacetFields().size()); - String facetQuery = "id:[1 TO 15]"; + String facetQuery = "id_i1:[1 TO 15]"; // simple range facet on one field query("q",facetQuery, "rows",100, "facet","true", @@ -390,7 +390,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { // we want a random query and not just *:* so we'll get zero counts in facets also // TODO: do a better random query - String q = random().nextBoolean() ? "*:*" : "id:(1 3 5 7 9 11 13) OR id:[100 TO " + random().nextInt(50) + "]"; + String q = random().nextBoolean() ? "*:*" : "id:(1 3 5 7 9 11 13) OR id_i1:[100 TO " + random().nextInt(50) + "]"; int nolimit = random().nextBoolean() ? -1 : 10000; // these should be equivalent @@ -412,7 +412,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { ,"facet.field","{!key=other ex=b}"+t1 ,"facet.field","{!key=again ex=a,b}"+t1 ,"facet.field",t1 - ,"fq","{!tag=a}id:[1 TO 7]", "fq","{!tag=b}id:[3 TO 9]" + ,"fq","{!tag=a}id_i1:[1 TO 7]", "fq","{!tag=b}id_i1:[3 TO 9]" ); queryAndCompareUIF("q", "*:*", "facet", "true", "facet.field", "{!ex=t1}SubjectTerms_mfacet", "fq", "{!tag=t1}SubjectTerms_mfacet:(test 1)", "facet.limit", "10", "facet.mincount", "1"); @@ -522,7 +522,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { "stats.field", "{!key=special_key}stats_dt"); query("q","*:*", "sort",i1+" desc", "stats", "true", "f.stats_dt.stats.calcdistinct", "true", - "fq", "{!tag=xxx}id:[3 TO 9]", + "fq", "{!tag=xxx}id_i1:[3 TO 9]", "stats.field", "{!key=special_key}stats_dt", "stats.field", "{!ex=xxx}stats_dt"); @@ -920,20 +920,20 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { handle.put("time", SKIPVAL); handle.put("track", SKIP); //track is not included in single node search query("q","now their fox sat had put","fl","*,score",CommonParams.DEBUG_QUERY, "true"); - query("q", "id:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); - query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING); - query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS); - query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY); // SOLR-6545, wild card field list indexr(id, "19", "text", "d", "cat_a_sS", "1" ,t1, "2"); commit(); rsp = query("q", "id:19", "fl", "id", "fl", "*a_sS"); - assertFieldValues(rsp.getResults(), "id", 19); + assertFieldValues(rsp.getResults(), "id", "19"); rsp = query("q", "id:19", "fl", "id," + t1 + ",cat*"); - assertFieldValues(rsp.getResults(), "id", 19); + assertFieldValues(rsp.getResults(), "id", "19"); // Check Info is added to for each shard ModifiableSolrParams q = new ModifiableSolrParams(); diff --git a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java index cb485d0ca81..ec1f16c09ad 100644 --- a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java +++ b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java @@ -150,7 +150,7 @@ public class TestTolerantSearch extends SolrJettyTestBase { } } assertTrue(foundError); - assertEquals(1, response.getResults().get(0).getFieldValue("id")); + assertEquals("1", response.getResults().get(0).getFieldValue("id")); assertEquals("batman", response.getResults().get(0).getFirstValue("subject")); unIgnoreException("Dummy exception in BadResponseWriter"); } @@ -199,7 +199,7 @@ public class TestTolerantSearch extends SolrJettyTestBase { } assertTrue(foundError); - assertEquals(1, response.getResults().get(0).getFieldValue("id")); + assertEquals("1", response.getResults().get(0).getFieldValue("id")); assertEquals("batman", response.getResults().get(0).getFirstValue("subject")); unIgnoreException("Dummy exception in BadResponseWriter"); } diff --git a/solr/core/src/test/org/apache/solr/analysis/PathHierarchyTokenizerFactoryTest.java b/solr/core/src/test/org/apache/solr/analysis/PathHierarchyTokenizerFactoryTest.java index d079166f7ae..ded7f18b610 100644 --- a/solr/core/src/test/org/apache/solr/analysis/PathHierarchyTokenizerFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/analysis/PathHierarchyTokenizerFactoryTest.java @@ -54,21 +54,21 @@ public class PathHierarchyTokenizerFactoryTest extends SolrTestCaseJ4 { assertQ(req("{!field f=cat_path}Books/NonFic") ,"//*[@numFound='4']" - ,"//int[@name='id' and .='40']" - ,"//int[@name='id' and .='41']" - ,"//int[@name='id' and .='42']" - ,"//int[@name='id' and .='43']" + ,"//str[@name='id' and .='40']" + ,"//str[@name='id' and .='41']" + ,"//str[@name='id' and .='42']" + ,"//str[@name='id' and .='43']" ); assertQ(req("{!field f=cat_path}Books/NonFic/Law") ,"//*[@numFound='2']" - ,"//int[@name='id' and .='41']" - ,"//int[@name='id' and .='42']" + ,"//str[@name='id' and .='41']" + ,"//str[@name='id' and .='42']" ); assertQ(req("{!field f=cat_path}Books/NonFic/Science") ,"//*[@numFound='2']" - ,"//int[@name='id' and .='42']" - ,"//int[@name='id' and .='43']" + ,"//str[@name='id' and .='42']" + ,"//str[@name='id' and .='43']" ); } @@ -76,21 +76,21 @@ public class PathHierarchyTokenizerFactoryTest extends SolrTestCaseJ4 { assertQ(req("{!field f=cat_ancestor}Books/NonFic/Science") ,"//*[@numFound='2']" - ,"//int[@name='id' and .='40']" - ,"//int[@name='id' and .='42']" + ,"//str[@name='id' and .='40']" + ,"//str[@name='id' and .='42']" ); assertQ(req("{!field f=cat_ancestor}Books/NonFic/Law") ,"//*[@numFound='3']" - ,"//int[@name='id' and .='40']" - ,"//int[@name='id' and .='41']" - ,"//int[@name='id' and .='42']" + ,"//str[@name='id' and .='40']" + ,"//str[@name='id' and .='41']" + ,"//str[@name='id' and .='42']" ); assertQ(req("{!field f=cat_ancestor}Books/NonFic/Science/Physics") ,"//*[@numFound='3']" - ,"//int[@name='id' and .='40']" - ,"//int[@name='id' and .='42']" - ,"//int[@name='id' and .='43']" + ,"//str[@name='id' and .='40']" + ,"//str[@name='id' and .='42']" + ,"//str[@name='id' and .='43']" ); } } diff --git a/solr/core/src/test/org/apache/solr/analysis/TestCharFilters.java b/solr/core/src/test/org/apache/solr/analysis/TestCharFilters.java index 984ad125f41..646bdbb11dc 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestCharFilters.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestCharFilters.java @@ -46,12 +46,12 @@ public class TestCharFilters extends SolrTestCaseJ4 { assertQ("Query analysis: ", req("fl", "id", "q", "content:ab", "sort", "id asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=1]" + "//result/doc[1]/str[@name='id'][.=1]" ); assertQ("Query analysis: ", req("fl", "id", "q", "content:aba", "sort", "id asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=2]" + "//result/doc[1]/str[@name='id'][.=2]" ); } @@ -65,12 +65,12 @@ public class TestCharFilters extends SolrTestCaseJ4 { assertQ("Index analysis: ", req("fl", "id", "q", "content2:aab", "sort", "id asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=3]" + "//result/doc[1]/str[@name='id'][.=3]" ); assertQ("Index analysis: ", req("fl", "id", "q", "content2:aabaa", "sort", "id asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=4]" ); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java index c8e92fc14f4..02692a0b3c6 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java @@ -463,10 +463,10 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase { handle.put("track", SKIP); query("q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true"); - query("q", "id:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); - query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING); - query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS); - query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 18caa5858c2..3ed6f74848c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -271,7 +271,7 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase { ,"facet.field","{!key=other ex=b}"+t1 ,"facet.field","{!key=again ex=a,b}"+t1 ,"facet.field",t1 - ,"fq","{!tag=a}id:[1 TO 7]", "fq","{!tag=b}id:[3 TO 9]"} + ,"fq","{!tag=a}id_i1:[1 TO 7]", "fq","{!tag=b}id_i1:[3 TO 9]"} ); query(false, new Object[] {"q", "*:*", "facet", "true", "facet.field", "{!ex=t1}SubjectTerms_mfacet", "fq", "{!tag=t1}SubjectTerms_mfacet:(test 1)", "facet.limit", "10", "facet.mincount", "1"}); @@ -321,10 +321,10 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase { handle.put("time", SKIPVAL); handle.put("track", SKIP); query(false, new Object[] {"q","now their fox sat had put","fl","*,score",CommonParams.DEBUG_QUERY, "true"}); - query(false, new Object[] {"q", "id:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"}); - query(false, new Object[] {"q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING}); - query(false, new Object[] {"q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS}); - query(false, new Object[] {"q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY}); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"}); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING}); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS}); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY}); // try add commitWithin long before = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java index f48f76b5736..7a9dbdb4cac 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java @@ -70,10 +70,10 @@ public class BasicZkTest extends AbstractZkTestCase { assertU("does commit work?", commit()); assertQ("backslash escaping semicolon", request("id:42 AND val_s:aa\\;bb"), - "//*[@numFound='1']", "//int[@name='id'][.='42']"); + "//*[@numFound='1']", "//str[@name='id'][.='42']"); assertQ("quote escaping semicolon", request("id:42 AND val_s:\"aa;bb\""), - "//*[@numFound='1']", "//int[@name='id'][.='42']"); + "//*[@numFound='1']", "//str[@name='id'][.='42']"); assertQ("no escaping semicolon", request("id:42 AND val_s:aa"), "//*[@numFound='0']"); diff --git a/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java b/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java index 0742ac7f5c5..5de766f3b62 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java @@ -556,7 +556,7 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { if (ids.size() < numInitialDocs) { message.append("Missing doc(s): "); for (SolrInputDocument doc : initialDocs) { - int id = ((Integer)doc.get("id").getValue()).intValue(); + int id = Integer.parseInt(doc.getFieldValue("id").toString()); if ( ! ids.exists(id)) { QueryResponse rsp = cloudClient.query(params("q", "id:" + id, "rows", "1")); @@ -664,12 +664,12 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { * "id" of the list of documents returned matches the expected list * @see org.apache.solr.client.solrj.SolrClient#query */ - private void assertDocList(QueryResponse rsp, Object... ids) { + private void assertDocList(QueryResponse rsp, int... ids) { SolrDocumentList docs = extractDocList(rsp); assertEquals("Wrong number of docs in response", ids.length, docs.size()); int i = 0; - for (Object id : ids) { - assertEquals(rsp.toString(), id, docs.get(i).get("id")); + for (int id : ids) { + assertEquals(rsp.toString(), ""+id, docs.get(i).get("id")); i++; } } @@ -730,7 +730,7 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { } for (SolrDocument doc : docs) { - int id = ((Integer)doc.get("id")).intValue(); + int id = Integer.parseInt(doc.getFieldValue("id").toString()); if (ids.exists(id)) { String msg = "(" + p + ") walk already seen: " + id; try { diff --git a/solr/core/src/test/org/apache/solr/cloud/DistribJoinFromCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/DistribJoinFromCollectionTest.java index cfebef256a6..383c1ae6f1e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DistribJoinFromCollectionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DistribJoinFromCollectionTest.java @@ -62,7 +62,7 @@ public class DistribJoinFromCollectionTest extends SolrCloudTestCase{ private static String toColl = "to_2x2"; private static String fromColl = "from_1x4"; - private static Integer toDocId; + private static String toDocId; @BeforeClass public static void setupCluster() throws Exception { @@ -140,7 +140,7 @@ public class DistribJoinFromCollectionTest extends SolrCloudTestCase{ log.info("DistribJoinFromCollectionTest succeeded ... shutting down now!"); } - private void testJoins(String toColl, String fromColl, Integer toDocId, boolean isScoresTest) + private void testJoins(String toColl, String fromColl, String toDocId, boolean isScoresTest) throws SolrServerException, IOException { // verify the join with fromIndex works final String fromQ = "match_s:c^2"; @@ -219,12 +219,12 @@ public class DistribJoinFromCollectionTest extends SolrCloudTestCase{ } } - protected static Integer indexDoc(String collection, int id, String joinField, String matchField, String getField) throws Exception { + protected static String indexDoc(String collection, int id, String joinField, String matchField, String getField) throws Exception { UpdateRequest up = new UpdateRequest(); up.setCommitWithin(50); up.setParam("collection", collection); SolrInputDocument doc = new SolrInputDocument(); - Integer docId = new Integer(id); + String docId = "" + id; doc.addField("id", docId); doc.addField("join_s", joinField); if (matchField != null) diff --git a/solr/core/src/test/org/apache/solr/cloud/FullThrottleStoppableIndexingThread.java b/solr/core/src/test/org/apache/solr/cloud/FullThrottleStoppableIndexingThread.java index b9e177a0d9f..909f39afab2 100644 --- a/solr/core/src/test/org/apache/solr/cloud/FullThrottleStoppableIndexingThread.java +++ b/solr/core/src/test/org/apache/solr/cloud/FullThrottleStoppableIndexingThread.java @@ -153,4 +153,4 @@ class FullThrottleStoppableIndexingThread extends StoppableIndexingThread { } } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java index 6c7de885e58..2f2ccb4ea78 100644 --- a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java @@ -598,7 +598,7 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase { @SuppressWarnings("rawtypes") protected void assertDocExists(HttpSolrClient solr, String coll, String docId) throws Exception { NamedList rsp = realTimeGetDocId(solr, docId); - String match = JSONTestUtil.matchObj("/id", rsp.get("doc"), new Integer(docId)); + String match = JSONTestUtil.matchObj("/id", rsp.get("doc"), docId); assertTrue("Doc with id=" + docId + " not found in " + solr.getBaseURL() + " due to: " + match + "; rsp="+rsp, match == null); } diff --git a/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java b/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java index 9824e3da4ac..118d5ede9ea 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java +++ b/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java @@ -106,8 +106,10 @@ class SegmentTerminateEarlyTestState { // check correctness of the first result if (rsp.getResults().getNumFound() > 0) { final SolrDocument solrDocument0 = rsp.getResults().get(0); - TestMiniSolrCloudCluster.assertTrue(KEY_FIELD +" of ("+solrDocument0+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")", - maxTimestampDocKeys.contains(solrDocument0.getFieldValue(KEY_FIELD))); + final Integer idAsInt = Integer.parseInt(solrDocument0.getFieldValue(KEY_FIELD).toString()); + TestMiniSolrCloudCluster.assertTrue + (KEY_FIELD +"="+idAsInt+" of ("+solrDocument0+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")", + maxTimestampDocKeys.contains(idAsInt)); TestMiniSolrCloudCluster.assertEquals(ODD_FIELD, oddFieldValue, solrDocument0.getFieldValue(ODD_FIELD)); } // check segmentTerminatedEarly flag @@ -136,8 +138,10 @@ class SegmentTerminateEarlyTestState { // check correctness of the first result if (rsp.getResults().getNumFound() > 0) { final SolrDocument solrDocument0 = rsp.getResults().get(0); - TestMiniSolrCloudCluster.assertTrue(KEY_FIELD +" of ("+solrDocument0+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")", - maxTimestampDocKeys.contains(solrDocument0.getFieldValue(KEY_FIELD))); + final Integer idAsInt = Integer.parseInt(solrDocument0.getFieldValue(KEY_FIELD).toString()); + TestMiniSolrCloudCluster.assertTrue + (KEY_FIELD +"="+idAsInt+" of ("+solrDocument0+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")", + maxTimestampDocKeys.contains(idAsInt)); TestMiniSolrCloudCluster.assertEquals(ODD_FIELD, oddFieldValue, rsp.getResults().get(0).getFieldValue(ODD_FIELD)); } // check segmentTerminatedEarly flag @@ -182,8 +186,10 @@ class SegmentTerminateEarlyTestState { // check correctness of the first result if (rsp.getResults().getNumFound() > 0) { final SolrDocument solrDocument0 = rsp.getResults().get(0); - TestMiniSolrCloudCluster.assertTrue(KEY_FIELD +" of ("+solrDocument0+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")", - maxTimestampDocKeys.contains(solrDocument0.getFieldValue(KEY_FIELD))); + final Integer idAsInt = Integer.parseInt(solrDocument0.getFieldValue(KEY_FIELD).toString()); + TestMiniSolrCloudCluster.assertTrue + (KEY_FIELD +"="+idAsInt+" of ("+solrDocument0+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")", + maxTimestampDocKeys.contains(idAsInt)); TestMiniSolrCloudCluster.assertEquals(ODD_FIELD, oddFieldValue, rsp.getResults().get(0).getFieldValue(ODD_FIELD)); } // check segmentTerminatedEarly flag @@ -226,8 +232,10 @@ class SegmentTerminateEarlyTestState { // check correctness of the first result if (rsp.getGroupResponse().getValues().get(0).getMatches() > 0) { final SolrDocument solrDocument = rsp.getGroupResponse().getValues().get(0).getValues().get(0).getResult().get(0); - TestMiniSolrCloudCluster.assertTrue(KEY_FIELD +" of ("+solrDocument+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")", - maxTimestampDocKeys.contains(solrDocument.getFieldValue(KEY_FIELD))); + final Integer idAsInt = Integer.parseInt(solrDocument.getFieldValue(KEY_FIELD).toString()); + TestMiniSolrCloudCluster.assertTrue + (KEY_FIELD +"="+idAsInt+" of ("+solrDocument+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")", + maxTimestampDocKeys.contains(idAsInt)); TestMiniSolrCloudCluster.assertEquals(ODD_FIELD, oddFieldValue, solrDocument.getFieldValue(ODD_FIELD)); } // check segmentTerminatedEarly flag @@ -251,8 +259,10 @@ class SegmentTerminateEarlyTestState { // check correctness of the first result if (rsp.getResults().getNumFound() > 0) { final SolrDocument solrDocument0 = rsp.getResults().get(0); - TestMiniSolrCloudCluster.assertTrue(KEY_FIELD +" of ("+solrDocument0+") is not in minTimestampDocKeys("+minTimestampDocKeys+")", - minTimestampDocKeys.contains(solrDocument0.getFieldValue(KEY_FIELD))); + final Integer idAsInt = Integer.parseInt(solrDocument0.getFieldValue(KEY_FIELD).toString()); + TestMiniSolrCloudCluster.assertTrue + (KEY_FIELD +"="+idAsInt+" of ("+solrDocument0+") is not in minTimestampDocKeys("+minTimestampDocKeys+")", + minTimestampDocKeys.contains(idAsInt)); TestMiniSolrCloudCluster.assertEquals(ODD_FIELD, oddFieldValue, solrDocument0.getFieldValue(ODD_FIELD)); } // check segmentTerminatedEarly flag diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java index 7bf461667b5..830bef2248f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java @@ -335,4 +335,4 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase { return null; } } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/core/HelloStream.java b/solr/core/src/test/org/apache/solr/core/HelloStream.java index d1c5ced21f5..be285e5886d 100644 --- a/solr/core/src/test/org/apache/solr/core/HelloStream.java +++ b/solr/core/src/test/org/apache/solr/core/HelloStream.java @@ -97,4 +97,4 @@ public class HelloStream extends TupleStream implements Expressible{ .withExpressionType(Explanation.ExpressionType.STREAM_SOURCE) .withExpression("--non-expressible--"); } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/core/MockInfoBean.java b/solr/core/src/test/org/apache/solr/core/MockInfoBean.java index dfa94ae1121..a4334bc6cf7 100644 --- a/solr/core/src/test/org/apache/solr/core/MockInfoBean.java +++ b/solr/core/src/test/org/apache/solr/core/MockInfoBean.java @@ -68,4 +68,4 @@ class MockInfoBean implements SolrInfoBean, SolrMetricProducer { }); manager.registerGauge(this, registryName, metricsMap, true, getClass().getSimpleName(), getCategory().toString(), scope); } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/core/SOLR749Test.java b/solr/core/src/test/org/apache/solr/core/SOLR749Test.java index 446c5062f17..2eda77d4f85 100644 --- a/solr/core/src/test/org/apache/solr/core/SOLR749Test.java +++ b/solr/core/src/test/org/apache/solr/core/SOLR749Test.java @@ -52,29 +52,29 @@ public class SOLR749Test extends SolrTestCaseJ4 { // NOTE: we can't rely on the default lucene syntax because "FooQParser" is registered as "lucene" assertQ(req("q","{!notfoo}*:*"), "//result[@numFound=100]"); - assertQ(req("q","{!notfoo}id:[* TO 49]"), "//result[@numFound=50]"); + assertQ(req("q","{!notfoo}id_i1:[* TO 49]"), "//result[@numFound=50]"); try { assertQ("query wrapped in boost func should only eval func for query matches", - req("q","{!boost b=$boostFunc defType=notfoo}id:[* TO 49]", + req("q","{!boost b=$boostFunc defType=notfoo}id_i1:[* TO 49]", "boostFunc", "countUsage('boost_func',3.4)"), "//result[@numFound=50]"); assertEquals(50, CountUsageValueSourceParser.getAndClearCount("boost_func")); assertQ("func query that is filtered should be evaled only for filtered docs", - req("q","{!func}product(id,countUsage('func_q',4.5))", - "fq", "{!notfoo}id:[30 TO 59]"), + req("q","{!func}product(id_i1,countUsage('func_q',4.5))", + "fq", "{!notfoo}id_i1:[30 TO 59]"), "//result[@numFound=30]"); assertEquals(30, CountUsageValueSourceParser.getAndClearCount("func_q")); assertQ("func query that wraps a query which is also used as a should be evaled only for filtered docs", req("q","{!func}product(query($qq),countUsage('func_q_wrapping_fq',4.5))", - "qq", "{!notfoo}id:[20 TO 39]", + "qq", "{!notfoo}id_i1:[20 TO 39]", "fq", "{!query v=$qq}"), "//result[@numFound=20]"); assertEquals(20, CountUsageValueSourceParser.getAndClearCount("func_q_wrapping_fq")); assertQ("frange in complex bq w/ other mandatory clauses to check skipping", - req("q","{!notfoo}(+id:[20 TO 39] -id:25 +{!frange l=4.5 u=4.5 v='countUsage(frange_in_bq,4.5)'})"), + req("q","{!notfoo}(+id_i1:[20 TO 39] -id:25 +{!frange l=4.5 u=4.5 v='countUsage(frange_in_bq,4.5)'})"), "//result[@numFound=19]"); // don't assume specific clause evaluation ordering. diff --git a/solr/core/src/test/org/apache/solr/core/TestJmxIntegration.java b/solr/core/src/test/org/apache/solr/core/TestJmxIntegration.java index db941f7efb3..70df3c52177 100644 --- a/solr/core/src/test/org/apache/solr/core/TestJmxIntegration.java +++ b/solr/core/src/test/org/apache/solr/core/TestJmxIntegration.java @@ -197,4 +197,4 @@ public class TestJmxIntegration extends AbstractSolrTestCase { log.info("After Reload: Size of infoRegistry: " + registrySize + " MBeans: " + newNumberOfObjects); assertEquals("Number of registered MBeans is not the same as info registry size", registrySize, newNumberOfObjects); } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java index 592200dc9a1..b65863eba1f 100644 --- a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java +++ b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java @@ -305,4 +305,4 @@ public class TestSolrCloudSnapshots extends SolrCloudTestCase { } return result; } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java index 7a9b0bb6c5f..d508050b3d7 100644 --- a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java +++ b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java @@ -309,4 +309,4 @@ public class TestSolrCoreSnapshots extends SolrCloudTestCase { return Collections.emptyList(); } } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java index 8e9c17016d1..6da06b1d384 100644 --- a/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java @@ -86,19 +86,19 @@ public class MoreLikeThisHandlerTest extends SolrTestCaseJ4 { SolrQueryRequest mltreq = new LocalSolrQueryRequest( core, params); assertQ("morelikethis - tom cruise",mltreq - ,"//result/doc[1]/int[@name='id'][.='46']" - ,"//result/doc[2]/int[@name='id'][.='43']"); + ,"//result/doc[1]/str[@name='id'][.='46']" + ,"//result/doc[2]/str[@name='id'][.='43']"); params.set(MoreLikeThisParams.BOOST, "true"); mltreq.close(); mltreq = new LocalSolrQueryRequest( core, params); assertQ("morelikethis - tom cruise",mltreq - ,"//result/doc[1]/int[@name='id'][.='46']" - ,"//result/doc[2]/int[@name='id'][.='43']"); + ,"//result/doc[1]/str[@name='id'][.='46']" + ,"//result/doc[2]/str[@name='id'][.='43']"); params.set(CommonParams.Q, "id:44"); mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); assertQ("morelike this - harrison ford",mltreq - ,"//result/doc[1]/int[@name='id'][.='45']"); + ,"//result/doc[1]/str[@name='id'][.='45']"); // test MoreLikeThis debug params.set(CommonParams.DEBUG_QUERY, "true"); @@ -115,14 +115,14 @@ public class MoreLikeThisHandlerTest extends SolrTestCaseJ4 { params.set(CommonParams.Q, "{!field f=id}44"); mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); assertQ(mltreq - ,"//result/doc[1]/int[@name='id'][.='45']"); + ,"//result/doc[1]/str[@name='id'][.='45']"); params.set(CommonParams.Q, "id:42"); params.set(MoreLikeThisParams.QF,"name^5.0 subword^0.1"); mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); assertQ("morelikethis with weights",mltreq - ,"//result/doc[1]/int[@name='id'][.='43']" - ,"//result/doc[2]/int[@name='id'][.='46']"); + ,"//result/doc[1]/str[@name='id'][.='43']" + ,"//result/doc[2]/str[@name='id'][.='46']"); // test that qparser plugins work w/ the MoreLikeThisHandler @@ -130,14 +130,14 @@ public class MoreLikeThisHandlerTest extends SolrTestCaseJ4 { params.set(CommonParams.Q, "{!field f=id}44"); mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); assertQ(mltreq - ,"//result/doc[1]/int[@name='id'][.='45']"); + ,"//result/doc[1]/str[@name='id'][.='45']"); // test that debugging works (test for MoreLikeThis*Handler*) params.set(CommonParams.QT, "/mlt"); params.set(CommonParams.DEBUG_QUERY, "true"); mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); assertQ(mltreq - ,"//result/doc[1]/int[@name='id'][.='45']" + ,"//result/doc[1]/str[@name='id'][.='45']" ,"//lst[@name='debug']/lst[@name='explain']" ); diff --git a/solr/core/src/test/org/apache/solr/handler/StandardRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/StandardRequestHandlerTest.java index 82e8cdeff46..668fefd4203 100644 --- a/solr/core/src/test/org/apache/solr/handler/StandardRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/StandardRequestHandlerTest.java @@ -49,16 +49,16 @@ public class StandardRequestHandlerTest extends AbstractSolrTestCase { assertQ(req("q", "title:test", "sort","val_s1 asc") ,"//*[@numFound='3']" - ,"//result/doc[1]/int[@name='id'][.='10']" - ,"//result/doc[2]/int[@name='id'][.='11']" - ,"//result/doc[3]/int[@name='id'][.='12']" + ,"//result/doc[1]/str[@name='id'][.='10']" + ,"//result/doc[2]/str[@name='id'][.='11']" + ,"//result/doc[3]/str[@name='id'][.='12']" ); assertQ(req("q", "title:test", "sort","val_s1 desc") ,"//*[@numFound='3']" - ,"//result/doc[1]/int[@name='id'][.='12']" - ,"//result/doc[2]/int[@name='id'][.='11']" - ,"//result/doc[3]/int[@name='id'][.='10']" + ,"//result/doc[1]/str[@name='id'][.='12']" + ,"//result/doc[2]/str[@name='id'][.='11']" + ,"//result/doc[3]/str[@name='id'][.='10']" ); // Make sure score parsing works @@ -73,16 +73,16 @@ public class StandardRequestHandlerTest extends AbstractSolrTestCase { // Using legacy ';' param assertQ(req("q", "title:test; val_s1 desc", "defType","lucenePlusSort") ,"//*[@numFound='3']" - ,"//result/doc[1]/int[@name='id'][.='12']" - ,"//result/doc[2]/int[@name='id'][.='11']" - ,"//result/doc[3]/int[@name='id'][.='10']" + ,"//result/doc[1]/str[@name='id'][.='12']" + ,"//result/doc[2]/str[@name='id'][.='11']" + ,"//result/doc[3]/str[@name='id'][.='10']" ); assertQ(req("q", "title:test; val_s1 asc", "defType","lucenePlusSort") ,"//*[@numFound='3']" - ,"//result/doc[1]/int[@name='id'][.='10']" - ,"//result/doc[2]/int[@name='id'][.='11']" - ,"//result/doc[3]/int[@name='id'][.='12']" + ,"//result/doc[1]/str[@name='id'][.='10']" + ,"//result/doc[2]/str[@name='id'][.='11']" + ,"//result/doc[3]/str[@name='id'][.='12']" ); } } diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java index 4889c9071a1..78bdbe0a5bc 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java @@ -172,7 +172,7 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase { assert(tuple.get("str_s").equals("a")); //Assert field order - assertResponseContains(clients.get(0), sParams, "{\"docs\":[{\"id\":8,\"field_i\":60,\"str_s\":\"c\"}"); + assertResponseContains(clients.get(0), sParams, "{\"docs\":[{\"id\":\"8\",\"field_i\":60,\"str_s\":\"c\"}"); //Test unlimited unsorted result. Should sort on _version_ desc sParams = mapParams(CommonParams.QT, "/sql", "stmt", "select id, field_i, str_s from collection1 where text='XXXX'"); @@ -375,7 +375,7 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase { assertEquals(1, tuples.size()); Tuple tuple = tuples.get(0); - assertEquals(1L, tuple.get("id")); + assertEquals("1", tuple.get("id")); // Not Equals <> sParams = mapParams(CommonParams.QT, "/sql", @@ -387,19 +387,19 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase { assertEquals(7, tuples.size()); tuple = tuples.get(0); - assertEquals(2L, tuple.get("id")); + assertEquals("2", tuple.get("id")); tuple = tuples.get(1); - assertEquals(3L, tuple.get("id")); + assertEquals("3", tuple.get("id")); tuple = tuples.get(2); - assertEquals(4L, tuple.get("id")); + assertEquals("4", tuple.get("id")); tuple = tuples.get(3); - assertEquals(5L, tuple.get("id")); + assertEquals("5", tuple.get("id")); tuple = tuples.get(4); - assertEquals(6L, tuple.get("id")); + assertEquals("6", tuple.get("id")); tuple = tuples.get(5); - assertEquals(7L, tuple.get("id")); + assertEquals("7", tuple.get("id")); tuple = tuples.get(6); - assertEquals(8L, tuple.get("id")); + assertEquals("8", tuple.get("id")); // TODO requires different Calcite SQL conformance level // Not Equals != @@ -436,7 +436,7 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase { assertEquals(1, tuples.size()); tuple = tuples.get(0); - assertEquals(1L, tuple.get("id")); + assertEquals("1", tuple.get("id")); // Less than equal sParams = mapParams(CommonParams.QT, "/sql", @@ -448,9 +448,9 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase { assertEquals(2, tuples.size()); tuple = tuples.get(0); - assertEquals(1L, tuple.get("id")); + assertEquals("1", tuple.get("id")); tuple = tuples.get(1); - assertEquals(2L, tuple.get("id")); + assertEquals("2", tuple.get("id")); // Greater than sParams = mapParams(CommonParams.QT, "/sql", @@ -462,7 +462,7 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase { assertEquals(1, tuples.size()); tuple = tuples.get(0); - assertEquals(8L, tuple.get("id")); + assertEquals("8", tuple.get("id")); // Greater than equal sParams = mapParams(CommonParams.QT, "/sql", @@ -474,9 +474,9 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase { assertEquals(2, tuples.size()); tuple = tuples.get(0); - assertEquals(7L, tuple.get("id")); + assertEquals("7", tuple.get("id")); tuple = tuples.get(1); - assertEquals(8L, tuple.get("id")); + assertEquals("8", tuple.get("id")); } finally { delete(); @@ -511,7 +511,7 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase { assert(tuples.size() == 8); Tuple tuple; - + tuple = tuples.get(0); assert(tuple.getLong("id") == 8); assert(tuple.getLong("Field_i") == 60); diff --git a/solr/core/src/test/org/apache/solr/handler/XsltUpdateRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/XsltUpdateRequestHandlerTest.java index 7062b43c14d..fb7bd1999bf 100644 --- a/solr/core/src/test/org/apache/solr/handler/XsltUpdateRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/XsltUpdateRequestHandlerTest.java @@ -90,7 +90,7 @@ public class XsltUpdateRequestHandlerTest extends SolrTestCaseJ4 { assertQ("test document was correctly committed", req("q","*:*") , "//result[@numFound='1']" - , "//int[@name='id'][.='12345']" + , "//str[@name='id'][.='12345']" ); } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java index c7622f640cc..40d7821efed 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java @@ -96,4 +96,4 @@ public class MBeansHandlerTest extends SolrTestCaseJ4 { assertTrue("external entity ignored properly", true); } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java index 1a959a45116..34ab7eb15c3 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java @@ -70,4 +70,4 @@ public class PropertiesRequestHandlerTest extends SolrTestCaseJ4 { return (NamedList>>) parser.processResponse(new StringReader(xml)).get("system.properties"); } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java index 5ca527facb1..5633c46f551 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java @@ -102,10 +102,10 @@ public class DistributedExpandComponentTest extends BaseDistributedSearchTestCas QueryResponse rsp = queryServer(params); Map results = rsp.getExpandedResults(); assertExpandGroups(results, "group1","group2", "group3", "group4"); - assertExpandGroupCountAndOrder("group1", 2, results, "1.0", "7.0"); - assertExpandGroupCountAndOrder("group2", 2, results, "5.0", "8.0"); - assertExpandGroupCountAndOrder("group3", 2, results, "11.0", "9.0"); - assertExpandGroupCountAndOrder("group4", 2, results, "12.0", "14.0"); + assertExpandGroupCountAndOrder("group1", 2, results, "1", "7"); + assertExpandGroupCountAndOrder("group2", 2, results, "5", "8"); + assertExpandGroupCountAndOrder("group3", 2, results, "11", "9"); + assertExpandGroupCountAndOrder("group4", 2, results, "12", "14"); //Test expand.sort @@ -121,10 +121,10 @@ public class DistributedExpandComponentTest extends BaseDistributedSearchTestCas rsp = queryServer(params); results = rsp.getExpandedResults(); assertExpandGroups(results, "group1","group2", "group3", "group4"); - assertExpandGroupCountAndOrder("group1", 2, results, "7.0", "1.0"); - assertExpandGroupCountAndOrder("group2", 2, results, "8.0", "5.0"); - assertExpandGroupCountAndOrder("group3", 2, results, "9.0", "11.0"); - assertExpandGroupCountAndOrder("group4", 2, results, "14.0", "12.0"); + assertExpandGroupCountAndOrder("group1", 2, results, "7", "1"); + assertExpandGroupCountAndOrder("group2", 2, results, "8", "5"); + assertExpandGroupCountAndOrder("group3", 2, results, "9", "11"); + assertExpandGroupCountAndOrder("group4", 2, results, "14", "12"); //Test expand.rows @@ -141,10 +141,10 @@ public class DistributedExpandComponentTest extends BaseDistributedSearchTestCas rsp = queryServer(params); results = rsp.getExpandedResults(); assertExpandGroups(results, "group1","group2", "group3", "group4"); - assertExpandGroupCountAndOrder("group1", 1, results, "7.0"); - assertExpandGroupCountAndOrder("group2", 1, results, "8.0"); - assertExpandGroupCountAndOrder("group3", 1, results, "9.0"); - assertExpandGroupCountAndOrder("group4", 1, results, "14.0"); + assertExpandGroupCountAndOrder("group1", 1, results, "7"); + assertExpandGroupCountAndOrder("group2", 1, results, "8"); + assertExpandGroupCountAndOrder("group3", 1, results, "9"); + assertExpandGroupCountAndOrder("group4", 1, results, "14"); //Test key-only fl @@ -161,10 +161,10 @@ public class DistributedExpandComponentTest extends BaseDistributedSearchTestCas rsp = queryServer(params); results = rsp.getExpandedResults(); assertExpandGroups(results, "group1","group2", "group3", "group4"); - assertExpandGroupCountAndOrder("group1", 2, results, "1.0", "7.0"); - assertExpandGroupCountAndOrder("group2", 2, results, "5.0", "8.0"); - assertExpandGroupCountAndOrder("group3", 2, results, "11.0", "9.0"); - assertExpandGroupCountAndOrder("group4", 2, results, "12.0", "14.0"); + assertExpandGroupCountAndOrder("group1", 2, results, "1", "7"); + assertExpandGroupCountAndOrder("group2", 2, results, "5", "8"); + assertExpandGroupCountAndOrder("group3", 2, results, "11", "9"); + assertExpandGroupCountAndOrder("group4", 2, results, "12", "14"); //Test distrib.singlePass true @@ -180,10 +180,10 @@ public class DistributedExpandComponentTest extends BaseDistributedSearchTestCas rsp = queryServer(params); results = rsp.getExpandedResults(); assertExpandGroups(results, "group1","group2", "group3", "group4"); - assertExpandGroupCountAndOrder("group1", 2, results, "1.0", "7.0"); - assertExpandGroupCountAndOrder("group2", 2, results, "5.0", "8.0"); - assertExpandGroupCountAndOrder("group3", 2, results, "11.0", "9.0"); - assertExpandGroupCountAndOrder("group4", 2, results, "12.0", "14.0"); + assertExpandGroupCountAndOrder("group1", 2, results, "1", "7"); + assertExpandGroupCountAndOrder("group2", 2, results, "5", "8"); + assertExpandGroupCountAndOrder("group3", 2, results, "11", "9"); + assertExpandGroupCountAndOrder("group4", 2, results, "12", "14"); } diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedMLTComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedMLTComponentTest.java index 10116b971c6..157d6a7cdac 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedMLTComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedMLTComponentTest.java @@ -140,28 +140,28 @@ public class DistributedMLTComponentTest extends BaseDistributedSearchTestCase { // we ask for an mlt.count of 20 to ensure both include all results query("q", "lowerfilt:moon", "fl", id, MoreLikeThisParams.MIN_TERM_FREQ, 2, - MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id desc", "mlt", "true", + MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id_i1 desc", "mlt", "true", "mlt.fl", "lowerfilt", "qt", requestHandlerName, "shards.qt", requestHandlerName, "mlt.count", "20"); query("q", "lowerfilt:fox", "fl", id, MoreLikeThisParams.MIN_TERM_FREQ, 1, - MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id desc", "mlt", "true", + MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id_i1 desc", "mlt", "true", "mlt.fl", "lowerfilt", "qt", requestHandlerName, "shards.qt", requestHandlerName, "mlt.count", "20"); query("q", "lowerfilt:the red fox", "fl", id, MoreLikeThisParams.MIN_TERM_FREQ, 1, - MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id desc", "mlt", "true", + MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id_i1 desc", "mlt", "true", "mlt.fl", "lowerfilt", "qt", requestHandlerName, "shards.qt", requestHandlerName, "mlt.count", "20"); query("q", "lowerfilt:blue moon", "fl", id, MoreLikeThisParams.MIN_TERM_FREQ, 1, - MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id desc", "mlt", "true", + MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id_i1 desc", "mlt", "true", "mlt.fl", "lowerfilt", "qt", requestHandlerName, "shards.qt", requestHandlerName, "mlt.count", "20"); // let's query by specifying multiple mlt.fl as comma-separated values QueryResponse response = query("q", "lowerfilt:moon", "fl", id, MoreLikeThisParams.MIN_TERM_FREQ, 2, - MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id desc", "mlt", "true", + MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id_i1 desc", "mlt", "true", "mlt.fl", "lowerfilt1,lowerfilt", "qt", requestHandlerName, "shards.qt", requestHandlerName, "mlt.count", "20"); NamedList moreLikeThis = (NamedList) response.getResponse().get("moreLikeThis"); @@ -173,7 +173,7 @@ public class DistributedMLTComponentTest extends BaseDistributedSearchTestCase { // let's query by specifying multiple mlt.fl as multiple request parameters response = query("q", "lowerfilt:moon", "fl", id, MoreLikeThisParams.MIN_TERM_FREQ, 2, - MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id desc", "mlt", "true", + MoreLikeThisParams.MIN_DOC_FREQ, 1, "sort", "id_i1 desc", "mlt", "true", "mlt.fl", "lowerfilt1", "mlt.fl", "lowerfilt", "qt", requestHandlerName, "shards.qt", requestHandlerName, "mlt.count", "20"); moreLikeThis = (NamedList) response.getResponse().get("moreLikeThis"); diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java index 4b3e92a77ec..7efde0c90c4 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java @@ -64,44 +64,44 @@ public class DistributedQueryComponentCustomSortTest extends BaseDistributedSear QueryResponse rsp; rsp = query("q", "*:*", "fl", "id", "sort", "payload asc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 7, 1, 6, 4, 2, 10, 12, 3, 5, 9, 8, 13, 11); + assertFieldValues(rsp.getResults(), id, "7", "1", "6", "4", "2", "10", "12", "3", "5", "9", "8", "13", "11"); rsp = query("q", "*:*", "fl", "id", "sort", "payload desc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 11, 13, 8, 9, 5, 3, 12, 10, 2, 4, 6, 1, 7); + assertFieldValues(rsp.getResults(), id, "11", "13", "8", "9", "5", "3", "12", "10", "2", "4", "6", "1", "7"); // SOLR-6744 rsp = query("q", "*:*", "fl", "key:id", "sort", "payload asc", "rows", "20"); - assertFieldValues(rsp.getResults(), "key", 7, 1, 6, 4, 2, 10, 12, 3, 5, 9, 8, 13, 11); + assertFieldValues(rsp.getResults(), "key", "7", "1", "6", "4", "2", "10", "12", "3", "5", "9", "8", "13", "11"); rsp = query("q", "*:*", "fl", "key:id,id:text", "sort", "payload asc", "rows", "20"); - assertFieldValues(rsp.getResults(), "key", 7, 1, 6, 4, 2, 10, 12, 3, 5, 9, 8, 13, 11); + assertFieldValues(rsp.getResults(), "key", "7", "1", "6", "4", "2", "10", "12", "3", "5", "9", "8", "13", "11"); rsp = query("q", "text:a", "fl", "id", "sort", "payload asc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 1, 3, 5, 9); + assertFieldValues(rsp.getResults(), id, "1", "3", "5", "9"); rsp = query("q", "text:a", "fl", "id", "sort", "payload desc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 9, 5, 3, 1); + assertFieldValues(rsp.getResults(), id, "9", "5", "3", "1"); rsp = query("q", "text:b", "fl", "id", "sort", "payload asc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 4, 2, 10); + assertFieldValues(rsp.getResults(), id, "4", "2", "10"); rsp = query("q", "text:b", "fl", "id", "sort", "payload desc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 10, 2, 4); + assertFieldValues(rsp.getResults(), id, "10", "2", "4"); // SOLR-6744 rsp = query("q", "text:b", "fl", "key:id", "sort", "payload asc", "rows", "20"); assertFieldValues(rsp.getResults(), id, null, null, null); rsp = query("q", "text:c", "fl", "id", "sort", "payload asc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 7, 6, 8); + assertFieldValues(rsp.getResults(), id, "7", "6", "8"); rsp = query("q", "text:c", "fl", "id", "sort", "payload desc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 8, 6, 7); + assertFieldValues(rsp.getResults(), id, "8", "6", "7"); rsp = query("q", "text:d", "fl", "id", "sort", "payload asc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 12, 13, 11); + assertFieldValues(rsp.getResults(), id, "12", "13", "11"); rsp = query("q", "text:d", "fl", "id", "sort", "payload desc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 11, 13, 12); + assertFieldValues(rsp.getResults(), id, "11", "13", "12"); // sanity check function sorting - rsp = query("q", "id:[1 TO 10]", "fl", "id", "rows", "20", - "sort", "abs(sub(5,id)) asc, id desc"); - assertFieldValues(rsp.getResults(), id, 5 , 6,4 , 7,3 , 8,2 , 9,1 , 10 ); + rsp = query("q", "id_i:[1 TO 10]", "fl", "id", "rows", "20", + "sort", "abs(sub(5,id_i)) asc, id desc"); + assertFieldValues(rsp.getResults(), id, "5", "6","4", "7","3" , "8","2" , "9","1" , "10" ); // Add two more docs with same payload as in doc #4 index(id, "14", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x15 })); @@ -114,9 +114,9 @@ public class DistributedQueryComponentCustomSortTest extends BaseDistributedSear commit(); - rsp = query("q", "*:*", "fl", "id", "sort", "payload asc, id desc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 7, 1, 6, 15,14,4, 2, 18,17,16,10, 12, 3, 5, 9, 8, 13, 11); - rsp = query("q", "*:*", "fl", "id", "sort", "payload desc, id asc", "rows", "20"); - assertFieldValues(rsp.getResults(), id, 11, 13, 8, 9, 5, 3, 12, 10,16,17,18, 2, 4,14,15, 6, 1, 7); + rsp = query("q", "*:*", "fl", "id", "sort", "payload asc, id_i desc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, "7", "1", "6", "15","14","4", "2", "18","17","16","10", "12", "3", "5", "9", "8", "13", "11"); + rsp = query("q", "*:*", "fl", "id", "sort", "payload desc, id_i asc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, "11", "13", "8", "9", "5", "3", "12", "10","16","17","18", "2", "4","14","15", "6", "1", "7"); } } diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java index fb90e83978e..c2115487caa 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java @@ -105,7 +105,7 @@ public class DistributedQueryElevationComponentTest extends BaseDistributedSearc assertTrue(response.getResults().getNumFound() > 0); SolrDocument document = response.getResults().get(0); - assertEquals(6.0f, document.getFieldValue("id")); + assertEquals("6", document.getFieldValue("id")); assertEquals(true, document.getFieldValue("[elevated]")); // Force javabin format @@ -121,7 +121,7 @@ public class DistributedQueryElevationComponentTest extends BaseDistributedSearc assertTrue(response.getResults().getNumFound() > 0); document = response.getResults().get(0); - assertEquals(6.0f, document.getFieldValue("id")); + assertEquals("6", document.getFieldValue("id")); assertEquals(true, document.getFieldValue("[elevated]")); } diff --git a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java index f439e87a8d2..829c585c44f 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java @@ -96,9 +96,9 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate", CommonParams.FL, "id, score, [elevated]") , "//*[@numFound='3']" - , "//result/doc[1]/float[@name='id'][.='7.0']" - , "//result/doc[2]/float[@name='id'][.='9.0']" - , "//result/doc[3]/float[@name='id'][.='8.0']", + , "//result/doc[1]/str[@name='id'][.='7']" + , "//result/doc[2]/str[@name='id'][.='9']" + , "//result/doc[3]/str[@name='id'][.='8']", "//result/doc[1]/bool[@name='[elevated]'][.='true']", "//result/doc[2]/bool[@name='[elevated]'][.='false']", "//result/doc[3]/bool[@name='[elevated]'][.='false']" @@ -144,19 +144,19 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { , "//*[@name='ngroups'][.='3']" , "//*[@name='matches'][.='6']" - , groups +"/lst[1]//doc[1]/float[@name='id'][.='6.0']" + , groups +"/lst[1]//doc[1]/str[@name='id'][.='6']" , groups +"/lst[1]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[1]//doc[2]/float[@name='id'][.='66.0']" + , groups +"/lst[1]//doc[2]/str[@name='id'][.='66']" , groups +"/lst[1]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[1]/float[@name='id'][.='7.0']" + , groups +"/lst[2]//doc[1]/str[@name='id'][.='7']" , groups +"/lst[2]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[2]/float[@name='id'][.='77.0']" + , groups +"/lst[2]//doc[2]/str[@name='id'][.='77']" , groups +"/lst[2]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[1]/float[@name='id'][.='2.0']" + , groups +"/lst[3]//doc[1]/str[@name='id'][.='2']" , groups +"/lst[3]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[2]/float[@name='id'][.='22.0']" + , groups +"/lst[3]//doc[2]/str[@name='id'][.='22']" , groups +"/lst[3]//doc[2]/bool[@name='[elevated]'][.='false']" ); @@ -171,19 +171,19 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { , "//*[@name='ngroups'][.='3']" , "//*[@name='matches'][.='6']" - , groups +"/lst[1]//doc[1]/float[@name='id'][.='7.0']" + , groups +"/lst[1]//doc[1]/str[@name='id'][.='7']" , groups +"/lst[1]//doc[1]/bool[@name='[elevated]'][.='true']" - , groups +"/lst[1]//doc[2]/float[@name='id'][.='77.0']" + , groups +"/lst[1]//doc[2]/str[@name='id'][.='77']" , groups +"/lst[1]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[1]/float[@name='id'][.='6.0']" + , groups +"/lst[2]//doc[1]/str[@name='id'][.='6']" , groups +"/lst[2]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[2]/float[@name='id'][.='66.0']" + , groups +"/lst[2]//doc[2]/str[@name='id'][.='66']" , groups +"/lst[2]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[1]/float[@name='id'][.='2.0']" + , groups +"/lst[3]//doc[1]/str[@name='id'][.='2']" , groups +"/lst[3]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[2]/float[@name='id'][.='22.0']" + , groups +"/lst[3]//doc[2]/str[@name='id'][.='22']" , groups +"/lst[3]//doc[2]/bool[@name='[elevated]'][.='false']" ); @@ -199,19 +199,19 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { , "//*[@name='ngroups'][.='3']" , "//*[@name='matches'][.='6']" - , groups +"/lst[1]//doc[1]/float[@name='id'][.='2.0']" + , groups +"/lst[1]//doc[1]/str[@name='id'][.='2']" , groups +"/lst[1]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[1]//doc[2]/float[@name='id'][.='22.0']" + , groups +"/lst[1]//doc[2]/str[@name='id'][.='22']" , groups +"/lst[1]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[1]/float[@name='id'][.='6.0']" + , groups +"/lst[2]//doc[1]/str[@name='id'][.='6']" , groups +"/lst[2]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[2]/float[@name='id'][.='66.0']" + , groups +"/lst[2]//doc[2]/str[@name='id'][.='66']" , groups +"/lst[2]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[1]/float[@name='id'][.='7.0']" + , groups +"/lst[3]//doc[1]/str[@name='id'][.='7']" , groups +"/lst[3]//doc[1]/bool[@name='[elevated]'][.='true']" - , groups +"/lst[3]//doc[2]/float[@name='id'][.='77.0']" + , groups +"/lst[3]//doc[2]/str[@name='id'][.='77']" , groups +"/lst[3]//doc[2]/bool[@name='[elevated]'][.='false']" ); @@ -228,19 +228,19 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { , "//*[@name='ngroups'][.='3']" , "//*[@name='matches'][.='6']" - , groups +"/lst[1]//doc[1]/float[@name='id'][.='7.0']" + , groups +"/lst[1]//doc[1]/str[@name='id'][.='7']" , groups +"/lst[1]//doc[1]/bool[@name='[elevated]'][.='true']" - , groups +"/lst[1]//doc[2]/float[@name='id'][.='77.0']" + , groups +"/lst[1]//doc[2]/str[@name='id'][.='77']" , groups +"/lst[1]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[1]/float[@name='id'][.='2.0']" + , groups +"/lst[2]//doc[1]/str[@name='id'][.='2']" , groups +"/lst[2]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[2]/float[@name='id'][.='22.0']" + , groups +"/lst[2]//doc[2]/str[@name='id'][.='22']" , groups +"/lst[2]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[1]/float[@name='id'][.='6.0']" + , groups +"/lst[3]//doc[1]/str[@name='id'][.='6']" , groups +"/lst[3]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[2]/float[@name='id'][.='66.0']" + , groups +"/lst[3]//doc[2]/str[@name='id'][.='66']" , groups +"/lst[3]//doc[2]/bool[@name='[elevated]'][.='false']" ); @@ -258,19 +258,19 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { , "//*[@name='ngroups'][.='3']" , "//*[@name='matches'][.='6']" - , groups +"/lst[1]//doc[1]/float[@name='id'][.='22.0']" + , groups +"/lst[1]//doc[1]/str[@name='id'][.='22']" , groups +"/lst[1]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[1]//doc[2]/float[@name='id'][.='2.0']" + , groups +"/lst[1]//doc[2]/str[@name='id'][.='2']" , groups +"/lst[1]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[1]/float[@name='id'][.='66.0']" + , groups +"/lst[2]//doc[1]/str[@name='id'][.='66']" , groups +"/lst[2]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[2]/float[@name='id'][.='6.0']" + , groups +"/lst[2]//doc[2]/str[@name='id'][.='6']" , groups +"/lst[2]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[1]/float[@name='id'][.='77.0']" + , groups +"/lst[3]//doc[1]/str[@name='id'][.='77']" , groups +"/lst[3]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[2]/float[@name='id'][.='7.0']" + , groups +"/lst[3]//doc[2]/str[@name='id'][.='7']" , groups +"/lst[3]//doc[2]/bool[@name='[elevated]'][.='true']" ); @@ -289,19 +289,19 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { , "//*[@name='ngroups'][.='3']" , "//*[@name='matches'][.='6']" - , groups +"/lst[1]//doc[1]/float[@name='id'][.='7.0']" + , groups +"/lst[1]//doc[1]/str[@name='id'][.='7']" , groups +"/lst[1]//doc[1]/bool[@name='[elevated]'][.='true']" - , groups +"/lst[1]//doc[2]/float[@name='id'][.='77.0']" + , groups +"/lst[1]//doc[2]/str[@name='id'][.='77']" , groups +"/lst[1]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[1]/float[@name='id'][.='22.0']" + , groups +"/lst[2]//doc[1]/str[@name='id'][.='22']" , groups +"/lst[2]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[2]//doc[2]/float[@name='id'][.='2.0']" + , groups +"/lst[2]//doc[2]/str[@name='id'][.='2']" , groups +"/lst[2]//doc[2]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[1]/float[@name='id'][.='66.0']" + , groups +"/lst[3]//doc[1]/str[@name='id'][.='66']" , groups +"/lst[3]//doc[1]/bool[@name='[elevated]'][.='false']" - , groups +"/lst[3]//doc[2]/float[@name='id'][.='6.0']" + , groups +"/lst[3]//doc[2]/str[@name='id'][.='6']" , groups +"/lst[3]//doc[2]/bool[@name='[elevated]'][.='false']" ); @@ -331,9 +331,9 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate", CommonParams.FL, "id, score, [elevated]") ,"//*[@numFound='3']" - ,"//result/doc[1]/int[@name='id'][.='7']" - ,"//result/doc[2]/int[@name='id'][.='8']" - ,"//result/doc[3]/int[@name='id'][.='9']", + ,"//result/doc[1]/str[@name='id'][.='7']" + ,"//result/doc[2]/str[@name='id'][.='8']" + ,"//result/doc[3]/str[@name='id'][.='9']", "//result/doc[1]/bool[@name='[elevated]'][.='true']", "//result/doc[2]/bool[@name='[elevated]'][.='false']", "//result/doc[3]/bool[@name='[elevated]'][.='false']" @@ -750,19 +750,19 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { assertQ("", req(CommonParams.Q, "AAAA", CommonParams.QT, "/elevate", CommonParams.FL, "id, score, [elevated]") , "//*[@numFound='1']" - , "//result/doc[1]/float[@name='id'][.='7.0']" + , "//result/doc[1]/str[@name='id'][.='7']" , "//result/doc[1]/bool[@name='[elevated]'][.='true']" ); assertQ("", req(CommonParams.Q, "{!q.op=AND}AAAA", CommonParams.QT, "/elevate", CommonParams.FL, "id, score, [elevated]") , "//*[@numFound='1']" - , "//result/doc[1]/float[@name='id'][.='7.0']" + , "//result/doc[1]/str[@name='id'][.='7']" , "//result/doc[1]/bool[@name='[elevated]'][.='true']" ); assertQ("", req(CommonParams.Q, "{!q.op=AND v='AAAA'}", CommonParams.QT, "/elevate", CommonParams.FL, "id, score, [elevated]") , "//*[@numFound='1']" - , "//result/doc[1]/float[@name='id'][.='7.0']" + , "//result/doc[1]/str[@name='id'][.='7']" , "//result/doc[1]/bool[@name='[elevated]'][.='true']" ); } finally { diff --git a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java index 1798ae53f9d..568676c7f38 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java @@ -94,8 +94,8 @@ public class StatsComponentTest extends AbstractSolrTestCase { // NOTE: doTestFieldStatisticsResult needs the full list of possible tags to exclude params("stats.field", f, "stats", "true"), params("stats.field", "{!ex=fq1,fq2}"+f, "stats", "true", - "fq", "{!tag=fq1}-id:[0 TO 2]", - "fq", "{!tag=fq2}-id:[2 TO 1000]"), + "fq", "{!tag=fq1}-id_i:[0 TO 2]", + "fq", "{!tag=fq2}-id_i:[2 TO 1000]"), params("stats.field", "{!ex=fq1}"+f, "stats", "true", "fq", "{!tag=fq1}id:1") }; @@ -299,8 +299,8 @@ public class StatsComponentTest extends AbstractSolrTestCase { params("stats.field", "{!ex=fq1}"+f, "stats", "true", "fq", "{!tag=fq1}id:1"), params("stats.field", "{!ex=fq1,fq2}"+f, "stats", "true", - "fq", "{!tag=fq1}-id:[0 TO 2]", - "fq", "{!tag=fq2}-id:[2 TO 1000]") }) { + "fq", "{!tag=fq1}-id_i:[0 TO 2]", + "fq", "{!tag=fq2}-id_i:[2 TO 1000]") }) { assertQ("test statistics values", @@ -878,19 +878,19 @@ public class StatsComponentTest extends AbstractSolrTestCase { Map args = new HashMap(); args.put(CommonParams.Q, "*:*"); args.put(StatsParams.STATS, "true"); - args.put(StatsParams.STATS_FIELD, "{!ex=id}id"); - args.put("fq", "{!tag=id}id:[2 TO 3]"); + args.put(StatsParams.STATS_FIELD, "{!ex=id}id_i"); + args.put("fq", "{!tag=id}id_i:[2 TO 3]"); SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args)); assertQ("test exluding filter query", req - , "//lst[@name='id']/double[@name='min'][.='1.0']" - , "//lst[@name='id']/double[@name='max'][.='4.0']"); + , "//lst[@name='id_i']/double[@name='min'][.='1.0']" + , "//lst[@name='id_i']/double[@name='max'][.='4.0']"); args = new HashMap(); args.put(CommonParams.Q, "*:*"); args.put(StatsParams.STATS, "true"); - args.put(StatsParams.STATS_FIELD, "{!key=id2}id"); - args.put("fq", "{!tag=id}id:[2 TO 3]"); + args.put(StatsParams.STATS_FIELD, "{!key=id2}id_i"); + args.put("fq", "{!tag=id}id_i:[2 TO 3]"); req = new LocalSolrQueryRequest(core, new MapSolrParams(args)); assertQ("test rename field", req diff --git a/solr/core/src/test/org/apache/solr/handler/component/TestDistributedStatsComponentCardinality.java b/solr/core/src/test/org/apache/solr/handler/component/TestDistributedStatsComponentCardinality.java index c7036836a2c..dca45dc63a0 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/TestDistributedStatsComponentCardinality.java +++ b/solr/core/src/test/org/apache/solr/handler/component/TestDistributedStatsComponentCardinality.java @@ -244,7 +244,7 @@ public class TestDistributedStatsComponentCardinality extends BaseDistributedSea final int highId, final int log2m, final int regwidth) { - ModifiableSolrParams p = params("q", "id:["+lowId+" TO "+highId+"]", + ModifiableSolrParams p = params("q", "id_i1:["+lowId+" TO "+highId+"]", "rows", "0", "stats", "true"); final String prefix = "{!cardinality=true hllLog2m="+log2m+" hllRegwidth="+regwidth; for (String f : STAT_FIELDS) { @@ -267,7 +267,7 @@ public class TestDistributedStatsComponentCardinality extends BaseDistributedSea final int highId, final double lowAccuracy, final double highAccuracy) { - ModifiableSolrParams p = params("q", "id:["+lowId+" TO "+highId+"]", + ModifiableSolrParams p = params("q", "id_i1:["+lowId+" TO "+highId+"]", "rows", "0", "stats", "true"); final String[] prefixes = new String[] { "{!cardinality=" + lowAccuracy + " key=low_", diff --git a/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java b/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java index 332e3462cb4..9c976d819b4 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java +++ b/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java @@ -119,12 +119,12 @@ public class TestExpandComponent extends SolrTestCaseJ4 { params.add("expand", "true"); assertQ(req(params), "*[count(/response/result/doc)=2]", "*[count(/response/lst[@name='expanded']/result)=2]", - "/response/result/doc[1]/float[@name='id'][.='2.0']", - "/response/result/doc[2]/float[@name='id'][.='6.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='1.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='7.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='5.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='8.0']" + "/response/result/doc[1]/str[@name='id'][.='2']", + "/response/result/doc[2]/str[@name='id'][.='6']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/str[@name='id'][.='1']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/str[@name='id'][.='7']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/str[@name='id'][.='5']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/str[@name='id'][.='8']" ); //Basic test case page 2 @@ -139,9 +139,9 @@ public class TestExpandComponent extends SolrTestCaseJ4 { params.add("start", "1"); assertQ(req(params), "*[count(/response/result/doc)=1]", "*[count(/response/lst[@name='expanded']/result)=1]", - "/response/result/doc[1]/float[@name='id'][.='6.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='5.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='8.0']" + "/response/result/doc[1]/str[@name='id'][.='6']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/str[@name='id'][.='5']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/str[@name='id'][.='8']" ); //Test expand.sort @@ -154,12 +154,12 @@ public class TestExpandComponent extends SolrTestCaseJ4 { params.add("expand.sort", "test_l desc, sub(1,1) asc");//the "sub()" just testing function queries assertQ(req(params), "*[count(/response/result/doc)=2]", "*[count(/response/lst[@name='expanded']/result)=2]", - "/response/result/doc[1]/float[@name='id'][.='2.0']", - "/response/result/doc[2]/float[@name='id'][.='6.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='1.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='5.0']" + "/response/result/doc[1]/str[@name='id'][.='2']", + "/response/result/doc[2]/str[@name='id'][.='6']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/str[@name='id'][.='7']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/str[@name='id'][.='1']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/str[@name='id'][.='8']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/str[@name='id'][.='5']" ); //Test with nullPolicy, ExpandComponent should ignore docs with null values in the collapse fields. @@ -173,13 +173,13 @@ public class TestExpandComponent extends SolrTestCaseJ4 { params.add("expand.sort", "test_l desc"); assertQ(req(params), "*[count(/response/result/doc)=3]", "*[count(/response/lst[@name='expanded']/result)=2]", - "/response/result/doc[1]/float[@name='id'][.='3.0']", - "/response/result/doc[2]/float[@name='id'][.='2.0']", - "/response/result/doc[3]/float[@name='id'][.='6.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='1.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='5.0']" + "/response/result/doc[1]/str[@name='id'][.='3']", + "/response/result/doc[2]/str[@name='id'][.='2']", + "/response/result/doc[3]/str[@name='id'][.='6']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/str[@name='id'][.='7']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/str[@name='id'][.='1']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/str[@name='id'][.='8']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/str[@name='id'][.='5']" ); @@ -195,12 +195,12 @@ public class TestExpandComponent extends SolrTestCaseJ4 { params.add("expand.sort", "test_l desc"); assertQ(req(params), "*[count(/response/result/doc)=2]", "*[count(/response/lst[@name='expanded']/result)=2]", - "/response/result/doc[1]/float[@name='id'][.='1.0']", - "/response/result/doc[2]/float[@name='id'][.='5.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='2.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='6.0']" + "/response/result/doc[1]/str[@name='id'][.='1']", + "/response/result/doc[2]/str[@name='id'][.='5']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/str[@name='id'][.='7']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/str[@name='id'][.='2']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/str[@name='id'][.='8']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/str[@name='id'][.='6']" ); @@ -217,12 +217,12 @@ public class TestExpandComponent extends SolrTestCaseJ4 { params.add("expand.sort", "test_l desc"); assertQ(req(params), "*[count(/response/result/doc)=2]", "*[count(/response/lst[@name='expanded']/result)=2]", - "/response/result/doc[1]/float[@name='id'][.='1.0']", - "/response/result/doc[2]/float[@name='id'][.='5.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='2.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='6.0']" + "/response/result/doc[1]/str[@name='id'][.='1']", + "/response/result/doc[2]/str[@name='id'][.='5']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/str[@name='id'][.='7']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/str[@name='id'][.='2']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/str[@name='id'][.='8']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/str[@name='id'][.='6']" ); //Test overide expand.fq and expand.q @@ -239,12 +239,12 @@ public class TestExpandComponent extends SolrTestCaseJ4 { params.add("expand.sort", "test_l desc"); assertQ(req(params), "*[count(/response/result/doc)=2]", "*[count(/response/lst[@name='expanded']/result)=2]", - "/response/result/doc[1]/float[@name='id'][.='1.0']", - "/response/result/doc[2]/float[@name='id'][.='5.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='2.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='6.0']" + "/response/result/doc[1]/str[@name='id'][.='1']", + "/response/result/doc[2]/str[@name='id'][.='5']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/str[@name='id'][.='7']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/str[@name='id'][.='2']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/str[@name='id'][.='8']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/str[@name='id'][.='6']" ); //Test expand.rows @@ -261,10 +261,10 @@ public class TestExpandComponent extends SolrTestCaseJ4 { "*[count(/response/lst[@name='expanded']/result)=2]", "*[count(/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc)=1]", "*[count(/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc)=1]", - "/response/result/doc[1]/float[@name='id'][.='2.0']", - "/response/result/doc[2]/float[@name='id'][.='6.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='7.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='8.0']" + "/response/result/doc[1]/str[@name='id'][.='2']", + "/response/result/doc[2]/str[@name='id'][.='6']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/str[@name='id'][.='7']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/str[@name='id'][.='8']" ); @@ -307,12 +307,12 @@ public class TestExpandComponent extends SolrTestCaseJ4 { params.add("fl", "id"); assertQ(req(params), "*[count(/response/result/doc)=2]", "*[count(/response/lst[@name='expanded']/result)=2]", - "/response/result/doc[1]/float[@name='id'][.='2.0']", - "/response/result/doc[2]/float[@name='id'][.='6.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/float[@name='id'][.='1.0']", - "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/float[@name='id'][.='7.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/float[@name='id'][.='5.0']", - "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/float[@name='id'][.='8.0']" + "/response/result/doc[1]/str[@name='id'][.='2']", + "/response/result/doc[2]/str[@name='id'][.='6']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[1]/str[@name='id'][.='1']", + "/response/lst[@name='expanded']/result[@name='1"+floatAppend+"']/doc[2]/str[@name='id'][.='7']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[1]/str[@name='id'][.='5']", + "/response/lst[@name='expanded']/result[@name='2"+floatAppend+"']/doc[2]/str[@name='id'][.='8']" ); } diff --git a/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java b/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java index fd8d6ecae5b..8f4a24d8b79 100644 --- a/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java +++ b/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java @@ -236,7 +236,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { @Test public void testCachingBigTerms() throws Exception { - assertQ( req("indent","true", "q", "id:[42 TO 47]", + assertQ( req("indent","true", "q", "id_i1:[42 TO 47]", "facet", "true", "facet.field", "foo_s" // big terms should cause foo_s:A to be cached ), @@ -270,7 +270,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { assertQ( req( "q", "*:*", - "fq", "id:[2000 TO 2004]", + "fq", "id_i1:[2000 TO 2004]", "group", "true", "group.facet", "true", "group.field", "hotel_s1", @@ -284,7 +284,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { assertQ( req( "q", "*:*", - "fq", "id:[2000 TO 2004]", + "fq", "id_i1:[2000 TO 2004]", "fq", "{!tag=dus}airport_s1:dus", "group", "true", "group.facet", "true", @@ -298,7 +298,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { assertQ( req( "q", "*:*", - "fq", "id:[2000 TO 2004]", + "fq", "id_i1:[2000 TO 2004]", "group", "true", "group.facet", "true", "group.field", "hotel_s1", @@ -320,7 +320,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { assertQ( req( "q", "*:*", - "fq", "id:[2000 TO 2004]", + "fq", "id_i1:[2000 TO 2004]", "fq", "{!tag=dus}airport_s1:dus", "group", "true", "group.facet", "true", @@ -344,7 +344,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { assertQ( req( "q", "*:*", - "fq", "id:[2000 TO 2004]", + "fq", "id_i1:[2000 TO 2004]", "fq", "{!tag=dus}airport_s1:dus", "group", "true", "group.facet", "true", @@ -380,14 +380,14 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { private void testSimpleGroupedFacets(String facetLimit) throws Exception { assertQ( "Return 5 docs with id range 1937 till 1940", - req("id:[2000 TO 2004]"), + req("id_i1:[2000 TO 2004]"), "*[count(//doc)=5]" ); assertQ( "Return two facet counts for field airport_a and duration_i1", req( "q", "*:*", - "fq", "id:[2000 TO 2004]", + "fq", "id_i1:[2000 TO 2004]", "group", "true", "group.facet", "true", "group.field", "hotel_s1", @@ -410,7 +410,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { "Return one facet count for field airport_a using facet.offset", req( "q", "*:*", - "fq", "id:[2000 TO 2004]", + "fq", "id_i1:[2000 TO 2004]", "group", "true", "group.facet", "true", "group.field", "hotel_s1", @@ -427,7 +427,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { "Return two facet counts for field airport_a with fq", req( "q", "*:*", - "fq", "id:[2000 TO 2004]", + "fq", "id_i1:[2000 TO 2004]", "fq", "duration_i1:5", "group", "true", "group.facet", "true", @@ -445,7 +445,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { "Return one facet count for field airport_s1 with prefix a", req( "q", "*:*", - "fq", "id:[2000 TO 2004]", + "fq", "id_i1:[2000 TO 2004]", "group", "true", "group.facet", "true", "group.field", "hotel_s1", @@ -463,7 +463,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { h.query( req( "q", "*:*", - "fq", "id:[2000 TO 2004]", + "fq", "id_i1:[2000 TO 2004]", "group.facet", "true", "facet", "true", "facet.field", "airport_s1", @@ -541,26 +541,26 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { public void testSimpleFacetCounts() { assertQ("standard request handler returns all matches", - req("id:[42 TO 47]"), + req("id_i1:[42 TO 47]"), "*[count(//doc)=6]" ); assertQ("filter results using fq", - req("q","id:[42 TO 46]", - "fq", "id:[43 TO 47]"), + req("q","id_i1:[42 TO 46]", + "fq", "id_i1:[43 TO 47]"), "*[count(//doc)=4]" ); assertQ("don't filter results using blank fq", - req("q","id:[42 TO 46]", + req("q","id_i1:[42 TO 46]", "fq", " "), "*[count(//doc)=5]" ); assertQ("filter results using multiple fq params", - req("q","id:[42 TO 46]", + req("q","id_i1:[42 TO 46]", "fq", "trait_s:Obnoxious", - "fq", "id:[43 TO 47]"), + "fq", "id_i1:[43 TO 47]"), "*[count(//doc)=1]" ); @@ -570,19 +570,19 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { for(String[] methodParam : new String[][]{ none, uifSwitch}){ assertQ("check counts for facet queries", req(methodParam - ,"q", "id:[42 TO 47]" + ,"q", "id_i1:[42 TO 47]" ,"facet", "true" ,"facet.query", "trait_s:Obnoxious" - ,"facet.query", "id:[42 TO 45]" - ,"facet.query", "id:[43 TO 47]" + ,"facet.query", "id_i1:[42 TO 45]" + ,"facet.query", "id_i1:[43 TO 47]" ,"facet.field", "trait_s" ) ,"*[count(//doc)=6]" ,"//lst[@name='facet_counts']/lst[@name='facet_queries']" ,"//lst[@name='facet_queries']/int[@name='trait_s:Obnoxious'][.='2']" - ,"//lst[@name='facet_queries']/int[@name='id:[42 TO 45]'][.='4']" - ,"//lst[@name='facet_queries']/int[@name='id:[43 TO 47]'][.='5']" + ,"//lst[@name='facet_queries']/int[@name='id_i1:[42 TO 45]'][.='4']" + ,"//lst[@name='facet_queries']/int[@name='id_i1:[43 TO 47]'][.='5']" ,"//lst[@name='facet_counts']/lst[@name='facet_fields']" ,"//lst[@name='facet_fields']/lst[@name='trait_s']" @@ -593,11 +593,11 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("check multi-select facets with naming", - req(methodParam, "q", "id:[42 TO 47]" + req(methodParam, "q", "id_i1:[42 TO 47]" ,"facet", "true" ,"facet.query", "{!ex=1}trait_s:Obnoxious" - ,"facet.query", "{!ex=2 key=foo}id:[42 TO 45]" // tag=2 same as 1 - ,"facet.query", "{!ex=3,4 key=bar}id:[43 TO 47]" // tag=3,4 don't exist + ,"facet.query", "{!ex=2 key=foo}id_i1:[42 TO 45]" // tag=2 same as 1 + ,"facet.query", "{!ex=3,4 key=bar}id_i1:[43 TO 47]" // tag=3,4 don't exist ,"facet.field", "{!ex=3,1}trait_s" // 3,1 same as 1 ,"fq", "{!tag=1,2}id:47" // tagged as 1 and 2 ) @@ -629,17 +629,17 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { for(String[] methodParam : new String[][]{ none, uifSwitch}){ assertQ("check counts for applied facet queries using filtering (fq)", req(methodParam - ,"q", "id:[42 TO 47]" + ,"q", "id_i1:[42 TO 47]" ,"facet", "true" - ,"fq", "id:[42 TO 45]" + ,"fq", "id_i1:[42 TO 45]" ,"facet.field", "trait_s" - ,"facet.query", "id:[42 TO 45]" - ,"facet.query", "id:[43 TO 47]" + ,"facet.query", "id_i1:[42 TO 45]" + ,"facet.query", "id_i1:[43 TO 47]" ) ,"*[count(//doc)=4]" ,"//lst[@name='facet_counts']/lst[@name='facet_queries']" - ,"//lst[@name='facet_queries']/int[@name='id:[42 TO 45]'][.='4']" - ,"//lst[@name='facet_queries']/int[@name='id:[43 TO 47]'][.='3']" + ,"//lst[@name='facet_queries']/int[@name='id_i1:[42 TO 45]'][.='4']" + ,"//lst[@name='facet_queries']/int[@name='id_i1:[43 TO 47]'][.='3']" ,"*[count(//lst[@name='trait_s']/int)=4]" ,"//lst[@name='trait_s']/int[@name='Tool'][.='2']" ,"//lst[@name='trait_s']/int[@name='Obnoxious'][.='1']" @@ -649,11 +649,11 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { assertQ("check counts with facet.zero=false&facet.missing=true using fq", req(methodParam - ,"q", "id:[42 TO 47]" + ,"q", "id_i1:[42 TO 47]" ,"facet", "true" ,"facet.zeros", "false" ,"f.trait_s.facet.missing", "true" - ,"fq", "id:[42 TO 45]" + ,"fq", "id_i1:[42 TO 45]" ,"facet.field", "trait_s" ) ,"*[count(//doc)=4]" @@ -666,11 +666,11 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { assertQ("check counts with facet.mincount=1&facet.missing=true using fq", req(methodParam - ,"q", "id:[42 TO 47]" + ,"q", "id_i1:[42 TO 47]" ,"facet", "true" ,"facet.mincount", "1" ,"f.trait_s.facet.missing", "true" - ,"fq", "id:[42 TO 45]" + ,"fq", "id_i1:[42 TO 45]" ,"facet.field", "trait_s" ) ,"*[count(//doc)=4]" @@ -683,11 +683,11 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { assertQ("check counts with facet.mincount=2&facet.missing=true using fq", req(methodParam - ,"q", "id:[42 TO 47]" + ,"q", "id_i1:[42 TO 47]" ,"facet", "true" ,"facet.mincount", "2" ,"f.trait_s.facet.missing", "true" - ,"fq", "id:[42 TO 45]" + ,"fq", "id_i1:[42 TO 45]" ,"facet.field", "trait_s" ) ,"*[count(//doc)=4]" @@ -698,9 +698,9 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { assertQ("check sorted paging", req(methodParam - ,"q", "id:[42 TO 47]" + ,"q", "id_i1:[42 TO 47]" ,"facet", "true" - ,"fq", "id:[42 TO 45]" + ,"fq", "id_i1:[42 TO 45]" ,"facet.field", "trait_s" ,"facet.mincount","0" ,"facet.offset","0" @@ -715,9 +715,9 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { // check that the default sort is by count assertQ("check sorted paging", - req(methodParam, "q", "id:[42 TO 47]" + req(methodParam, "q", "id_i1:[42 TO 47]" ,"facet", "true" - ,"fq", "id:[42 TO 45]" + ,"fq", "id_i1:[42 TO 45]" ,"facet.field", "trait_s" ,"facet.mincount","0" ,"facet.offset","0" @@ -732,9 +732,9 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { // // check that legacy facet.sort=true/false works // - assertQ(req(methodParam, "q", "id:[42 TO 47]" + assertQ(req(methodParam, "q", "id_i1:[42 TO 47]" ,"facet", "true" - ,"fq", "id:[42 TO 45]" + ,"fq", "id_i1:[42 TO 45]" ,"facet.field", "trait_s" ,"facet.mincount","0" ,"facet.offset","0" @@ -747,9 +747,9 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ,"//int[3][@name='Obnoxious'][.='1']" ); - assertQ(req(methodParam, "q", "id:[42 TO 47]" + assertQ(req(methodParam, "q", "id_i1:[42 TO 47]" ,"facet", "true" - ,"fq", "id:[42 TO 45]" + ,"fq", "id_i1:[42 TO 45]" ,"facet.field", "trait_s" ,"facet.mincount","1" ,"facet.offset","0" @@ -764,9 +764,9 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { } for(String method : new String[]{ "fc","uif"}){ - assertQ(req("q", "id:[42 TO 47]" + assertQ(req("q", "id_i1:[42 TO 47]" ,"facet", "true" - ,"fq", "id:[42 TO 45]" + ,"fq", "id_i1:[42 TO 45]" ,"facet.field", "zerolen_s" ,(random().nextBoolean() ? "":"f.zerolen_s.")+"facet.method", method ) @@ -1475,7 +1475,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { String mid = (new Double( ((double)Float.MAX_VALUE) * 2D )).toString(); assertQ(f+": checking counts for lower", - req( "q", "id:[30 TO 60]" + req( "q", "id_i1:[30 TO 60]" ,"rows", "0" ,"facet", "true" ,"facet.range", f @@ -1714,7 +1714,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { @Test public void testNumericRangeFacetsTrieInt() { - helpTestWholeNumberRangeFacets("id"); + helpTestWholeNumberRangeFacets("id_i1"); } @Test public void testNumericRangeFacetsTrieLong() { @@ -1723,7 +1723,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { @Test public void testNumericRangeFacetsTrieIntDocValues() { - helpTestWholeNumberRangeFacets("id", FacetRangeMethod.DV); + helpTestWholeNumberRangeFacets("id_i1", FacetRangeMethod.DV); } @Test @@ -1752,7 +1752,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { String mid = (new Long( ((long)Integer.MAX_VALUE) * 2L )).toString(); assertQ(f+": checking counts for lower", - req( "q", "id:[30 TO 60]" + req( "q", "id_i1:[30 TO 60]" ,"rows", "0" ,"facet", "true" ,"facet.range", f @@ -1788,7 +1788,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { final String meta = pre + "/../"; assertQ(f+": checking counts for lower", - req( "q", "id:[30 TO 60]" + req( "q", "id_i1:[30 TO 60]" ,"rows", "0" ,"facet", "true" ,"facet.range", f @@ -1810,7 +1810,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ(f + ":checking counts for upper", - req( "q", "id:[30 TO 60]" + req( "q", "id_i1:[30 TO 60]" ,"rows", "0" ,"facet", "true" ,"facet.range", f @@ -2847,7 +2847,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { private void doTestRangeQueryHardEndParam(String field, FacetRangeMethod method) { assertQ("Test facet.range.hardend", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -2868,7 +2868,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("Test facet.range.hardend", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -2901,7 +2901,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { private void doTestRangeQueryOtherParam(String field, FacetRangeMethod method) { assertQ("Test facet.range.other", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -2921,7 +2921,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("Test facet.range.other", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -2941,7 +2941,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("Test facet.range.other", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -2961,7 +2961,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("Test facet.range.other", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -2982,7 +2982,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { // these should have equivilent behavior (multivalued 'other' param: top level vs local) for (SolrQueryRequest req : new SolrQueryRequest[] { - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -2992,7 +2992,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ,"facet.range.gap","1" ,"facet.range.other",FacetRangeOther.BEFORE.toString() ,"facet.range.other",FacetRangeOther.AFTER.toString()), - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", "{!facet.range.other=before facet.range.other=after}" + field @@ -3013,7 +3013,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { } assertQ("Test facet.range.other", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -3035,7 +3035,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("Test facet.range.other", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -3056,7 +3056,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("Test facet.range.other", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -3077,7 +3077,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("Test facet.range.other", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -3098,7 +3098,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("Test facet.range.other", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -3119,7 +3119,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("Test facet.range.other", - req("q", "id:[12345 TO 12345]" + req("q", "id_i1:[12345 TO 12345]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -3138,7 +3138,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { ); assertQ("Test facet.range.other", - req("q", "id:[42 TO 47]" + req("q", "id_i1:[42 TO 47]" ,"facet","true" ,"fl","id," + field ,"facet.range", field @@ -3332,7 +3332,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { public void testFacetPrefixWithFacetThreads() throws Exception { assertQ("Test facet.prefix with facet.thread", - req("q", "id:[101 TO 102]" + req("q", "id_i1:[101 TO 102]" ,"facet","true" ,"facet.field", "{!key=key1 facet.prefix=foo}myfield_s" ,"facet.field", "{!key=key2 facet.prefix=bar}myfield_s" @@ -3352,7 +3352,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { values[0] = random().nextInt(3000); values[1] = random().nextInt(3000); Arrays.sort(values); - return String.format(Locale.ROOT, "id: [%d TO %d]", values[0], values[1]); + return String.format(Locale.ROOT, "id_i1:[%d TO %d]", values[0], values[1]); } @@ -3375,7 +3375,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { } private ModifiableSolrParams getRandomParamsInt() { - String field = new String[]{"range_facet_l_dv", "range_facet_i_dv", "range_facet_l", "duration_i1", "id"}[random().nextInt(5)]; + String field = new String[]{"range_facet_l_dv", "range_facet_i_dv", "range_facet_l", "duration_i1", "id_i1"}[random().nextInt(5)]; ModifiableSolrParams params = new ModifiableSolrParams(); Integer[] values = new Integer[2]; do { diff --git a/solr/core/src/test/org/apache/solr/request/TestFaceting.java b/solr/core/src/test/org/apache/solr/request/TestFaceting.java index a80669a7c33..8933c196cd7 100644 --- a/solr/core/src/test/org/apache/solr/request/TestFaceting.java +++ b/solr/core/src/test/org/apache/solr/request/TestFaceting.java @@ -64,7 +64,7 @@ public class TestFaceting extends SolrTestCaseJ4 { void createIndex(int nTerms) { assertU(delQ("*:*")); for (int i=0; i(); @@ -143,7 +143,7 @@ public class CopyFieldTest extends SolrTestCaseJ4 { req = new LocalSolrQueryRequest( core, new MapSolrParams( args) ); assertQ("dynamic source", req ,"//*[@numFound='1']" - ,"//result/doc[1]/int[@name='id'][.='5']" + ,"//result/doc[1]/str[@name='id'][.='5']" ,"//result/doc[1]/arr[@name='highlight']/str[.='this is a simple test of ']" ); @@ -194,7 +194,7 @@ public class CopyFieldTest extends SolrTestCaseJ4 { SolrQueryRequest req = new LocalSolrQueryRequest( core, new MapSolrParams( args) ); assertQ("sku2 copied to text", req ,"//*[@numFound='1']" - ,"//result/doc[1]/int[@name='id'][.='5']" + ,"//result/doc[1]/str[@name='id'][.='5']" ); args = new HashMap<>(); @@ -203,7 +203,7 @@ public class CopyFieldTest extends SolrTestCaseJ4 { req = new LocalSolrQueryRequest( core, new MapSolrParams( args) ); assertQ("sku1 copied to dynamic dest *_s", req ,"//*[@numFound='1']" - ,"//result/doc[1]/int[@name='id'][.='5']" + ,"//result/doc[1]/str[@name='id'][.='5']" ,"//result/doc[1]/arr[@name='sku1']/str[.='10-1839ACX-93']" ); @@ -242,7 +242,7 @@ public class CopyFieldTest extends SolrTestCaseJ4 { SolrQueryRequest req = new LocalSolrQueryRequest( core, new MapSolrParams( args) ); assertQ("sku2 copied to text", req ,"//*[@numFound='1']" - ,"//result/doc[1]/int[@name='id'][.='5']" + ,"//result/doc[1]/str[@name='id'][.='5']" ); } @@ -257,7 +257,7 @@ public class CopyFieldTest extends SolrTestCaseJ4 { for (String q : new String[] {"5", "10-1839ACX-93", "AAM46" }) { assertQ(req("q","catchall_t:" + q) ,"//*[@numFound='1']" - ,"//result/doc[1]/int[@name='id'][.='5']"); + ,"//result/doc[1]/str[@name='id'][.='5']"); } } } diff --git a/solr/core/src/test/org/apache/solr/schema/CurrencyFieldXmlFileTest.java b/solr/core/src/test/org/apache/solr/schema/CurrencyFieldXmlFileTest.java index f3872d08c50..e8bffe7023f 100644 --- a/solr/core/src/test/org/apache/solr/schema/CurrencyFieldXmlFileTest.java +++ b/solr/core/src/test/org/apache/solr/schema/CurrencyFieldXmlFileTest.java @@ -33,8 +33,8 @@ public class CurrencyFieldXmlFileTest extends AbstractCurrencyFieldTest { assertU(adoc("id", "" + 2, field(), "15.00,EUR")); assertU(commit()); - assertQ(req("fl", "*,score", "q", field()+":15.00,EUR"), "//int[@name='id']='2'"); - assertQ(req("fl", "*,score", "q", field()+":7.50,USD"), "//int[@name='id']='2'"); + assertQ(req("fl", "*,score", "q", field()+":15.00,EUR"), "//str[@name='id']='2'"); + assertQ(req("fl", "*,score", "q", field()+":7.50,USD"), "//str[@name='id']='2'"); assertQ(req("fl", "*,score", "q", field()+":7.49,USD"), "//*[@numFound='0']"); assertQ(req("fl", "*,score", "q", field()+":7.51,USD"), "//*[@numFound='0']"); } diff --git a/solr/core/src/test/org/apache/solr/schema/DateRangeFieldTest.java b/solr/core/src/test/org/apache/solr/schema/DateRangeFieldTest.java index e76f8217cfd..edd25b0352e 100644 --- a/solr/core/src/test/org/apache/solr/schema/DateRangeFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/DateRangeFieldTest.java @@ -119,7 +119,7 @@ public class DateRangeFieldTest extends SolrTestCaseJ4 { if (docIds != null && docIds.length > 0) { int i = 1; for (int docId : docIds) { - tests[i++] = "//result/doc/int[@name='id'][.='" + docId + "']"; + tests[i++] = "//result/doc/str[@name='id'][.='" + docId + "']"; } } return tests; diff --git a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java index cf43a68d2e1..43c0006c782 100644 --- a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java +++ b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java @@ -166,31 +166,31 @@ public class DocValuesTest extends SolrTestCaseJ4 { assertU(adoc("id", "4")); assertU(commit()); assertQ(req("q", "*:*", "sort", "floatdv desc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='2']"); + "//str[@name='id'][.='2']"); assertQ(req("q", "*:*", "sort", "intdv desc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='2']"); + "//str[@name='id'][.='2']"); assertQ(req("q", "*:*", "sort", "doubledv desc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='1']"); + "//str[@name='id'][.='1']"); assertQ(req("q", "*:*", "sort", "longdv desc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='1']"); + "//str[@name='id'][.='1']"); assertQ(req("q", "*:*", "sort", "datedv desc", "rows", "1", "fl", "id,datedv"), - "//int[@name='id'][.='2']", + "//str[@name='id'][.='2']", "//result/doc[1]/date[@name='datedv'][.='1997-12-31T23:59:59.999Z']" ); assertQ(req("q", "*:*", "sort", "stringdv desc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='4']"); + "//str[@name='id'][.='4']"); assertQ(req("q", "*:*", "sort", "floatdv asc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='4']"); + "//str[@name='id'][.='4']"); assertQ(req("q", "*:*", "sort", "intdv asc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='3']"); + "//str[@name='id'][.='3']"); assertQ(req("q", "*:*", "sort", "doubledv asc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='3']"); + "//str[@name='id'][.='3']"); assertQ(req("q", "*:*", "sort", "longdv asc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='3']"); + "//str[@name='id'][.='3']"); assertQ(req("q", "*:*", "sort", "datedv asc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='1']"); + "//str[@name='id'][.='1']"); assertQ(req("q", "*:*", "sort", "stringdv asc", "rows", "1", "fl", "id"), - "//int[@name='id'][.='2']"); + "//str[@name='id'][.='2']"); assertQ(req("q", "*:*", "sort", "booldv asc", "rows", "10", "fl", "booldv,stringdv"), "//result/doc[1]/bool[@name='booldv'][.='false']", "//result/doc[2]/bool[@name='booldv'][.='true']", @@ -212,13 +212,13 @@ public class DocValuesTest extends SolrTestCaseJ4 { assertU(adoc("id", "7", "doubledv", "1.7976931348623157E308")); assertU(commit()); assertQ(req("fl", "id", "q", "*:*", "sort", "doubledv asc"), - "//result/doc[1]/int[@name='id'][.='6']", - "//result/doc[2]/int[@name='id'][.='5']", - "//result/doc[3]/int[@name='id'][.='3']", - "//result/doc[4]/int[@name='id'][.='4']", - "//result/doc[5]/int[@name='id'][.='1']", - "//result/doc[6]/int[@name='id'][.='2']", - "//result/doc[7]/int[@name='id'][.='7']" + "//result/doc[1]/str[@name='id'][.='6']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='3']", + "//result/doc[4]/str[@name='id'][.='4']", + "//result/doc[5]/str[@name='id'][.='1']", + "//result/doc[6]/str[@name='id'][.='2']", + "//result/doc[7]/str[@name='id'][.='7']" ); } @@ -366,156 +366,156 @@ public class DocValuesTest extends SolrTestCaseJ4 { assertU(commit()); // string: termquery - assertQ(req("q", "stringdv:car", "sort", "id asc"), + assertQ(req("q", "stringdv:car", "sort", "id_i asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=4]" ); // string: range query - assertQ(req("q", "stringdv:[b TO d]", "sort", "id asc"), + assertQ(req("q", "stringdv:[b TO d]", "sort", "id_i asc"), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=3]", - "//result/doc[3]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=4]" ); // string: prefix query - assertQ(req("q", "stringdv:c*", "sort", "id asc"), + assertQ(req("q", "stringdv:c*", "sort", "id_i asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=3]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=3]", + "//result/doc[2]/str[@name='id'][.=4]" ); // string: wildcard query - assertQ(req("q", "stringdv:c?r", "sort", "id asc"), + assertQ(req("q", "stringdv:c?r", "sort", "id_i asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=4]" ); // string: regexp query - assertQ(req("q", "stringdv:/c[a-b]r/", "sort", "id asc"), + assertQ(req("q", "stringdv:/c[a-b]r/", "sort", "id_i asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=4]" ); // float: termquery - assertQ(req("q", "floatdv:3", "sort", "id asc"), + assertQ(req("q", "floatdv:3", "sort", "id_i asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=3]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=3]", + "//result/doc[2]/str[@name='id'][.=4]" ); // float: rangequery - assertQ(req("q", "floatdv:[2 TO 3]", "sort", "id asc"), + assertQ(req("q", "floatdv:[2 TO 3]", "sort", "id_i asc"), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=3]", - "//result/doc[3]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=4]" ); // (neg) float: termquery - assertQ(req("q", "floatdv:\"-5\"", "sort", "id asc"), + assertQ(req("q", "floatdv:\"-5\"", "sort", "id_i asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=2]" + "//result/doc[1]/str[@name='id'][.=2]" ); // (neg) float: rangequery - assertQ(req("q", "floatdv:[-6 TO -4]", "sort", "id asc"), + assertQ(req("q", "floatdv:[-6 TO -4]", "sort", "id_i asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=2]" + "//result/doc[1]/str[@name='id'][.=2]" ); // (cross zero bounds) float: rangequery - assertQ(req("q", "floatdv:[-6 TO 2.1]", "sort", "id asc"), + assertQ(req("q", "floatdv:[-6 TO 2.1]", "sort", "id_i asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=2]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=2]" ); // int: termquery - assertQ(req("q", "intdv:1", "sort", "id asc"), + assertQ(req("q", "intdv:1", "sort", "id_i asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=3]" + "//result/doc[1]/str[@name='id'][.=3]" ); // int: rangequery - assertQ(req("q", "intdv:[3 TO 4]", "sort", "id asc"), + assertQ(req("q", "intdv:[3 TO 4]", "sort", "id_i asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=2]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=2]" ); // (neg) int: termquery - assertQ(req("q", "intdv:\"-1\"", "sort", "id asc"), + assertQ(req("q", "intdv:\"-1\"", "sort", "id_i asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=4]" ); // (neg) int: rangequery - assertQ(req("q", "intdv:[-1 TO 1]", "sort", "id asc"), + assertQ(req("q", "intdv:[-1 TO 1]", "sort", "id_i asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=3]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=3]", + "//result/doc[2]/str[@name='id'][.=4]" ); // long: termquery - assertQ(req("q", "longdv:1", "sort", "id asc"), + assertQ(req("q", "longdv:1", "sort", "id_i asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=3]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=3]", + "//result/doc[2]/str[@name='id'][.=4]" ); // long: rangequery - assertQ(req("q", "longdv:[1 TO 2]", "sort", "id asc"), + assertQ(req("q", "longdv:[1 TO 2]", "sort", "id_i asc"), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=3]", - "//result/doc[3]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=4]" ); // double: termquery - assertQ(req("q", "doubledv:3.1", "sort", "id asc"), + assertQ(req("q", "doubledv:3.1", "sort", "id_i asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=1]" + "//result/doc[1]/str[@name='id'][.=1]" ); // double: rangequery - assertQ(req("q", "doubledv:[2 TO 3.3]", "sort", "id asc"), + assertQ(req("q", "doubledv:[2 TO 3.3]", "sort", "id_i asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=3]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=3]" ); // (neg) double: termquery - assertQ(req("q", "doubledv:\"-4.3\"", "sort", "id asc"), + assertQ(req("q", "doubledv:\"-4.3\"", "sort", "id_i asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=2]" + "//result/doc[1]/str[@name='id'][.=2]" ); // (neg) double: rangequery - assertQ(req("q", "doubledv:[-6 TO -4]", "sort", "id asc"), + assertQ(req("q", "doubledv:[-6 TO -4]", "sort", "id_i asc"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=2]" + "//result/doc[1]/str[@name='id'][.=2]" ); // (cross zero bounds) double: rangequery - assertQ(req("q", "doubledv:[-6 TO 2.0]", "sort", "id asc"), + assertQ(req("q", "doubledv:[-6 TO 2.0]", "sort", "id_i asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=4]" ); // boolean basic queries: - assertQ(req("q", "booldv:false", "sort", "id asc"), + assertQ(req("q", "booldv:false", "sort", "id_i asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=3]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=3]" ); - assertQ(req("q", "booldv:true", "sort", "id asc"), + assertQ(req("q", "booldv:true", "sort", "id_i asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=4]" ); } @@ -617,7 +617,7 @@ public class DocValuesTest extends SolrTestCaseJ4 { if((minInclusive && minSortable<=valSortable || !minInclusive && minSortable=valSortable || !maxInclusive && maxSortable>valSortable)) { counter++; - tests.add("//result/doc["+counter+"]/int[@name='id'][.="+(k+1)+"]"); + tests.add("//result/doc["+counter+"]/str[@name='id'][.="+(k+1)+"]"); tests.add("//result/doc["+counter+"]/float[@name='score'][.=1.0]"); } } @@ -630,7 +630,7 @@ public class DocValuesTest extends SolrTestCaseJ4 { } log.info("Expected: "+tests); assertQ(req("q", fieldName[i] + ":" + (minInclusive? '[': '{') + min + " TO " + max + (maxInclusive? ']': '}'), - "sort", "id asc", "fl", "id,"+fieldName[i]+",score"), + "sort", "id_i asc", "fl", "id,"+fieldName[i]+",score"), testsArr); } } @@ -654,88 +654,88 @@ public class DocValuesTest extends SolrTestCaseJ4 { assertU(commit()); // Negative Zero to Positive - assertQ(req("q", fieldName[i]+":[-0.0 TO 2.5]", "sort", "id asc", "fl", "id,"+fieldName[i]+",score"), + assertQ(req("q", fieldName[i]+":[-0.0 TO 2.5]", "sort", "id_i asc", "fl", "id,"+fieldName[i]+",score"), "//*[@numFound='1']", - "//result/doc[1]/int[@name='id'][.=1]" + "//result/doc[1]/str[@name='id'][.=1]" ); // Negative to Positive Zero - assertQ(req("q", fieldName[i]+":[-6 TO 0]", "sort", "id asc", "fl", "id,"+fieldName[i]+",score"), + assertQ(req("q", fieldName[i]+":[-6 TO 0]", "sort", "id_i asc", "fl", "id,"+fieldName[i]+",score"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=5]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=5]" ); // Negative to Positive - assertQ(req("q", fieldName[i]+":[-6 TO 2.5]", "sort", "id asc", "fl", "id,"+fieldName[i]+",score"), + assertQ(req("q", fieldName[i]+":[-6 TO 2.5]", "sort", "id_i asc", "fl", "id,"+fieldName[i]+",score"), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=2]", - "//result/doc[3]/int[@name='id'][.=5]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=2]", + "//result/doc[3]/str[@name='id'][.=5]" ); // Positive to Positive - assertQ(req("q", fieldName[i]+":[2 TO 3]", "sort", "id asc", "fl", "id,"+fieldName[i]+",score"), + assertQ(req("q", fieldName[i]+":[2 TO 3]", "sort", "id_i asc", "fl", "id,"+fieldName[i]+",score"), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=3]", - "//result/doc[3]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=4]" ); // Positive to POSITIVE_INF - assertQ(req("q", fieldName[i]+":[2 TO *]", "sort", "id asc", "fl", "id,"+fieldName[i]+",score"), + assertQ(req("q", fieldName[i]+":[2 TO *]", "sort", "id_i asc", "fl", "id,"+fieldName[i]+",score"), "//*[@numFound='4']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=3]", - "//result/doc[3]/int[@name='id'][.=4]", - "//result/doc[4]/int[@name='id'][.=7]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=4]", + "//result/doc[4]/str[@name='id'][.=7]" ); // NEGATIVE_INF to Negative - assertQ(req("q", fieldName[i]+":[* TO -1]", "sort", "id asc", "fl", "id,"+fieldName[i]+",score"), + assertQ(req("q", fieldName[i]+":[* TO -1]", "sort", "id_i asc", "fl", "id,"+fieldName[i]+",score"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=6]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=6]" ); // NEGATIVE_INF to Positive - assertQ(req("q", fieldName[i]+":[* TO 2]", "sort", "id asc", "fl", "id,"+fieldName[i]), + assertQ(req("q", fieldName[i]+":[* TO 2]", "sort", "id_i asc", "fl", "id,"+fieldName[i]), "//*[@numFound='4']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=2]", - "//result/doc[3]/int[@name='id'][.=5]", - "//result/doc[4]/int[@name='id'][.=6]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=2]", + "//result/doc[3]/str[@name='id'][.=5]", + "//result/doc[4]/str[@name='id'][.=6]" ); // NEGATIVE_INF to Positive (non-inclusive) - assertQ(req("q", fieldName[i]+":[* TO 2}", "sort", "id asc", "fl", "id,"+fieldName[i]), + assertQ(req("q", fieldName[i]+":[* TO 2}", "sort", "id_i asc", "fl", "id,"+fieldName[i]), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=5]", - "//result/doc[3]/int[@name='id'][.=6]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=5]", + "//result/doc[3]/str[@name='id'][.=6]" ); // Negative to POSITIVE_INF - assertQ(req("q", fieldName[i]+":[-6 TO *]", "sort", "id asc", "fl", "id,"+fieldName[i]), + assertQ(req("q", fieldName[i]+":[-6 TO *]", "sort", "id_i asc", "fl", "id,"+fieldName[i]), "//*[@numFound='6']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=2]", - "//result/doc[3]/int[@name='id'][.=3]", - "//result/doc[4]/int[@name='id'][.=4]", - "//result/doc[5]/int[@name='id'][.=5]", - "//result/doc[6]/int[@name='id'][.=7]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=2]", + "//result/doc[3]/str[@name='id'][.=3]", + "//result/doc[4]/str[@name='id'][.=4]", + "//result/doc[5]/str[@name='id'][.=5]", + "//result/doc[6]/str[@name='id'][.=7]" ); // NEGATIVE_INF to POSITIVE_INF - assertQ(req("q", fieldName[i]+":[* TO *]", "sort", "id asc", "fl", "id,"+fieldName[i]+",score"), + assertQ(req("q", fieldName[i]+":[* TO *]", "sort", "id_i asc", "fl", "id,"+fieldName[i]+",score"), "//*[@numFound='7']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=2]", - "//result/doc[3]/int[@name='id'][.=3]", - "//result/doc[4]/int[@name='id'][.=4]", - "//result/doc[5]/int[@name='id'][.=5]", - "//result/doc[6]/int[@name='id'][.=6]", - "//result/doc[7]/int[@name='id'][.=7]", + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=2]", + "//result/doc[3]/str[@name='id'][.=3]", + "//result/doc[4]/str[@name='id'][.=4]", + "//result/doc[5]/str[@name='id'][.=5]", + "//result/doc[6]/str[@name='id'][.=6]", + "//result/doc[7]/str[@name='id'][.=7]", "//result/doc[1]/float[@name='score'][.=1.0]", "//result/doc[2]/float[@name='score'][.=1.0]", "//result/doc[3]/float[@name='score'][.=1.0]", diff --git a/solr/core/src/test/org/apache/solr/schema/IndexSchemaRuntimeFieldTest.java b/solr/core/src/test/org/apache/solr/schema/IndexSchemaRuntimeFieldTest.java index 1a429b291a9..3237b654c4c 100644 --- a/solr/core/src/test/org/apache/solr/schema/IndexSchemaRuntimeFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/IndexSchemaRuntimeFieldTest.java @@ -57,14 +57,14 @@ public class IndexSchemaRuntimeFieldTest extends SolrTestCaseJ4 { assertQ("Make sure they got in", req ,"//*[@numFound='1']" - ,"//result/doc[1]/int[@name='id'][.='10']" + ,"//result/doc[1]/str[@name='id'][.='10']" ); // Check to see if our copy field made it out safely query.setQuery( "dynamic_runtime:aaa" ); assertQ("Make sure they got in", req ,"//*[@numFound='1']" - ,"//result/doc[1]/int[@name='id'][.='10']" + ,"//result/doc[1]/str[@name='id'][.='10']" ); clearIndex(); } diff --git a/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java b/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java index 44716bb3ad3..7790859fcde 100644 --- a/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java +++ b/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java @@ -58,7 +58,7 @@ public class IndexSchemaTest extends SolrTestCaseJ4 { assertQ("Make sure they got in", req ,"//*[@numFound='1']" - ,"//result/doc[1]/int[@name='id'][.='10']" + ,"//result/doc[1]/str[@name='id'][.='10']" ); args = new HashMap<>(); @@ -67,7 +67,7 @@ public class IndexSchemaTest extends SolrTestCaseJ4 { req = new LocalSolrQueryRequest( core, new MapSolrParams( args) ); assertQ("dynamic source", req ,"//*[@numFound='1']" - ,"//result/doc[1]/int[@name='id'][.='10']" + ,"//result/doc[1]/str[@name='id'][.='10']" ); args = new HashMap<>(); @@ -76,7 +76,7 @@ public class IndexSchemaTest extends SolrTestCaseJ4 { req = new LocalSolrQueryRequest( core, new MapSolrParams( args) ); assertQ("dynamic destination", req ,"//*[@numFound='1']" - ,"//result/doc[1]/int[@name='id'][.='10']" + ,"//result/doc[1]/str[@name='id'][.='10']" ); clearIndex(); } diff --git a/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java b/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java index 9b427492e66..b3376c811a6 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java +++ b/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java @@ -105,7 +105,7 @@ public class TestBinaryField extends SolrJettyTestBase { assertEquals(3, beans.size()); for (SolrDocument d : res) { - Integer id = (Integer) d.getFieldValue("id"); + Integer id = Integer.parseInt(d.getFieldValue("id").toString()); byte[] data = (byte[]) d.getFieldValue("data"); if (id == 1) { assertEquals(5, data.length); @@ -133,7 +133,7 @@ public class TestBinaryField extends SolrJettyTestBase { } for (Bean d : beans) { - Integer id = d.id; + Integer id = Integer.parseInt(d.id); byte[] data = d.data; if (id == 1) { assertEquals(5, data.length); @@ -165,7 +165,7 @@ public class TestBinaryField extends SolrJettyTestBase { } public static class Bean{ @Field - int id; + String id; @Field byte [] data; } diff --git a/solr/core/src/test/org/apache/solr/schema/TestCollationField.java b/solr/core/src/test/org/apache/solr/schema/TestCollationField.java index 6351b82932d..f33a902beac 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestCollationField.java +++ b/solr/core/src/test/org/apache/solr/schema/TestCollationField.java @@ -98,8 +98,8 @@ public class TestCollationField extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_de:tone", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" + "//result/doc[1]/str[@name='id'][.=4]", + "//result/doc[2]/str[@name='id'][.=7]" ); } @@ -112,8 +112,8 @@ public class TestCollationField extends SolrTestCaseJ4 { assertQ("Collated RangeQ: ", req("fl", "id", "q", "sort_de:[tone TO tp]", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" + "//result/doc[1]/str[@name='id'][.=4]", + "//result/doc[2]/str[@name='id'][.=7]" ); } @@ -124,8 +124,8 @@ public class TestCollationField extends SolrTestCaseJ4 { assertQ("Collated Sort: ", req("fl", "id", "q", "sort_da:[tz TO töz]", "sort", "sort_da asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=11]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=11]", + "//result/doc[2]/str[@name='id'][.=4]" ); } @@ -137,8 +137,8 @@ public class TestCollationField extends SolrTestCaseJ4 { assertQ("Collated Sort: ", req("fl", "id", "q", "sort_ar:[\u0698 TO \u0633\u0633]", "sort", "sort_ar asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=12]", - "//result/doc[2]/int[@name='id'][.=1]" + "//result/doc[1]/str[@name='id'][.=12]", + "//result/doc[2]/str[@name='id'][.=1]" ); } @@ -161,9 +161,9 @@ public class TestCollationField extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_tr_canon:\"I Will Use Turkish Casıng\"", "sort", "id asc" ), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=3]", - "//result/doc[3]/int[@name='id'][.=5]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=5]" ); } @@ -175,8 +175,8 @@ public class TestCollationField extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_zh_full:Testing", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=6]", - "//result/doc[2]/int[@name='id'][.=8]" + "//result/doc[1]/str[@name='id'][.=6]", + "//result/doc[2]/str[@name='id'][.=8]" ); } @@ -186,10 +186,10 @@ public class TestCollationField extends SolrTestCaseJ4 { */ public void testCustomCollation() { assertQ("Collated TQ: ", - req("fl", "id", "q", "sort_custom:toene", "sort", "id asc" ), + req("fl", "id", "q", "sort_custom:toene" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=10]" + "//result/doc/str[@name='id'][.=4]", + "//result/doc/str[@name='id'][.=10]" ); } } diff --git a/solr/core/src/test/org/apache/solr/schema/TestCollationFieldDocValues.java b/solr/core/src/test/org/apache/solr/schema/TestCollationFieldDocValues.java index 1de339a7c99..c950f673649 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestCollationFieldDocValues.java +++ b/solr/core/src/test/org/apache/solr/schema/TestCollationFieldDocValues.java @@ -97,8 +97,8 @@ public class TestCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_de:tone", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" + "//result/doc[1]/str[@name='id'][.=4]", + "//result/doc[2]/str[@name='id'][.=7]" ); } @@ -111,8 +111,8 @@ public class TestCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated RangeQ: ", req("fl", "id", "q", "sort_de:[tone TO tp]", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" + "//result/doc[1]/str[@name='id'][.=4]", + "//result/doc[2]/str[@name='id'][.=7]" ); } @@ -123,8 +123,8 @@ public class TestCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated Sort: ", req("fl", "id", "q", "sort_da:[tz TO töz]", "sort", "sort_da asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=11]", - "//result/doc[2]/int[@name='id'][.=4]" + "//result/doc[1]/str[@name='id'][.=11]", + "//result/doc[2]/str[@name='id'][.=4]" ); } @@ -136,8 +136,8 @@ public class TestCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated Sort: ", req("fl", "id", "q", "sort_ar:[\u0698 TO \u0633\u0633]", "sort", "sort_ar asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=12]", - "//result/doc[2]/int[@name='id'][.=1]" + "//result/doc[1]/str[@name='id'][.=12]", + "//result/doc[2]/str[@name='id'][.=1]" ); } @@ -160,9 +160,9 @@ public class TestCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_tr_canon:\"I Will Use Turkish Casıng\"", "sort", "id asc" ), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=3]", - "//result/doc[3]/int[@name='id'][.=5]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=5]" ); } @@ -174,8 +174,8 @@ public class TestCollationFieldDocValues extends SolrTestCaseJ4 { assertQ("Collated TQ: ", req("fl", "id", "q", "sort_zh_full:Testing", "sort", "id asc" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=6]", - "//result/doc[2]/int[@name='id'][.=8]" + "//result/doc[1]/str[@name='id'][.=6]", + "//result/doc[2]/str[@name='id'][.=8]" ); } @@ -185,10 +185,10 @@ public class TestCollationFieldDocValues extends SolrTestCaseJ4 { */ public void testCustomCollation() { assertQ("Collated TQ: ", - req("fl", "id", "q", "sort_custom:toene", "sort", "id asc" ), + req("fl", "id", "q", "sort_custom:toene" ), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=10]" + "//result/doc/str[@name='id'][.=4]", + "//result/doc/str[@name='id'][.=10]" ); } } diff --git a/solr/core/src/test/org/apache/solr/schema/TestHalfAndHalfDocValues.java b/solr/core/src/test/org/apache/solr/schema/TestHalfAndHalfDocValues.java index feb9236a550..eb8c196df63 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestHalfAndHalfDocValues.java +++ b/solr/core/src/test/org/apache/solr/schema/TestHalfAndHalfDocValues.java @@ -123,9 +123,9 @@ public class TestHalfAndHalfDocValues extends SolrTestCaseJ4 { // Assert sort order is correct assertQ(req("q", "string_add_dv_later:*", "sort", "string_add_dv_later asc"), "//*[@numFound='3']", - "//result/doc[1]/int[@name='id'][.=1]", - "//result/doc[2]/int[@name='id'][.=2]", - "//result/doc[3]/int[@name='id'][.=3]" + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=2]", + "//result/doc[3]/str[@name='id'][.=3]" ); } diff --git a/solr/core/src/test/org/apache/solr/schema/TestOmitPositions.java b/solr/core/src/test/org/apache/solr/schema/TestOmitPositions.java index 19081b96df6..0fcc9fee06f 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestOmitPositions.java +++ b/solr/core/src/test/org/apache/solr/schema/TestOmitPositions.java @@ -34,8 +34,8 @@ public class TestOmitPositions extends SolrTestCaseJ4 { assertQ("term query: ", req("fl", "id", "q", "nopositionstext:test"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=2]", - "//result/doc[2]/int[@name='id'][.=1]" + "//result/doc[1]/str[@name='id'][.=2]", + "//result/doc[2]/str[@name='id'][.=1]" ); } diff --git a/solr/core/src/test/org/apache/solr/search/SpatialFilterTest.java b/solr/core/src/test/org/apache/solr/search/SpatialFilterTest.java index 720f9b5cfd1..66034c94854 100644 --- a/solr/core/src/test/org/apache/solr/search/SpatialFilterTest.java +++ b/solr/core/src/test/org/apache/solr/search/SpatialFilterTest.java @@ -142,7 +142,7 @@ public class SpatialFilterTest extends SolrTestCaseJ4 { if (docIds != null && docIds.length > 0) { int i = 1; for (int docId : docIds) { - tests[i++] = "//result/doc/int[@name='id'][.='" + docId + "']"; + tests[i++] = "//result/doc/str[@name='id'][.='" + docId + "']"; } } diff --git a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java index ef0404cf810..087c7e11e26 100644 --- a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java @@ -70,111 +70,111 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field=group_s sort=$sort}"); - params.add("sort", "test_i asc, test_l desc, id desc"); + params.add("sort", "test_i asc, test_l desc, id_i desc"); assertQ(req(params) , "*[count(//doc)=2]" - ,"//result/doc[1]/float[@name='id'][.='7.0']" - ,"//result/doc[2]/float[@name='id'][.='3.0']" + ,"//result/doc[1]/str[@name='id'][.='7']" + ,"//result/doc[2]/str[@name='id'][.='3']" ); // group heads are selected using a complex sort, simpler sort used for final groups params = new ModifiableSolrParams(); params.add("q", "*:*"); - params.add("fq", "{!collapse field=group_s sort='test_i asc, test_l desc, id desc'}"); - params.add("sort", "id asc"); + params.add("fq", "{!collapse field=group_s sort='test_i asc, test_l desc, id_i desc'}"); + params.add("sort", "id_i asc"); assertQ(req(params) , "*[count(//doc)=2]" - ,"//result/doc[1]/float[@name='id'][.='3.0']" - ,"//result/doc[2]/float[@name='id'][.='7.0']" + ,"//result/doc[1]/str[@name='id'][.='3']" + ,"//result/doc[2]/str[@name='id'][.='7']" ); // diff up the sort directions, only first clause matters with our data params = new ModifiableSolrParams(); params.add("q", "*:*"); - params.add("fq", "{!collapse field=group_s sort='test_i desc, test_l asc, id asc'}"); - params.add("sort", "id desc"); + params.add("fq", "{!collapse field=group_s sort='test_i desc, test_l asc, id_i asc'}"); + params.add("sort", "id_i desc"); assertQ(req(params) , "*[count(//doc)=2]" - ,"//result/doc[1]/float[@name='id'][.='8.0']" - ,"//result/doc[2]/float[@name='id'][.='4.0']" + ,"//result/doc[1]/str[@name='id'][.='8']" + ,"//result/doc[2]/str[@name='id'][.='4']" ); // tie broken by index order params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field=group_s sort='test_l desc'}"); - params.add("sort", "id desc"); + params.add("sort", "id_i desc"); assertQ(req(params) , "*[count(//doc)=2]" - ,"//result/doc[1]/float[@name='id'][.='6.0']" - ,"//result/doc[2]/float[@name='id'][.='2.0']" + ,"//result/doc[1]/str[@name='id'][.='6']" + ,"//result/doc[2]/str[@name='id'][.='2']" ); // score, then tiebreakers; note top level sort by score ASCENDING (just for weirdness) params = new ModifiableSolrParams(); params.add("q", "*:* term_s:YYYY"); - params.add("fq", "{!collapse field=group_s sort='score desc, test_l desc, test_i asc, id asc'}"); + params.add("fq", "{!collapse field=group_s sort='score desc, test_l desc, test_i asc, id_i asc'}"); params.add("sort", "score asc"); assertQ(req(params) , "*[count(//doc)=2]" - ,"//result/doc[1]/float[@name='id'][.='2.0']" - ,"//result/doc[2]/float[@name='id'][.='5.0']" + ,"//result/doc[1]/str[@name='id'][.='2']" + ,"//result/doc[2]/str[@name='id'][.='5']" ); // score, then tiebreakers; note no score in top level sort/fl to check needsScores logic params = new ModifiableSolrParams(); params.add("q", "*:* term_s:YYYY"); - params.add("fq", "{!collapse field=group_s sort='score desc, test_l desc, test_i asc, id asc'}"); - params.add("sort", "id desc"); + params.add("fq", "{!collapse field=group_s sort='score desc, test_l desc, test_i asc, id_i asc'}"); + params.add("sort", "id_i desc"); assertQ(req(params) , "*[count(//doc)=2]" - ,"//result/doc[1]/float[@name='id'][.='5.0']" - ,"//result/doc[2]/float[@name='id'][.='2.0']" + ,"//result/doc[1]/str[@name='id'][.='5']" + ,"//result/doc[2]/str[@name='id'][.='2']" ); // term_s desc -- term_s is missing from many docs, and uses sortMissingLast=true params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field=group_s sort='term_s desc, test_l asc'}"); - params.add("sort", "id asc"); + params.add("sort", "id_i asc"); assertQ(req(params) , "*[count(//doc)=2]" - ,"//result/doc[1]/float[@name='id'][.='1.0']" - ,"//result/doc[2]/float[@name='id'][.='5.0']" + ,"//result/doc[1]/str[@name='id'][.='1']" + ,"//result/doc[2]/str[@name='id'][.='5']" ); // term_s asc -- term_s is missing from many docs, and uses sortMissingLast=true params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field=group_s sort='term_s asc, test_l asc'}"); - params.add("sort", "id asc"); + params.add("sort", "id_i asc"); assertQ(req(params) , "*[count(//doc)=2]" - ,"//result/doc[1]/float[@name='id'][.='1.0']" - ,"//result/doc[2]/float[@name='id'][.='7.0']" + ,"//result/doc[1]/str[@name='id'][.='1']" + ,"//result/doc[2]/str[@name='id'][.='7']" ); // collapse on int field params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field=test_i sort='term_s asc, group_s asc'}"); - params.add("sort", "id asc"); + params.add("sort", "id_i asc"); assertQ(req(params) , "*[count(//doc)=2]" - ,"//result/doc[1]/float[@name='id'][.='4.0']" - ,"//result/doc[2]/float[@name='id'][.='7.0']" + ,"//result/doc[1]/str[@name='id'][.='4']" + ,"//result/doc[2]/str[@name='id'][.='7']" ); // collapse on term_s (very sparse) with nullPolicy=collapse params = new ModifiableSolrParams(); params.add("q", "*:*"); - params.add("fq", "{!collapse field=term_s nullPolicy=collapse sort='test_i asc, test_l desc, id asc'}"); - params.add("sort", "test_l asc, id asc"); + params.add("fq", "{!collapse field=term_s nullPolicy=collapse sort='test_i asc, test_l desc, id_i asc'}"); + params.add("sort", "test_l asc, id_i asc"); assertQ(req(params) , "*[count(//doc)=3]" - ,"//result/doc[1]/float[@name='id'][.='5.0']" - ,"//result/doc[2]/float[@name='id'][.='2.0']" - ,"//result/doc[3]/float[@name='id'][.='7.0']" + ,"//result/doc[1]/str[@name='id'][.='5']" + ,"//result/doc[2]/str[@name='id'][.='2']" + ,"//result/doc[3]/str[@name='id'][.='7']" ); // sort local param + elevation @@ -184,11 +184,11 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("sort", "test_l asc"); params.add("qt", "/elevate"); params.add("forceElevation", "true"); - params.add("elevateIds", "4.0"); + params.add("elevateIds", "4"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='4.0']", - "//result/doc[2]/float[@name='id'][.='5.0']"); + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='5']"); // params = new ModifiableSolrParams(); params.add("q", "*:*"); @@ -196,11 +196,11 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("sort", "test_l asc"); params.add("qt", "/elevate"); params.add("forceElevation", "true"); - params.add("elevateIds", "7.0"); + params.add("elevateIds", "7"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='7.0']", - "//result/doc[2]/float[@name='id'][.='1.0']"); + "//result/doc[1]/str[@name='id'][.='7']", + "//result/doc[2]/str[@name='id'][.='1']"); } @@ -250,13 +250,13 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("q", "*:*"); params.add("fq", "{!collapse field=group_i min=test_f}"); assertQ(req(params), "*[count(//doc)=1]", - "//result/doc[1]/float[@name='id'][.='6.0']"); + "//result/doc[1]/str[@name='id'][.='6']"); params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field=group_i max=test_f}"); assertQ(req(params), "*[count(//doc)=1]", - "//result/doc[1]/float[@name='id'][.='2.0']"); + "//result/doc[1]/str[@name='id'][.='2']"); } @@ -378,8 +378,8 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("defType", "edismax"); params.add("bf", "field(test_i)"); assertQ(req(params, "indent", "on"), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='2.0']", - "//result/doc[2]/float[@name='id'][.='6.0']" + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='6']" ); @@ -391,10 +391,10 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("bf", "field(test_i)"); params.add("sort",""); assertQ(req(params), "*[count(//doc)=4]", - "//result/doc[1]/float[@name='id'][.='3.0']", - "//result/doc[2]/float[@name='id'][.='4.0']", - "//result/doc[3]/float[@name='id'][.='2.0']", - "//result/doc[4]/float[@name='id'][.='6.0']" + "//result/doc[1]/str[@name='id'][.='3']", + "//result/doc[2]/str[@name='id'][.='4']", + "//result/doc[3]/str[@name='id'][.='2']", + "//result/doc[4]/str[@name='id'][.='6']" ); // Test value source collapse criteria @@ -403,9 +403,9 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("fq", "{!collapse field="+group+" nullPolicy=collapse min=field(test_i)"+hint+"}"); params.add("sort", "test_i desc"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='4.0']", - "//result/doc[2]/float[@name='id'][.='1.0']", - "//result/doc[3]/float[@name='id'][.='5.0']" + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='1']", + "//result/doc[3]/str[@name='id'][.='5']" ); // Test value source collapse criteria with cscore function @@ -415,9 +415,9 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("defType", "edismax"); params.add("bf", "field(test_i)"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='4.0']", - "//result/doc[2]/float[@name='id'][.='1.0']", - "//result/doc[3]/float[@name='id'][.='5.0']" + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='1']", + "//result/doc[3]/str[@name='id'][.='5']" ); // Test value source collapse criteria with cscore function but no top level score sort @@ -427,11 +427,11 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("defType", "edismax"); params.add("bf", "field(test_i)"); params.add("fl", "id"); - params.add("sort", "id desc"); + params.add("sort", "id_i desc"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='5.0']", - "//result/doc[2]/float[@name='id'][.='4.0']", - "//result/doc[3]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='5']", + "//result/doc[2]/str[@name='id'][.='4']", + "//result/doc[3]/str[@name='id'][.='1']" ); // Test value source collapse criteria with compound cscore function @@ -441,9 +441,9 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("defType", "edismax"); params.add("bf", "field(test_i)"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='4.0']", - "//result/doc[2]/float[@name='id'][.='1.0']", - "//result/doc[3]/float[@name='id'][.='5.0']" + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='1']", + "//result/doc[3]/str[@name='id'][.='5']" ); //Test collapse by score with elevation @@ -456,10 +456,10 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("qf", "term_s"); params.add("qt", "/elevate"); assertQ(req(params), "*[count(//doc)=4]", - "//result/doc[1]/float[@name='id'][.='1.0']", - "//result/doc[2]/float[@name='id'][.='2.0']", - "//result/doc[3]/float[@name='id'][.='3.0']", - "//result/doc[4]/float[@name='id'][.='6.0']"); + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='3']", + "//result/doc[4]/str[@name='id'][.='6']"); //Test SOLR-5773 with score collapse criteria // try both default & sort localparams as alternate ways to ask for max score @@ -473,9 +473,9 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("qt", "/elevate"); params.add("elevateIds", "1,5"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='1.0']", - "//result/doc[2]/float[@name='id'][.='5.0']", - "//result/doc[3]/float[@name='id'][.='3.0']"); + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='3']"); } //Test SOLR-5773 with max field collapse criteria @@ -490,9 +490,9 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("qt", "/elevate"); params.add("elevateIds", "1,5"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='1.0']", - "//result/doc[2]/float[@name='id'][.='5.0']", - "//result/doc[3]/float[@name='id'][.='3.0']"); + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='3']"); } //Test SOLR-5773 with min field collapse criteria @@ -507,9 +507,9 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("qt", "/elevate"); params.add("elevateIds", "1,5"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='1.0']", - "//result/doc[2]/float[@name='id'][.='5.0']", - "//result/doc[3]/float[@name='id'][.='4.0']"); + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='4']"); } //Test SOLR-5773 elevating documents with null group @@ -522,33 +522,33 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("qt", "/elevate"); params.add("elevateIds", "3,4"); assertQ(req(params), "*[count(//doc)=4]", - "//result/doc[1]/float[@name='id'][.='3.0']", - "//result/doc[2]/float[@name='id'][.='4.0']", - "//result/doc[3]/float[@name='id'][.='2.0']", - "//result/doc[4]/float[@name='id'][.='6.0']"); + "//result/doc[1]/str[@name='id'][.='3']", + "//result/doc[2]/str[@name='id'][.='4']", + "//result/doc[3]/str[@name='id'][.='2']", + "//result/doc[4]/str[@name='id'][.='6']"); // Non trivial sort local param for picking group head params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field="+group+" nullPolicy=collapse sort='term_s asc, test_i asc' "+hint+"}"); - params.add("sort", "id desc"); + params.add("sort", "id_i desc"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='5.0']", - "//result/doc[2]/float[@name='id'][.='4.0']", - "//result/doc[3]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='5']", + "//result/doc[2]/str[@name='id'][.='4']", + "//result/doc[3]/str[@name='id'][.='1']" ); // params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field="+group+" nullPolicy=collapse sort='term_s asc, test_i desc' "+hint+"}"); - params.add("sort", "id desc"); + params.add("sort", "id_i desc"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='6.0']", - "//result/doc[2]/float[@name='id'][.='3.0']", - "//result/doc[3]/float[@name='id'][.='2.0']" + "//result/doc[1]/str[@name='id'][.='6']", + "//result/doc[2]/str[@name='id'][.='3']", + "//result/doc[3]/str[@name='id'][.='2']" ); @@ -559,40 +559,40 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field="+group + min + hint+"}"); - params.add("sort", "id desc"); + params.add("sort", "id_i desc"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='5.0']", - "//result/doc[2]/float[@name='id'][.='1.0']"); + "//result/doc[1]/str[@name='id'][.='5']", + "//result/doc[2]/str[@name='id'][.='1']"); params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field="+group + min + hint+"}"); - params.add("sort", "id asc"); + params.add("sort", "id_i asc"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='1.0']", - "//result/doc[2]/float[@name='id'][.='5.0']"); + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='5']"); params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field="+group + min + hint+"}"); - params.add("sort", "test_l asc,id desc"); + params.add("sort", "test_l asc,id_i desc"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='5.0']", - "//result/doc[2]/float[@name='id'][.='1.0']"); + "//result/doc[1]/str[@name='id'][.='5']", + "//result/doc[2]/str[@name='id'][.='1']"); params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field="+group + min + hint+"}"); - params.add("sort", "score desc,id asc"); + params.add("sort", "score desc,id_i asc"); params.add("defType", "edismax"); - params.add("bf", "field(id)"); + params.add("bf", "field(id_i)"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='5.0']", - "//result/doc[2]/float[@name='id'][.='1.0']"); + "//result/doc[1]/str[@name='id'][.='5']", + "//result/doc[2]/str[@name='id'][.='1']"); } @@ -602,8 +602,8 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("fq", "{!collapse field="+group+" max=test_i"+hint+"}"); params.add("sort", "test_i asc"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='6.0']", - "//result/doc[2]/float[@name='id'][.='2.0']" + "//result/doc[1]/str[@name='id'][.='6']", + "//result/doc[2]/str[@name='id'][.='2']" ); try { @@ -613,8 +613,8 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("fq", "{!collapse field="+group+" min=test_l"+hint+"}"); params.add("sort", "test_i desc"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='1.0']", - "//result/doc[2]/float[@name='id'][.='5.0']"); + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='5']"); //Test collapse by max long field @@ -623,8 +623,8 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("fq", "{!collapse field="+group+" max=test_l"+hint+"}"); params.add("sort", "test_i desc"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='2.0']", - "//result/doc[2]/float[@name='id'][.='6.0']"); + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='6']"); } catch (Exception e) { if(!numeric) { throw e; @@ -638,8 +638,8 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("fq", "{!collapse field="+group+" min=test_f"+hint+"}"); params.add("sort", "test_i desc"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='2.0']", - "//result/doc[2]/float[@name='id'][.='6.0']"); + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='6']"); //Test collapse by min float field params = new ModifiableSolrParams(); @@ -647,23 +647,23 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("fq", "{!collapse field="+group+" max=test_f"+hint+"}"); params.add("sort", "test_i asc"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='5.0']", - "//result/doc[2]/float[@name='id'][.='1.0']"); + "//result/doc[1]/str[@name='id'][.='5']", + "//result/doc[2]/str[@name='id'][.='1']"); //Test collapse by min float field sort by score params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field="+group+" max=test_f"+hint+"}"); params.add("defType", "edismax"); - params.add("bf", "field(id)"); + params.add("bf", "field(id_i)"); params.add("fl", "score, id"); params.add("facet","true"); params.add("fq", "{!tag=test}term_s:YYYY"); params.add("facet.field", "{!ex=test}term_s"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='5.0']", - "//result/doc[2]/float[@name='id'][.='1.0']"); + "//result/doc[1]/str[@name='id'][.='5']", + "//result/doc[2]/str[@name='id'][.='1']"); // Test collapse using selector field in no docs // tie selector in all of these cases @@ -690,8 +690,8 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { // attempting to use cscore() in sort local param should fail assertQEx("expected error trying to sort on a function that includes cscore()", - req(params("q", "{!func}sub(sub(test_l,1000),id)", - "fq", "{!collapse field="+group+" sort='abs(cscore()) asc, id asc'}", + req(params("q", "{!func}sub(sub(test_l,1000),id_i)", + "fq", "{!collapse field="+group+" sort='abs(cscore()) asc, id_i asc'}", "sort", "score asc")), SolrException.ErrorCode.BAD_REQUEST); @@ -718,9 +718,9 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { assertQ(req(collapse, "q", "*:*", "sort", "test_i desc"), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='4.0']", - "//result/doc[2]/float[@name='id'][.='1.0']", - "//result/doc[3]/float[@name='id'][.='5.0']"); + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='1']", + "//result/doc[3]/str[@name='id'][.='5']"); } @@ -728,22 +728,22 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field="+group+" max=test_f nullPolicy=expand"+hint+"}"); - params.add("sort", "id desc"); + params.add("sort", "id_i desc"); assertQ(req(params), "*[count(//doc)=4]", - "//result/doc[1]/float[@name='id'][.='5.0']", - "//result/doc[2]/float[@name='id'][.='4.0']", - "//result/doc[3]/float[@name='id'][.='3.0']", - "//result/doc[4]/float[@name='id'][.='1.0']"); + "//result/doc[1]/str[@name='id'][.='5']", + "//result/doc[2]/str[@name='id'][.='4']", + "//result/doc[3]/str[@name='id'][.='3']", + "//result/doc[4]/str[@name='id'][.='1']"); //Test nullPolicy collapse params = new ModifiableSolrParams(); params.add("q", "*:*"); params.add("fq", "{!collapse field="+group+" max=test_f nullPolicy=collapse"+hint+"}"); - params.add("sort", "id desc"); + params.add("sort", "id_i desc"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='5.0']", - "//result/doc[2]/float[@name='id'][.='4.0']", - "//result/doc[3]/float[@name='id'][.='1.0']"); + "//result/doc[1]/str[@name='id'][.='5']", + "//result/doc[2]/str[@name='id'][.='4']", + "//result/doc[3]/str[@name='id'][.='1']"); params = new ModifiableSolrParams(); @@ -778,9 +778,9 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { params.add("qf", "term_s"); params.add("qt", "/elevate"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='3.0']", - "//result/doc[2]/float[@name='id'][.='6.0']", - "//result/doc[3]/float[@name='id'][.='7.0']"); + "//result/doc[1]/str[@name='id'][.='3']", + "//result/doc[2]/str[@name='id'][.='6']", + "//result/doc[3]/str[@name='id'][.='7']"); } @@ -850,25 +850,25 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 { // w/default nullPolicy, no groups found params = new ModifiableSolrParams(); params.add("q", "*:*"); - params.add("sort", "id desc"); + params.add("sort", "id_i desc"); params.add("fq", "{!collapse "+group+" "+selector+"}"); assertQ(req(params), "*[count(//doc)=0]"); // w/nullPolicy=expand, every doc found params = new ModifiableSolrParams(); params.add("q", "*:*"); - params.add("sort", "id desc"); + params.add("sort", "id_i desc"); params.add("fq", "{!collapse field="+group+" nullPolicy=expand "+selector+"}"); assertQ(req(params) , "*[count(//doc)=8]" - ,"//result/doc[1]/float[@name='id'][.='8.0']" - ,"//result/doc[2]/float[@name='id'][.='7.0']" - ,"//result/doc[3]/float[@name='id'][.='6.0']" - ,"//result/doc[4]/float[@name='id'][.='5.0']" - ,"//result/doc[5]/float[@name='id'][.='4.0']" - ,"//result/doc[6]/float[@name='id'][.='3.0']" - ,"//result/doc[7]/float[@name='id'][.='2.0']" - ,"//result/doc[8]/float[@name='id'][.='1.0']" + ,"//result/doc[1]/str[@name='id'][.='8']" + ,"//result/doc[2]/str[@name='id'][.='7']" + ,"//result/doc[3]/str[@name='id'][.='6']" + ,"//result/doc[4]/str[@name='id'][.='5']" + ,"//result/doc[5]/str[@name='id'][.='4']" + ,"//result/doc[6]/str[@name='id'][.='3']" + ,"//result/doc[7]/str[@name='id'][.='2']" + ,"//result/doc[8]/str[@name='id'][.='1']" ); diff --git a/solr/core/src/test/org/apache/solr/search/TestComplexPhraseLeadingWildcard.java b/solr/core/src/test/org/apache/solr/search/TestComplexPhraseLeadingWildcard.java index 6c48cc34e63..ffccaca1d55 100644 --- a/solr/core/src/test/org/apache/solr/search/TestComplexPhraseLeadingWildcard.java +++ b/solr/core/src/test/org/apache/solr/search/TestComplexPhraseLeadingWildcard.java @@ -106,7 +106,7 @@ public class TestComplexPhraseLeadingWildcard extends SolrTestCaseJ4 { xpathes[0]= "//result[@numFound=" +ids.length+ "]"; int i=1; for(String id : ids) { - xpathes[i++] = "//doc/int[@name='id' and text()='"+id+"']"; + xpathes[i++] = "//doc/str[@name='id' and text()='"+id+"']"; } return xpathes; } diff --git a/solr/core/src/test/org/apache/solr/search/TestCustomSort.java b/solr/core/src/test/org/apache/solr/search/TestCustomSort.java index ca33ea0f4cf..74b83011fa0 100644 --- a/solr/core/src/test/org/apache/solr/search/TestCustomSort.java +++ b/solr/core/src/test/org/apache/solr/search/TestCustomSort.java @@ -49,7 +49,7 @@ public class TestCustomSort extends SolrTestCaseJ4 { assertU(adoc(sdoc("id", "13", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0x80, 0x11, 0x33 })))); // 12 assertU(commit()); - assertQ(req("q", "*:*", "fl", "id", "sort", "payload asc", "rows", "20") + assertQ(req("q", "*:*", "fl", "id_i", "sort", "payload asc", "rows", "20") , "//result[@numFound='13']" // , "//result/doc[int='7' and position()=1]" // 7 00 3c 73 , "//result/doc[int='1' and position()=2]" // 1 12 62 15 @@ -64,7 +64,7 @@ public class TestCustomSort extends SolrTestCaseJ4 { , "//result/doc[int='8' and position()=11]" // 8 59 2d 4d , "//result/doc[int='13' and position()=12]" // 13 80 11 33 , "//result/doc[int='11' and position()=13]"); // 11 ff af 9c - assertQ(req("q", "*:*", "fl", "id", "sort", "payload desc", "rows", "20") + assertQ(req("q", "*:*", "fl", "id_i", "sort", "payload desc", "rows", "20") , "//result[@numFound='13']" // , "//result/doc[int='11' and position()=1]" // 11 ff af 9c , "//result/doc[int='13' and position()=2]" // 13 80 11 33 @@ -79,44 +79,44 @@ public class TestCustomSort extends SolrTestCaseJ4 { , "//result/doc[int='6' and position()=11]" // 6 1a 2b 3c 00 00 03 , "//result/doc[int='1' and position()=12]" // 1 12 62 15 , "//result/doc[int='7' and position()=13]"); // 7 00 3c 73 - assertQ(req("q", "text:a", "fl", "id", "sort", "payload asc", "rows", "20") + assertQ(req("q", "text:a", "fl", "id_i", "sort", "payload asc", "rows", "20") , "//result[@numFound='4']" // , "//result/doc[int='1' and position()=1]" // 1 12 62 15 , "//result/doc[int='3' and position()=2]" // 3 35 32 58 , "//result/doc[int='5' and position()=3]" // 5 35 35 10 00 , "//result/doc[int='9' and position()=4]"); // 9 39 79 7a - assertQ(req("q", "text:a", "fl", "id", "sort", "payload desc", "rows", "20") + assertQ(req("q", "text:a", "fl", "id_i", "sort", "payload desc", "rows", "20") , "//result[@numFound='4']" // , "//result/doc[int='9' and position()=1]" // 9 39 79 7a , "//result/doc[int='5' and position()=2]" // 5 35 35 10 00 , "//result/doc[int='3' and position()=3]" // 3 35 32 58 , "//result/doc[int='1' and position()=4]"); // 1 12 62 15 - assertQ(req("q", "text:b", "fl", "id", "sort", "payload asc", "rows", "20") + assertQ(req("q", "text:b", "fl", "id_i", "sort", "payload asc", "rows", "20") , "//result[@numFound='3']" // , "//result/doc[int='4' and position()=1]" // 4 25 21 15 , "//result/doc[int='2' and position()=2]" // 2 25 21 16 , "//result/doc[int='10' and position()=3]"); // 10 31 39 7c - assertQ(req("q", "text:b", "fl", "id", "sort", "payload desc", "rows", "20") + assertQ(req("q", "text:b", "fl", "id_i", "sort", "payload desc", "rows", "20") , "//result[@numFound='3']" // , "//result/doc[int='10' and position()=1]" // 10 31 39 7c , "//result/doc[int='2' and position()=2]" // 2 25 21 16 , "//result/doc[int='4' and position()=3]"); // 4 25 21 15 - assertQ(req("q", "text:c", "fl", "id", "sort", "payload asc", "rows", "20") + assertQ(req("q", "text:c", "fl", "id_i", "sort", "payload asc", "rows", "20") , "//result[@numFound='3']" // , "//result/doc[int='7' and position()=1]" // 7 00 3c 73 , "//result/doc[int='6' and position()=2]" // 6 1a 2b 3c 00 00 03 , "//result/doc[int='8' and position()=3]"); // 8 59 2d 4d - assertQ(req("q", "text:c", "fl", "id", "sort", "payload desc", "rows", "20") + assertQ(req("q", "text:c", "fl", "id_i", "sort", "payload desc", "rows", "20") , "//result[@numFound='3']" // , "//result/doc[int='8' and position()=1]" // 8 59 2d 4d , "//result/doc[int='6' and position()=2]" // 6 1a 2b 3c 00 00 03 , "//result/doc[int='7' and position()=3]"); // 7 00 3c 73 - assertQ(req("q", "text:d", "fl", "id", "sort", "payload asc", "rows", "20") + assertQ(req("q", "text:d", "fl", "id_i", "sort", "payload asc", "rows", "20") , "//result[@numFound='3']" // , "//result/doc[int='12' and position()=1]" // 12 34 dd 4d , "//result/doc[int='13' and position()=2]" // 13 80 11 33 , "//result/doc[int='11' and position()=3]"); // 11 ff af 9c - assertQ(req("q", "text:d", "fl", "id", "sort", "payload desc", "rows", "20") + assertQ(req("q", "text:d", "fl", "id_i", "sort", "payload desc", "rows", "20") , "//result[@numFound='3']" // , "//result/doc[int='11' and position()=1]" // 11 ff af 9c , "//result/doc[int='13' and position()=2]" // 13 80 11 33 diff --git a/solr/core/src/test/org/apache/solr/search/TestFieldSortValues.java b/solr/core/src/test/org/apache/solr/search/TestFieldSortValues.java index 781127b92dd..3e07bf311f3 100644 --- a/solr/core/src/test/org/apache/solr/search/TestFieldSortValues.java +++ b/solr/core/src/test/org/apache/solr/search/TestFieldSortValues.java @@ -41,10 +41,10 @@ public class TestFieldSortValues extends SolrTestCaseJ4 { // payload is backed by a custom sort field which returns the payload value mod 3 assertQ(req("q", "*:*", "fl", "id", "sort", "payload asc, id asc", "fsv", "true") - , "//result/doc[int='2' and position()=1]" - , "//result/doc[int='3' and position()=2]" - , "//result/doc[int='5' and position()=3]" - , "//result/doc[int='1' and position()=4]" - , "//result/doc[int='4' and position()=5]"); + , "//result/doc[str='2' and position()=1]" + , "//result/doc[str='3' and position()=2]" + , "//result/doc[str='5' and position()=3]" + , "//result/doc[str='1' and position()=4]" + , "//result/doc[str='4' and position()=5]"); } } diff --git a/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java b/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java index fb982b871df..5ceb2246a66 100644 --- a/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java +++ b/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java @@ -346,4 +346,4 @@ public class TestFoldingMultitermQuery extends SolrTestCaseJ4 { public void testCJKWidth() { assertQ(req("q", "content_width:ヴィ*"), "//result[@numFound='1']"); } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/search/TestGraphTermsQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestGraphTermsQParserPlugin.java index 05ec9822577..1cb927d3644 100644 --- a/solr/core/src/test/org/apache/solr/search/TestGraphTermsQParserPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestGraphTermsQParserPlugin.java @@ -74,11 +74,11 @@ public class TestGraphTermsQParserPlugin extends SolrTestCaseJ4 { params.add("q", "{!graphTerms f=group_s maxDocFreq=10}1,2"); params.add("sort", "id asc"); assertQ(req(params, "indent", "on"), "*[count(//doc)=5]", - "//result/doc[1]/float[@name='id'][.='1.0']", - "//result/doc[2]/float[@name='id'][.='2.0']", - "//result/doc[3]/float[@name='id'][.='5.0']", - "//result/doc[4]/float[@name='id'][.='6.0']", - "//result/doc[5]/float[@name='id'][.='7.0']" + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='5']", + "//result/doc[4]/str[@name='id'][.='6']", + "//result/doc[5]/str[@name='id'][.='7']" ); //Test without maxDocFreq param. Should default to Integer.MAX_VALUE and match all terms. @@ -86,11 +86,11 @@ public class TestGraphTermsQParserPlugin extends SolrTestCaseJ4 { params.add("q", "{!graphTerms f=group_s}1,2"); params.add("sort", "id asc"); assertQ(req(params, "indent", "on"), "*[count(//doc)=5]", - "//result/doc[1]/float[@name='id'][.='1.0']", - "//result/doc[2]/float[@name='id'][.='2.0']", - "//result/doc[3]/float[@name='id'][.='5.0']", - "//result/doc[4]/float[@name='id'][.='6.0']", - "//result/doc[5]/float[@name='id'][.='7.0']" + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='5']", + "//result/doc[4]/str[@name='id'][.='6']", + "//result/doc[5]/str[@name='id'][.='7']" ); params = new ModifiableSolrParams(); @@ -104,11 +104,11 @@ public class TestGraphTermsQParserPlugin extends SolrTestCaseJ4 { params.add("q", "{!graphTerms f=test_ti maxDocFreq=10}5,10"); params.add("sort", "id asc"); assertQ(req(params, "indent", "on"), "*[count(//doc)=5]", - "//result/doc[1]/float[@name='id'][.='1.0']", - "//result/doc[2]/float[@name='id'][.='2.0']", - "//result/doc[3]/float[@name='id'][.='5.0']", - "//result/doc[4]/float[@name='id'][.='6.0']", - "//result/doc[5]/float[@name='id'][.='7.0']" + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='5']", + "//result/doc[4]/str[@name='id'][.='6']", + "//result/doc[5]/str[@name='id'][.='7']" ); //Test with int field @@ -116,8 +116,8 @@ public class TestGraphTermsQParserPlugin extends SolrTestCaseJ4 { params.add("q", "{!graphTerms f=test_ti maxDocFreq=2}5,10"); params.add("sort", "id asc"); assertQ(req(params, "indent", "on"), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='6.0']", - "//result/doc[2]/float[@name='id'][.='7.0']" + "//result/doc[1]/str[@name='id'][.='6']", + "//result/doc[2]/str[@name='id'][.='7']" ); } } diff --git a/solr/core/src/test/org/apache/solr/search/TestHashQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestHashQParserPlugin.java index 46b7b60e1cf..a4fa1b8ecf2 100644 --- a/solr/core/src/test/org/apache/solr/search/TestHashQParserPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestHashQParserPlugin.java @@ -91,7 +91,7 @@ public class TestHashQParserPlugin extends SolrTestCaseJ4 { while(it.hasNext()) { String s = it.next(); - String results = h.validateXPath(response, "*[count(//int[@name='id'][.='"+s+"'])=1]"); + String results = h.validateXPath(response, "*[count(//str[@name='id'][.='"+s+"'])=1]"); if(results == null) { set1.add(s); } @@ -109,7 +109,7 @@ public class TestHashQParserPlugin extends SolrTestCaseJ4 { while(it.hasNext()) { String s = it.next(); - String results = h.validateXPath(response, "*[count(//int[@name='id'][.='"+s+"'])=1]"); + String results = h.validateXPath(response, "*[count(//str[@name='id'][.='"+s+"'])=1]"); if(results == null) { set2.add(s); } @@ -128,7 +128,7 @@ public class TestHashQParserPlugin extends SolrTestCaseJ4 { while(it.hasNext()) { String s = it.next(); - String results = h.validateXPath(response, "*[count(//int[@name='id'][.='"+s+"'])=1]"); + String results = h.validateXPath(response, "*[count(//str[@name='id'][.='"+s+"'])=1]"); if(results == null) { set3.add(s); } @@ -158,7 +158,7 @@ public class TestHashQParserPlugin extends SolrTestCaseJ4 { while(it.hasNext()) { String s = it.next(); - String results = h.validateXPath(response, "*[count(//int[@name='id'][.='"+s+"'])=1]"); + String results = h.validateXPath(response, "*[count(//str[@name='id'][.='"+s+"'])=1]"); if(results == null) { set1.add(s); } @@ -176,7 +176,7 @@ public class TestHashQParserPlugin extends SolrTestCaseJ4 { while(it.hasNext()) { String s = it.next(); - String results = h.validateXPath(response, "*[count(//int[@name='id'][.='"+s+"'])=1]"); + String results = h.validateXPath(response, "*[count(//str[@name='id'][.='"+s+"'])=1]"); if(results == null) { set2.add(s); } @@ -203,7 +203,7 @@ public class TestHashQParserPlugin extends SolrTestCaseJ4 { while(it.hasNext()) { String s = it.next(); - String results = h.validateXPath(response, "*[count(//int[@name='id'][.='"+s+"'])=1]"); + String results = h.validateXPath(response, "*[count(//str[@name='id'][.='"+s+"'])=1]"); if(results == null) { set1.add(s); } @@ -221,7 +221,7 @@ public class TestHashQParserPlugin extends SolrTestCaseJ4 { while(it.hasNext()) { String s = it.next(); - String results = h.validateXPath(response, "*[count(//int[@name='id'][.='"+s+"'])=1]"); + String results = h.validateXPath(response, "*[count(//str[@name='id'][.='"+s+"'])=1]"); if(results == null) { set2.add(s); } diff --git a/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java b/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java index f282c3f2171..0945ea2413c 100644 --- a/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java +++ b/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java @@ -45,7 +45,7 @@ public class TestQueryTypes extends AbstractSolrTestCase { assertU(adoc("id","10","text_no_analyzer","should just work")); Object[] arr = new Object[] { - "id",999.0 + "id",999 ,"v_s","wow dude" ,"v_t","wow" ,"v_ti",-1 @@ -75,7 +75,7 @@ public class TestQueryTypes extends AbstractSolrTestCase { // normal lucene fielded query assertQ(req( "q",f+":\""+v+'"') ,"//result[@numFound='1']" - ,"//*[@name='id'][.='999.0']" + ,"//*[@name='id'][.='999']" ,"//*[@name='" + f + "'][.='" + v + "']" ); @@ -116,7 +116,7 @@ public class TestQueryTypes extends AbstractSolrTestCase { // frange and function query only work on single valued field types Object[] fc_vals = new Object[] { - "id",999.0 + "id_i",999 ,"v_s","wow dude" ,"v_ti",-1 ,"v_tl",-1234567891234567890L @@ -154,7 +154,7 @@ public class TestQueryTypes extends AbstractSolrTestCase { ,"//result[@numFound='1']" ); - if (!"id".equals(f)) { + if (!"id_i".equals(f)) { assertQ(req( "fq","id:1", "q", "{!frange l=1 u=1}if(exists("+f+"),1,0)" ) ,"//result[@numFound='0']" ); @@ -352,40 +352,40 @@ public class TestQueryTypes extends AbstractSolrTestCase { req("df", "v_t", "q", "{!switch case.x=Dude case.z=Yonik} x ") ,"//result[@numFound='1']" - ,"//*[@name='id'][.='1.0']"); + ,"//*[@name='id'][.='1']"); assertQ("test empty matching switch query", req("df", "v_t", "q", "{!switch case.x=Dude case=Yonik} ") ,"//result[@numFound='1']" - ,"//*[@name='id'][.='2.0']"); + ,"//*[@name='id'][.='2']"); assertQ("test empty matching switch query", req("df", "v_t", "q", "{!switch case.x=Dude case=Yonik v=''}") ,"//result[@numFound='1']" - ,"//*[@name='id'][.='2.0']"); + ,"//*[@name='id'][.='2']"); assertQ("test empty matching switch query", req("df", "v_t", "q", "{!switch case.x=Dude case=Yonik v=$qq}") ,"//result[@numFound='1']" - ,"//*[@name='id'][.='2.0']"); + ,"//*[@name='id'][.='2']"); assertQ("test matching switch query w/deref", req("q", "{!switch case.x=$d case.z=Yonik} x ", "df", "v_t", "d", "Dude") ,"//result[@numFound='1']" - ,"//*[@name='id'][.='1.0']"); + ,"//*[@name='id'][.='1']"); assertQ("test default switch query", req("q", "{!switch default=$d case.x=$d case.z=Yonik}asdf", "df", "v_t", "d", "Dude") ,"//result[@numFound='1']" - ,"//*[@name='id'][.='1.0']"); + ,"//*[@name='id'][.='1']"); assertQ("test empty default switch query", req("q", "{!switch default=$d case.x=$d case.z=Yonik v=$qq}", "df", "v_t", "d", "Dude") ,"//result[@numFound='1']" - ,"//*[@name='id'][.='1.0']"); + ,"//*[@name='id'][.='1']"); try { ignoreException("No\\ default\\, and no switch case"); diff --git a/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java b/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java index 0ba716d87dd..959386a4b2d 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java +++ b/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java @@ -135,8 +135,8 @@ public class TestRangeQuery extends SolrTestCaseJ4 { // simple test of a function rather than just the field assertQ(req("{!frange l=0 u=2}id"), "*[count(//doc)=3]"); - assertQ(req("{!frange l=0 u=2}product(id,2)"), "*[count(//doc)=2]"); - assertQ(req("{!frange l=100 u=102}sum(id,100)"), "*[count(//doc)=3]"); + assertQ(req("{!frange l=0 u=2}product(id_i,2)"), "*[count(//doc)=2]"); + assertQ(req("{!frange l=100 u=102}sum(id_i,100)"), "*[count(//doc)=3]"); for (Map.Entry entry : norm_fields.entrySet()) { diff --git a/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java index 93cc294368e..0fd1a4ae48e 100644 --- a/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java @@ -93,12 +93,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("rows", "6"); params.add("df", "text"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='3.0']", - "//result/doc[2]/float[@name='id'][.='4.0']", - "//result/doc[3]/float[@name='id'][.='2.0']", - "//result/doc[4]/float[@name='id'][.='6.0']", - "//result/doc[5]/float[@name='id'][.='1.0']", - "//result/doc[6]/float[@name='id'][.='5.0']" + "//result/doc[1]/str[@name='id'][.='3']", + "//result/doc[2]/str[@name='id'][.='4']", + "//result/doc[3]/str[@name='id'][.='2']", + "//result/doc[4]/str[@name='id'][.='6']", + "//result/doc[5]/str[@name='id'][.='1']", + "//result/doc[6]/str[@name='id'][.='5']" ); params = new ModifiableSolrParams(); @@ -113,12 +113,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("df", "text"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='2.0']", - "//result/doc[2]/float[@name='id'][.='6.0']", - "//result/doc[3]/float[@name='id'][.='5.0']", - "//result/doc[4]/float[@name='id'][.='4.0']", - "//result/doc[5]/float[@name='id'][.='3.0']", - "//result/doc[6]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='6']", + "//result/doc[3]/str[@name='id'][.='5']", + "//result/doc[4]/str[@name='id'][.='4']", + "//result/doc[5]/str[@name='id'][.='3']", + "//result/doc[6]/str[@name='id'][.='1']" ); //Test with sort by score. @@ -134,12 +134,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("sort", "score desc"); params.add("df", "text"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='2.0']", - "//result/doc[2]/float[@name='id'][.='6.0']", - "//result/doc[3]/float[@name='id'][.='5.0']", - "//result/doc[4]/float[@name='id'][.='4.0']", - "//result/doc[5]/float[@name='id'][.='3.0']", - "//result/doc[6]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='6']", + "//result/doc[3]/str[@name='id'][.='5']", + "//result/doc[4]/str[@name='id'][.='4']", + "//result/doc[5]/str[@name='id'][.='3']", + "//result/doc[6]/str[@name='id'][.='1']" ); @@ -157,12 +157,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("df", "text"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='2.0']", - "//result/doc[2]/float[@name='id'][.='6.0']", - "//result/doc[3]/float[@name='id'][.='5.0']", - "//result/doc[4]/float[@name='id'][.='4.0']", - "//result/doc[5]/float[@name='id'][.='3.0']", - "//result/doc[6]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='6']", + "//result/doc[3]/str[@name='id'][.='5']", + "//result/doc[4]/str[@name='id'][.='4']", + "//result/doc[5]/str[@name='id'][.='3']", + "//result/doc[6]/str[@name='id'][.='1']" ); @@ -179,12 +179,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("qt", "/elevate"); params.add("elevateIds", "1"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='1.0']", - "//result/doc[2]/float[@name='id'][.='2.0']", - "//result/doc[3]/float[@name='id'][.='6.0']", - "//result/doc[4]/float[@name='id'][.='5.0']", - "//result/doc[5]/float[@name='id'][.='4.0']", - "//result/doc[6]/float[@name='id'][.='3.0']" + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='6']", + "//result/doc[4]/str[@name='id'][.='5']", + "//result/doc[5]/str[@name='id'][.='4']", + "//result/doc[6]/str[@name='id'][.='3']" ); @@ -201,12 +201,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("df", "text"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='2.0']", - "//result/doc[2]/float[@name='id'][.='6.0']", - "//result/doc[3]/float[@name='id'][.='5.0']", - "//result/doc[4]/float[@name='id'][.='4.0']", - "//result/doc[5]/float[@name='id'][.='3.0']", - "//result/doc[6]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='6']", + "//result/doc[3]/str[@name='id'][.='5']", + "//result/doc[4]/str[@name='id'][.='4']", + "//result/doc[5]/str[@name='id'][.='3']", + "//result/doc[6]/str[@name='id'][.='1']" ); @@ -223,12 +223,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("elevateIds", "1,4"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='1.0']", //Elevated - "//result/doc[2]/float[@name='id'][.='4.0']", //Elevated - "//result/doc[3]/float[@name='id'][.='2.0']", //Boosted during rerank. - "//result/doc[4]/float[@name='id'][.='6.0']", - "//result/doc[5]/float[@name='id'][.='5.0']", - "//result/doc[6]/float[@name='id'][.='3.0']" + "//result/doc[1]/str[@name='id'][.='1']", //Elevated + "//result/doc[2]/str[@name='id'][.='4']", //Elevated + "//result/doc[3]/str[@name='id'][.='2']", //Boosted during rerank. + "//result/doc[4]/str[@name='id'][.='6']", + "//result/doc[5]/str[@name='id'][.='5']", + "//result/doc[6]/str[@name='id'][.='3']" ); @@ -245,12 +245,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("elevateIds", "4,1"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='4.0']", //Elevated - "//result/doc[2]/float[@name='id'][.='1.0']", //Elevated - "//result/doc[3]/float[@name='id'][.='2.0']", //Boosted during rerank. - "//result/doc[4]/float[@name='id'][.='6.0']", - "//result/doc[5]/float[@name='id'][.='5.0']", - "//result/doc[6]/float[@name='id'][.='3.0']" + "//result/doc[1]/str[@name='id'][.='4']", //Elevated + "//result/doc[2]/str[@name='id'][.='1']", //Elevated + "//result/doc[3]/str[@name='id'][.='2']", //Boosted during rerank. + "//result/doc[4]/str[@name='id'][.='6']", + "//result/doc[5]/str[@name='id'][.='5']", + "//result/doc[6]/str[@name='id'][.='3']" ); @@ -268,12 +268,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("elevateIds", "4,1"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='4.0']", //Elevated - "//result/doc[2]/float[@name='id'][.='1.0']", //Elevated - "//result/doc[3]/float[@name='id'][.='6.0']", - "//result/doc[4]/float[@name='id'][.='5.0']", - "//result/doc[5]/float[@name='id'][.='3.0']", - "//result/doc[6]/float[@name='id'][.='2.0']" //Not in reRankeDocs + "//result/doc[1]/str[@name='id'][.='4']", //Elevated + "//result/doc[2]/str[@name='id'][.='1']", //Elevated + "//result/doc[3]/str[@name='id'][.='6']", + "//result/doc[4]/str[@name='id'][.='5']", + "//result/doc[5]/str[@name='id'][.='3']", + "//result/doc[6]/str[@name='id'][.='2']" //Not in reRankeDocs ); //Test Elevation with start beyond the rerank docs @@ -289,8 +289,8 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("elevateIds", "4,1"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='3.0']", - "//result/doc[2]/float[@name='id'][.='2.0']" //Was not in reRankDocs + "//result/doc[1]/str[@name='id'][.='3']", + "//result/doc[2]/str[@name='id'][.='2']" //Was not in reRankDocs ); //Test Elevation with zero results @@ -320,12 +320,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("rows", "10"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='6.0']", - "//result/doc[2]/float[@name='id'][.='5.0']", - "//result/doc[3]/float[@name='id'][.='4.0']", - "//result/doc[4]/float[@name='id'][.='3.0']", - "//result/doc[5]/float[@name='id'][.='2.0']", - "//result/doc[6]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='6']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='4']", + "//result/doc[4]/str[@name='id'][.='3']", + "//result/doc[5]/str[@name='id'][.='2']", + "//result/doc[6]/str[@name='id'][.='1']" ); params = new ModifiableSolrParams(); @@ -338,12 +338,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("rows", "10"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='6.0']", - "//result/doc[2]/float[@name='id'][.='5.0']", - "//result/doc[3]/float[@name='id'][.='4.0']", - "//result/doc[4]/float[@name='id'][.='3.0']", - "//result/doc[5]/float[@name='id'][.='2.0']", - "//result/doc[6]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='6']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='4']", + "//result/doc[4]/str[@name='id'][.='3']", + "//result/doc[5]/str[@name='id'][.='2']", + "//result/doc[6]/str[@name='id'][.='1']" ); params = new ModifiableSolrParams(); @@ -356,12 +356,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("rows", "10"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='5.0']", - "//result/doc[2]/float[@name='id'][.='6.0']", - "//result/doc[3]/float[@name='id'][.='4.0']", - "//result/doc[4]/float[@name='id'][.='3.0']", - "//result/doc[5]/float[@name='id'][.='2.0']", - "//result/doc[6]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='5']", + "//result/doc[2]/str[@name='id'][.='6']", + "//result/doc[3]/str[@name='id'][.='4']", + "//result/doc[4]/str[@name='id'][.='3']", + "//result/doc[5]/str[@name='id'][.='2']", + "//result/doc[6]/str[@name='id'][.='1']" ); //Test reRankWeight of 0, reranking will have no effect. @@ -375,11 +375,11 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("rows", "5"); assertQ(req(params), "*[count(//doc)=5]", - "//result/doc[1]/float[@name='id'][.='6.0']", - "//result/doc[2]/float[@name='id'][.='5.0']", - "//result/doc[3]/float[@name='id'][.='4.0']", - "//result/doc[4]/float[@name='id'][.='3.0']", - "//result/doc[5]/float[@name='id'][.='2.0']" + "//result/doc[1]/str[@name='id'][.='6']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='4']", + "//result/doc[4]/str[@name='id'][.='3']", + "//result/doc[5]/str[@name='id'][.='2']" ); MetricsMap metrics = (MetricsMap)h.getCore().getCoreMetricManager().getRegistry().getMetrics().get("CACHE.searcher.queryResultCache"); @@ -399,11 +399,11 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("rows", "6"); assertQ(req(params), "*[count(//doc)=5]", - "//result/doc[1]/float[@name='id'][.='6.0']", - "//result/doc[2]/float[@name='id'][.='5.0']", - "//result/doc[3]/float[@name='id'][.='4.0']", - "//result/doc[4]/float[@name='id'][.='2.0']", - "//result/doc[5]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='6']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='4']", + "//result/doc[4]/str[@name='id'][.='2']", + "//result/doc[5]/str[@name='id'][.='1']" ); @@ -424,11 +424,11 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("rows", "6"); assertQ(req(params), "*[count(//doc)=5]", - "//result/doc[1]/float[@name='id'][.='6.0']", - "//result/doc[2]/float[@name='id'][.='5.0']", - "//result/doc[3]/float[@name='id'][.='4.0']", - "//result/doc[4]/float[@name='id'][.='2.0']", - "//result/doc[5]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='6']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='4']", + "//result/doc[4]/str[@name='id'][.='2']", + "//result/doc[5]/str[@name='id'][.='1']" ); stats = metrics.getValue(); @@ -441,7 +441,7 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params = new ModifiableSolrParams(); params.add("rq", "{!"+ReRankQParserPlugin.NAME+" "+ReRankQParserPlugin.RERANK_QUERY+"=$rqq "+ReRankQParserPlugin.RERANK_DOCS+"=6}"); // function query for predictible scores (relative to id) independent of similarity - params.add("q", "{!func}id"); + params.add("q", "{!func}id_i"); // constant score for each clause (unique per doc) for predictible scores independent of similarity // NOTE: biased in favor of doc id == 2 params.add("rqq", "id:1^=10 id:2^=40 id:3^=30 id:4^=40 id:5^=50 id:6^=60"); @@ -450,12 +450,12 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("rows", "6"); assertQ(req(params), "*[count(//doc)=6]", - "//result/doc[1]/float[@name='id'][.='6.0']", - "//result/doc[2]/float[@name='id'][.='5.0']", - "//result/doc[3]/float[@name='id'][.='4.0']", - "//result/doc[4]/float[@name='id'][.='2.0']", // reranked out of orig order - "//result/doc[5]/float[@name='id'][.='3.0']", - "//result/doc[6]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='6']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='4']", + "//result/doc[4]/str[@name='id'][.='2']", // reranked out of orig order + "//result/doc[5]/str[@name='id'][.='3']", + "//result/doc[6]/str[@name='id'][.='1']" ); @@ -469,8 +469,8 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("rows", "5"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='2.0']", - "//result/doc[2]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='1']" ); @@ -485,7 +485,7 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("rows", "1"); assertQ(req(params), "*[count(//doc)=1]", - "//result/doc[1]/float[@name='id'][.='1.0']" + "//result/doc[1]/str[@name='id'][.='1']" ); @@ -556,8 +556,8 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("df", "text"); assertQ(req(params), "*[count(//doc)=2]", - "//result/doc[1]/float[@name='id'][.='8.0']", - "//result/doc[2]/float[@name='id'][.='2.0']" + "//result/doc[1]/str[@name='id'][.='8']", + "//result/doc[2]/str[@name='id'][.='2']" ); //Test Elevation @@ -573,9 +573,9 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 { params.add("elevateIds", "1,4"); assertQ(req(params), "*[count(//doc)=3]", - "//result/doc[1]/float[@name='id'][.='1.0']", //Elevated - "//result/doc[2]/float[@name='id'][.='4.0']", //Elevated - "//result/doc[3]/float[@name='id'][.='8.0']"); //Boosted during rerank. + "//result/doc[1]/str[@name='id'][.='1']", //Elevated + "//result/doc[2]/str[@name='id'][.='4']", //Elevated + "//result/doc[3]/str[@name='id'][.='8']"); //Boosted during rerank. } @Test diff --git a/solr/core/src/test/org/apache/solr/search/TestReload.java b/solr/core/src/test/org/apache/solr/search/TestReload.java index f721d415fd9..872abecda15 100644 --- a/solr/core/src/test/org/apache/solr/search/TestReload.java +++ b/solr/core/src/test/org/apache/solr/search/TestReload.java @@ -98,4 +98,4 @@ public class TestReload extends TestRTGBase { // test framework should ensure that all searchers opened have been closed. } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java b/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java index a4b506b9902..4c633093c84 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java +++ b/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java @@ -246,4 +246,4 @@ public class TestSearchPerf extends AbstractSolrTestCase { } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java index 4b82e6243d3..21bf2c7e06d 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java +++ b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java @@ -1074,4 +1074,4 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 { } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java index b8dbf2a6263..d1be4fe12a1 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java +++ b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java @@ -142,4 +142,4 @@ public class DebugAgg extends AggValueSource { return new FacetLongMerger(); } -} \ No newline at end of file +} diff --git a/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java b/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java index 58a1b533e86..ada12086e4d 100644 --- a/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java +++ b/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java @@ -49,51 +49,51 @@ public class SortByFunctionTest extends AbstractSolrTestCase { assertQ(req("fl", "*,score", "q", "*:*"), "//*[@numFound='4']", "//float[@name='score']='1.0'", - "//result/doc[1]/int[@name='id'][.='1']", - "//result/doc[2]/int[@name='id'][.='2']", - "//result/doc[3]/int[@name='id'][.='3']", - "//result/doc[4]/int[@name='id'][.='4']" + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='3']", + "//result/doc[4]/str[@name='id'][.='4']" ); assertQ(req("fl", "*,score", "q", "*:*", "sort", "score desc"), "//*[@numFound='4']", "//float[@name='score']='1.0'", - "//result/doc[1]/int[@name='id'][.='1']", - "//result/doc[2]/int[@name='id'][.='2']", - "//result/doc[3]/int[@name='id'][.='3']", - "//result/doc[4]/int[@name='id'][.='4']" + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='3']", + "//result/doc[4]/str[@name='id'][.='4']" ); assertQ(req("fl", "id,score", "q", "f_t:ipod", "sort", "score desc"), "//*[@numFound='4']", - "//result/doc[1]/int[@name='id'][.='1']", - "//result/doc[2]/int[@name='id'][.='2']", - "//result/doc[3]/int[@name='id'][.='3']", - "//result/doc[4]/int[@name='id'][.='4']" + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='3']", + "//result/doc[4]/str[@name='id'][.='4']" ); assertQ(req("fl", "*,score", "q", "*:*", "sort", "sum(x_td1, y_td1) desc"), "//*[@numFound='4']", "//float[@name='score']='1.0'", - "//result/doc[1]/int[@name='id'][.='4']", - "//result/doc[2]/int[@name='id'][.='3']", - "//result/doc[3]/int[@name='id'][.='2']", - "//result/doc[4]/int[@name='id'][.='1']" + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='3']", + "//result/doc[3]/str[@name='id'][.='2']", + "//result/doc[4]/str[@name='id'][.='1']" ); assertQ(req("fl", "*,score", "q", "*:*", "sort", "sum(x_td1, y_td1) asc"), "//*[@numFound='4']", "//float[@name='score']='1.0'", - "//result/doc[1]/int[@name='id'][.='1']", - "//result/doc[2]/int[@name='id'][.='2']", - "//result/doc[3]/int[@name='id'][.='3']", - "//result/doc[4]/int[@name='id'][.='4']" + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='3']", + "//result/doc[4]/str[@name='id'][.='4']" ); //the function is equal, w_td1 separates assertQ(req("q", "*:*", "fl", "id", "sort", "sum(z_td1, y_td1) asc, w_td1 asc"), "//*[@numFound='4']", - "//result/doc[1]/int[@name='id'][.='2']", - "//result/doc[2]/int[@name='id'][.='1']", - "//result/doc[3]/int[@name='id'][.='4']", - "//result/doc[4]/int[@name='id'][.='3']" + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='1']", + "//result/doc[3]/str[@name='id'][.='4']", + "//result/doc[4]/str[@name='id'][.='3']" ); } @@ -108,24 +108,24 @@ public class SortByFunctionTest extends AbstractSolrTestCase { assertQ(req("q", "links_mfacet:B", "fl", "id", "sort", "id asc"), "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.='3']", - "//result/doc[2]/int[@name='id'][.='4']" + "//result/doc[1]/str[@name='id'][.='3']", + "//result/doc[2]/str[@name='id'][.='4']" ); assertQ(req("q", "*:*", "fl", "id", "sort", "joindf(id_s1, links_mfacet) desc"), "//*[@numFound='4']", - "//result/doc[1]/int[@name='id'][.='1']", - "//result/doc[2]/int[@name='id'][.='2']", - "//result/doc[3]/int[@name='id'][.='3']", - "//result/doc[4]/int[@name='id'][.='4']" + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='3']", + "//result/doc[4]/str[@name='id'][.='4']" ); assertQ(req("q", "*:*", "fl", "id", "sort", "joindf(id_s1, links_mfacet) asc"), "//*[@numFound='4']", - "//result/doc[1]/int[@name='id'][.='4']", - "//result/doc[2]/int[@name='id'][.='3']", - "//result/doc[3]/int[@name='id'][.='2']", - "//result/doc[4]/int[@name='id'][.='1']" + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='3']", + "//result/doc[3]/str[@name='id'][.='2']", + "//result/doc[4]/str[@name='id'][.='1']" ); } diff --git a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java index 675123c2ddf..f19e4b08530 100644 --- a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java +++ b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java @@ -59,10 +59,10 @@ public class TestFunctionQuery extends SolrTestCaseJ4 { } - void createIndex(String field, float... values) { + void createIndex(String field, int... values) { // lrf.args.put("version","2.0"); - for (float val : values) { - String s = Float.toString(val); + for (int val : values) { + String s = Integer.toString(val); if (field!=null) assertU(adoc("id", s, field, s)); else assertU(adoc("id", s)); @@ -98,6 +98,10 @@ public class TestFunctionQuery extends SolrTestCaseJ4 { } protected void singleTest(String field, String funcTemplate, List args, float... results) { + // NOTE: we're abusing the "results" float[] here ... + // - even elements are ids which must be valid 'ints' + // - odd elements are the expected score values + String parseableQuery = func(field, funcTemplate); List nargs = new ArrayList<>(Arrays.asList("q", parseableQuery @@ -113,13 +117,12 @@ public class TestFunctionQuery extends SolrTestCaseJ4 { List tests = new ArrayList<>(); - // Construct xpaths like the following: - // "//doc[./float[@name='foo_f']='10.0' and ./float[@name='score']='10.0']" - for (int i=0; i - + diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml index da9ad08fda5..f3187ecb748 100644 --- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml +++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml @@ -436,7 +436,7 @@ --> - + diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml index cd1422ba0d6..1886681647a 100644 --- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml +++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml @@ -411,7 +411,7 @@ --> - + diff --git a/solr/solrj/src/test-files/solrj/solr/crazy-path-to-schema.xml b/solr/solrj/src/test-files/solrj/solr/crazy-path-to-schema.xml index 6901cd8aaf2..6954fc643f7 100644 --- a/solr/solrj/src/test-files/solrj/solr/crazy-path-to-schema.xml +++ b/solr/solrj/src/test-files/solrj/solr/crazy-path-to-schema.xml @@ -28,7 +28,7 @@ that just finds leaf and nodes and there's no reason to brea --> - + @@ -42,7 +42,7 @@ that just finds leaf and nodes and there's no reason to brea - + From 3b5f3cc3edfaf22b41f3f969391b56be482fb7b4 Mon Sep 17 00:00:00 2001 From: yonik Date: Thu, 15 Jun 2017 12:53:09 -0400 Subject: [PATCH 082/131] SOLR-7452: add CHANGES for refinement, remove debugging output, don't indent _facet_ refinement info --- solr/CHANGES.txt | 7 +++++++ .../src/java/org/apache/solr/search/facet/FacetModule.java | 6 ++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8db87529101..ec80aece376 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -136,6 +136,13 @@ New Features * SOLR-10677: Expose a diagnostics API to return nodes sorted by load in descending order and any policy violations. (shalin) +* SOLR-7452: Refinement for JSON Facet API: Adding refine:true to any terms facet will cause an + additional distributed search phase (overlapped with field retrieval) that requests additional info + for top facet buckets from shards that did not previously contribute to that bucket. + This will correct counts (and other statistics) for those top buckets collected in the first + phase. (yonik) + + Bug Fixes ---------------------- * SOLR-9262: Connection and read timeouts are being ignored by UpdateShardHandler after SOLR-4509. diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java b/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java index 3407ae41c1b..7fc016bc4b8 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java @@ -234,7 +234,7 @@ public class FacetModule extends SearchComponent { Map finfo = new HashMap<>(1); finfo.put(FACET_REFINE, refinement); - String finfoStr = JSONUtil.toJSON(finfo); + String finfoStr = JSONUtil.toJSON(finfo, -1); // System.err.println("##################### REFINE=" + finfoStr); shardsRefineRequest.params.add(FACET_INFO, finfoStr); @@ -281,10 +281,8 @@ public class FacetModule extends SearchComponent { } if ((sreq.purpose & PURPOSE_REFINE_JSON_FACETS) != 0) { - System.err.println("REFINE FACET RESULT FROM SHARD = " + facet); + // System.err.println("REFINE FACET RESULT FROM SHARD = " + facet); // call merge again with a diff flag set on the context??? -// throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "WORK IN PROGRESS, MERGING FACET REFINEMENT NOT SUPPORTED YET!"); - facetState.mcontext.root = facet; facetState.mcontext.setShard(shardRsp.getShard()); // TODO: roll newShard into setShard? facetState.merger.merge(facet , facetState.mcontext); From 4fe9d4402fea49b11adbff6529c150974a4c7e32 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Thu, 15 Jun 2017 12:20:47 -0400 Subject: [PATCH 083/131] SOLR-10891: BBoxField should support point-based number sub-fields --- solr/CHANGES.txt | 2 ++ .../src/java/org/apache/solr/schema/BBoxField.java | 10 ++++++++-- .../solr/collection1/conf/schema-spatial.xml | 6 ++++++ .../test/org/apache/solr/search/TestSolr4Spatial.java | 10 +++++----- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index ec80aece376..a9ab28ac062 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -430,6 +430,8 @@ Other Changes * SOLR-10851: SolrClients should clarify expectations for solrServerUrl parameter (Jason Gerlowski via Tomás Fernández Löbbe) + +* SOLR-10891: BBoxField should support point-based number sub-fields. (Steve Rowe) * SOLR-10834: Fixed tests and test configs to stop using numeric uniqueKey fields (hossman) diff --git a/solr/core/src/java/org/apache/solr/schema/BBoxField.java b/solr/core/src/java/org/apache/solr/schema/BBoxField.java index 4d773c96ac4..957059f840e 100644 --- a/solr/core/src/java/org/apache/solr/schema/BBoxField.java +++ b/solr/core/src/java/org/apache/solr/schema/BBoxField.java @@ -89,8 +89,11 @@ public class BBoxField extends AbstractSpatialFieldType implements if (!(booleanType instanceof BoolField)) { throw new RuntimeException("Must be a BoolField: " + booleanType); } - if (!(numberType instanceof TrieDoubleField)) { // TODO support TrieField (any trie) once BBoxStrategy does - throw new RuntimeException("Must be TrieDoubleField: " + numberType); + if (numberType.getNumberType() != NumberType.DOUBLE) { + throw new RuntimeException("Must be Double number type: " + numberType); + } + if ( ! numberType.hasProperty(DOC_VALUES)) { + throw new RuntimeException("Must have doc values: " + numberType); } //note: this only works for explicit fields, not dynamic fields @@ -138,6 +141,9 @@ public class BBoxField extends AbstractSpatialFieldType implements final SchemaField solrNumField = new SchemaField("_", numberType);//dummy temp org.apache.lucene.document.FieldType luceneType = (org.apache.lucene.document.FieldType) solrNumField.createField(0.0).fieldType(); + if ( ! (luceneType instanceof LegacyFieldType)) { + luceneType = new org.apache.lucene.document.FieldType(luceneType); + } luceneType.setStored(storeSubFields); //and annoyingly this Field isn't going to have a docValues format because Solr uses a separate Field for that diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml b/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml index 9c7a36f6b7b..a90fa3d00e0 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml @@ -24,6 +24,8 @@ + + @@ -54,6 +56,9 @@ + + @@ -65,6 +70,7 @@ + diff --git a/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java b/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java index 895fb831708..4217a00dee5 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java +++ b/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java @@ -53,7 +53,7 @@ public class TestSolr4Spatial extends SolrTestCaseJ4 { @ParametersFactory public static Iterable parameters() { return Arrays.asList(new Object[][]{ - {"llp"}, {"llp_idx"}, {"llp_dv"}, {"srpt_geohash"}, {"srpt_quad"}, {"srpt_packedquad"}, {"stqpt_geohash"}, {"pointvector"}, {"bbox"} + {"llp"}, {"llp_idx"}, {"llp_dv"}, {"srpt_geohash"}, {"srpt_quad"}, {"srpt_packedquad"}, {"stqpt_geohash"}, {"pointvector"}, {"bbox"}, {"pbbox"} }); } @@ -179,7 +179,7 @@ public class TestSolr4Spatial extends SolrTestCaseJ4 { } private void checkHits(String fieldName, boolean exact, String ptStr, double distKM, double sphereRadius, int count, int ... docIds) throws ParseException { - if (exact && fieldName.equalsIgnoreCase("bbox")) { + if (exact && fieldName.equalsIgnoreCase("bbox") | fieldName.equalsIgnoreCase("pbbox")) { return; // bbox field only supports rectangular query } String [] tests = new String[docIds != null && docIds.length > 0 ? docIds.length + 1 : 1]; @@ -369,9 +369,9 @@ public class TestSolr4Spatial extends SolrTestCaseJ4 { private String radiusQuery(double lat, double lon, double dDEG, String score, String filter) { //Choose between the Solr/Geofilt syntax, and the Lucene spatial module syntax - if (fieldName.equals("bbox") || random().nextBoolean()) { + if (fieldName.equals("bbox") || fieldName.equals("pbbox") || random().nextBoolean()) { //we cheat for bbox strategy which doesn't do radius, only rect. - final String qparser = fieldName.equals("bbox") ? "bbox" : "geofilt"; + final String qparser = (fieldName.equals("bbox") || fieldName.equals("pbbox")) ? "bbox" : "geofilt"; return "{!" + qparser + " " + "sfield=" + fieldName + " " + (score != null ? "score="+score : "") + " " @@ -389,7 +389,7 @@ public class TestSolr4Spatial extends SolrTestCaseJ4 { public void testSortMultiVal() throws Exception { assumeTrue("dist sorting not supported on field " + fieldName, canCalcDistance); assumeFalse("Multivalue not supported for this field", - fieldName.equals("pointvector") || fieldName.equals("bbox")); + fieldName.equals("pointvector") || fieldName.equals("bbox") || fieldName.equals("pbbox")); assertU(adoc("id", "100", fieldName, "1,2"));//1 point assertU(adoc("id", "101", fieldName, "4,-1", fieldName, "3,5"));//2 points, 2nd is pretty close to query point From ee18a5f64101eb25e914e64eecbbc306d591b9ad Mon Sep 17 00:00:00 2001 From: Tomas Fernandez Lobbe Date: Thu, 15 Jun 2017 10:06:37 -0700 Subject: [PATCH 084/131] SOLR-10795: Better testing of PointFields multivalued sort using field(name, min|max) syntax --- .../solr/collection1/conf/schema11.xml | 5 +- .../TestMinMaxOnMultiValuedField.java | 145 +++++++++++++----- 2 files changed, 109 insertions(+), 41 deletions(-) diff --git a/solr/core/src/test-files/solr/collection1/conf/schema11.xml b/solr/core/src/test-files/solr/collection1/conf/schema11.xml index 9584061897b..5c124c797f1 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema11.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema11.xml @@ -365,7 +365,8 @@ valued. --> - + + @@ -420,6 +421,8 @@ valued. --> + + diff --git a/solr/core/src/test/org/apache/solr/search/function/TestMinMaxOnMultiValuedField.java b/solr/core/src/test/org/apache/solr/search/function/TestMinMaxOnMultiValuedField.java index 7ec690f7ece..ff5d8a710b9 100644 --- a/solr/core/src/test/org/apache/solr/search/function/TestMinMaxOnMultiValuedField.java +++ b/solr/core/src/test/org/apache/solr/search/function/TestMinMaxOnMultiValuedField.java @@ -18,12 +18,15 @@ package org.apache.solr.search.function; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.TestUtil; - import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.SchemaField; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.schema.DoubleValueFieldType; +import org.apache.solr.schema.FloatValueFieldType; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.IntValueFieldType; +import org.apache.solr.schema.LongValueFieldType; +import org.apache.solr.schema.SchemaField; import org.junit.Before; import org.junit.BeforeClass; @@ -34,16 +37,20 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { initCore("solrconfig-functionquery.xml","schema11.xml"); - - // sanity check the expected properties of our fields (ie: who broke the schema?) + checkFields(new String[] {"i", "l", "f", "d"}, new String [] {"_p", "_ni_p"}); + checkFields(new String[] {"ti", "tl", "tf", "td"}, new String [] {"", "_dv", "_ni_dv"}); + } + + private static void checkFields(String[] types, String[] suffixes) { + // sanity check the expected properties of our fields (ie: who broke the schema?) IndexSchema schema = h.getCore().getLatestSchema(); - for (String type : new String[] {"i", "l", "f", "d"}) { - for (String suffix : new String [] {"", "_dv", "_ni_dv"}) { - String f = "val_t" + type + "s" + suffix; + for (String type : types) { + for (String suffix : suffixes) { + String f = "val_" + type + "s" + suffix; SchemaField sf = schema.getField(f); assertTrue(f + " is not multivalued", sf.multiValued()); assertEquals(f + " doesn't have expected docValues status", - f.contains("dv"), sf.hasDocValues()); + f.contains("dv") || sf.getType().isPointField(), sf.hasDocValues()); assertEquals(f + " doesn't have expected index status", ! f.contains("ni"), sf.indexed()); } @@ -56,49 +63,62 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { assertU(delQ("*:*")); assertU(commit()); } - + public void testBasics() throws Exception { + testBasics("val_tis_dv", "val_tls_dv", "val_tfs_dv", "val_tds_dv"); + testBasics("val_tis_ni_dv", "val_tls_ni_dv", "val_tfs_ni_dv", "val_tds_ni_dv"); + testBasics("val_is_p", "val_ls_p", "val_fs_p", "val_ds_p"); + testBasics("val_is_ni_p", "val_ls_ni_p", "val_fs_ni_p", "val_ds_ni_p"); + } + + private void testBasics(String intField, String longField, String floatField, String doubleField) throws Exception { + assertTrue("Unexpected int field", h.getCore().getLatestSchema().getField(intField).getType() instanceof IntValueFieldType); + assertTrue("Unexpected long field", h.getCore().getLatestSchema().getField(longField).getType() instanceof LongValueFieldType); + assertTrue("Unexpected float field", h.getCore().getLatestSchema().getField(floatField).getType() instanceof FloatValueFieldType); + assertTrue("Unexpected double field", h.getCore().getLatestSchema().getField(doubleField).getType() instanceof DoubleValueFieldType); + + assertU(delQ("*:*")); assertU(adoc(sdoc("id", "1" // int - ,"val_tis_dv", "42" - ,"val_tis_dv", "9" - ,"val_tis_dv", "-54" + ,intField, "42" + ,intField, "9" + ,intField, "-54" // long - ,"val_tls_dv", "420" - ,"val_tls_dv", "90" - ,"val_tls_dv", "-540" + ,longField, "420" + ,longField, "90" + ,longField, "-540" // float - ,"val_tfs_dv", "-42.5" - ,"val_tfs_dv", "-4.5" - ,"val_tfs_dv", "-13.5" + ,floatField, "-42.5" + ,floatField, "-4.5" + ,floatField, "-13.5" // double - ,"val_tds_dv", "-420.5" - ,"val_tds_dv", "-40.5" - ,"val_tds_dv", "-130.5" + ,doubleField, "-420.5" + ,doubleField, "-40.5" + ,doubleField, "-130.5" ))); assertU(commit()); assertQ(req("q","id:1" // int - ,"fl","exists_min_i:exists(field(val_tis_dv,min))" - ,"fl","exists_max_i:exists(field(val_tis_dv,max))" - ,"fl","min_i:field(val_tis_dv,min)" - ,"fl","max_i:field(val_tis_dv,max)" + ,"fl","exists_min_i:exists(field(" + intField + ",min))" + ,"fl","exists_max_i:exists(field(" + intField + ",max))" + ,"fl","min_i:field(" + intField + ",min)" + ,"fl","max_i:field(" + intField + ",max)" // long - ,"fl","exists_min_l:exists(field(val_tls_dv,min))" - ,"fl","exists_max_l:exists(field(val_tls_dv,max))" - ,"fl","min_l:field(val_tls_dv,min)" - ,"fl","max_l:field(val_tls_dv,max)" + ,"fl","exists_min_l:exists(field(" + longField + ",min))" + ,"fl","exists_max_l:exists(field(" + longField + ",max))" + ,"fl","min_l:field(" + longField + ",min)" + ,"fl","max_l:field(" + longField + ",max)" // float - ,"fl","exists_min_f:exists(field(val_tfs_dv,min))" - ,"fl","exists_max_f:exists(field(val_tfs_dv,max))" - ,"fl","min_f:field(val_tfs_dv,min)" - ,"fl","max_f:field(val_tfs_dv,max)" + ,"fl","exists_min_f:exists(field(" + floatField + ",min))" + ,"fl","exists_max_f:exists(field(" + floatField + ",max))" + ,"fl","min_f:field(" + floatField + ",min)" + ,"fl","max_f:field(" + floatField + ",max)" // double - ,"fl","exists_min_d:exists(field(val_tds_dv,min))" - ,"fl","exists_max_d:exists(field(val_tds_dv,max))" - ,"fl","min_d:field(val_tds_dv,min)" - ,"fl","max_d:field(val_tds_dv,max)" + ,"fl","exists_min_d:exists(field(" + doubleField + ",min))" + ,"fl","exists_max_d:exists(field(" + doubleField + ",max))" + ,"fl","min_d:field(" + doubleField + ",min)" + ,"fl","max_d:field(" + doubleField + ",max)" ) ,"//*[@numFound='1']" @@ -127,12 +147,16 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { } - @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-6709") public void testIntFieldCache() { testSimpleInt("val_tis"); } + public void testPointInt() { + testSimpleInt("val_is_p"); + testSimpleInt("val_is_ni_p"); + } + public void testIntDocValues() { testSimpleInt("val_tis_dv"); testSimpleInt("val_tis_ni_dv"); @@ -147,6 +171,11 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { testSimpleLong("val_tls_dv"); testSimpleLong("val_tls_ni_dv"); } + + public void testPointLong() { + testSimpleLong("val_ls_p"); + testSimpleLong("val_ls_ni_p"); + } @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-6709") @@ -159,6 +188,11 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { testSimpleFloat("val_tfs_ni_dv"); } + public void testPointFloat() { + testSimpleFloat("val_fs_p"); + testSimpleFloat("val_fs_ni_p"); + } + @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-6709") public void testDoubleFieldCache() { testSimpleDouble("val_tds"); @@ -169,6 +203,11 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { testSimpleDouble("val_tds_ni_dv"); } + public void testPointDouble() { + testSimpleDouble("val_ds_p"); + testSimpleDouble("val_ds_ni_p"); + } + public void testBadRequests() { // useful error msg when bogus selector is requested (ie: not min or max) @@ -176,6 +215,11 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { "hoss", req("q","*:*", "fl", "field(val_tds_dv,'hoss')"), SolrException.ErrorCode.BAD_REQUEST); + + assertQEx("no error asking for bogus selector", + "hoss", + req("q","*:*", "fl", "field(val_ds_p,'hoss')"), + SolrException.ErrorCode.BAD_REQUEST); // useful error until/unless LUCENE-6709 assertQEx("no error asking for max on a non docVals field", @@ -200,6 +244,15 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { "string", req("q","*:*", "fl", "field(cat_docValues,'max')"), SolrException.ErrorCode.BAD_REQUEST); + + // MultiValued point field with dv=false + assertFalse(h.getCore().getLatestSchema().getField("val_is_ndv_p").hasDocValues()); + assertTrue(h.getCore().getLatestSchema().getField("val_is_ndv_p").getType().isPointField()); + assertTrue(h.getCore().getLatestSchema().getField("val_is_ndv_p").multiValued()); + assertQEx("no error asking for non-dv point fields", + "docValues", + req("q","*:*", "fl", "field(val_is_ndv_p,'max')"), + SolrException.ErrorCode.BAD_REQUEST); } @@ -212,12 +265,18 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { vals[i] = random().nextInt(); } testSimpleValues("val_tis_dv", int.class, vals); + testSimpleValues("val_is_p", int.class, vals); + testSimpleValues("val_tis_ni_dv", int.class, vals); + testSimpleValues("val_is_ni_p", int.class, vals); // random longs for (int i = 0; i < vals.length; i++) { vals[i] = random().nextLong(); } testSimpleValues("val_tls_dv", long.class, vals); + testSimpleValues("val_ls_p", long.class, vals); + testSimpleValues("val_tls_ni_dv", long.class, vals); + testSimpleValues("val_ls_ni_p", long.class, vals); // random floats for (int i = 0; i < vals.length; i++) { @@ -229,6 +288,9 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { vals[i] = f; } testSimpleValues("val_tfs_dv", float.class, vals); + testSimpleValues("val_fs_p", float.class, vals); + testSimpleValues("val_tfs_ni_dv", float.class, vals); + testSimpleValues("val_fs_ni_p", float.class, vals); // random doubles for (int i = 0; i < vals.length; i++) { @@ -240,6 +302,9 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { vals[i] = d; } testSimpleValues("val_tds_dv", double.class, vals); + testSimpleValues("val_ds_p", double.class, vals); + testSimpleValues("val_tds_ni_dv", double.class, vals); + testSimpleValues("val_ds_ni_p", double.class, vals); } @@ -309,7 +374,7 @@ public class TestMinMaxOnMultiValuedField extends SolrTestCaseJ4 { } /** Tests a single doc with a few explicit values, as well as testing exists with and w/o values */ - protected void testSimpleValues(final String fieldname, final Class clazz, final Comparable... vals) { + protected void testSimpleValues(final String fieldname, final Class clazz, final Comparable... vals) { clearIndex(); assert 0 < vals.length; From 5c498b411d8f89c7630fc6a5ca7237c496ed45d4 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Thu, 15 Jun 2017 13:28:17 -0400 Subject: [PATCH 085/131] SOLR-10891: Don't require docvalues on BBoxField --- .../java/org/apache/solr/schema/BBoxField.java | 3 --- .../solr/collection1/conf/schema-spatial.xml | 4 ++++ .../org/apache/solr/search/TestSolr4Spatial.java | 16 +++++++++++----- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/schema/BBoxField.java b/solr/core/src/java/org/apache/solr/schema/BBoxField.java index 957059f840e..357ccf1fc44 100644 --- a/solr/core/src/java/org/apache/solr/schema/BBoxField.java +++ b/solr/core/src/java/org/apache/solr/schema/BBoxField.java @@ -92,9 +92,6 @@ public class BBoxField extends AbstractSpatialFieldType implements if (numberType.getNumberType() != NumberType.DOUBLE) { throw new RuntimeException("Must be Double number type: " + numberType); } - if ( ! numberType.hasProperty(DOC_VALUES)) { - throw new RuntimeException("Must have doc values: " + numberType); - } //note: this only works for explicit fields, not dynamic fields List fields = new ArrayList<>(schema.getFields().values());//copy, because we modify during iteration diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml b/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml index a90fa3d00e0..6126e264477 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml @@ -56,6 +56,9 @@ + + @@ -71,6 +74,7 @@ + diff --git a/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java b/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java index 4217a00dee5..7ef8f4f7d62 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java +++ b/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java @@ -53,7 +53,7 @@ public class TestSolr4Spatial extends SolrTestCaseJ4 { @ParametersFactory public static Iterable parameters() { return Arrays.asList(new Object[][]{ - {"llp"}, {"llp_idx"}, {"llp_dv"}, {"srpt_geohash"}, {"srpt_quad"}, {"srpt_packedquad"}, {"stqpt_geohash"}, {"pointvector"}, {"bbox"}, {"pbbox"} + {"llp"}, {"llp_idx"}, {"llp_dv"}, {"srpt_geohash"}, {"srpt_quad"}, {"srpt_packedquad"}, {"stqpt_geohash"}, {"pointvector"}, {"bbox"}, {"pbbox"}, {"bbox_ndv"} }); } @@ -178,8 +178,14 @@ public class TestSolr4Spatial extends SolrTestCaseJ4 { checkHits(fieldName, true, pt, distKM, sphereRadius, count, docIds); } + private boolean isBBoxField(String fieldName) { + return fieldName.equalsIgnoreCase("bbox") + || fieldName.equalsIgnoreCase("pbbox") + || fieldName.equalsIgnoreCase("bbox_ndv"); + } + private void checkHits(String fieldName, boolean exact, String ptStr, double distKM, double sphereRadius, int count, int ... docIds) throws ParseException { - if (exact && fieldName.equalsIgnoreCase("bbox") | fieldName.equalsIgnoreCase("pbbox")) { + if (exact && isBBoxField(fieldName)) { return; // bbox field only supports rectangular query } String [] tests = new String[docIds != null && docIds.length > 0 ? docIds.length + 1 : 1]; @@ -369,9 +375,9 @@ public class TestSolr4Spatial extends SolrTestCaseJ4 { private String radiusQuery(double lat, double lon, double dDEG, String score, String filter) { //Choose between the Solr/Geofilt syntax, and the Lucene spatial module syntax - if (fieldName.equals("bbox") || fieldName.equals("pbbox") || random().nextBoolean()) { + if (isBBoxField(fieldName) || random().nextBoolean()) { //we cheat for bbox strategy which doesn't do radius, only rect. - final String qparser = (fieldName.equals("bbox") || fieldName.equals("pbbox")) ? "bbox" : "geofilt"; + final String qparser = isBBoxField(fieldName) ? "bbox" : "geofilt"; return "{!" + qparser + " " + "sfield=" + fieldName + " " + (score != null ? "score="+score : "") + " " @@ -389,7 +395,7 @@ public class TestSolr4Spatial extends SolrTestCaseJ4 { public void testSortMultiVal() throws Exception { assumeTrue("dist sorting not supported on field " + fieldName, canCalcDistance); assumeFalse("Multivalue not supported for this field", - fieldName.equals("pointvector") || fieldName.equals("bbox") || fieldName.equals("pbbox")); + fieldName.equals("pointvector") || isBBoxField(fieldName)); assertU(adoc("id", "100", fieldName, "1,2"));//1 point assertU(adoc("id", "101", fieldName, "4,-1", fieldName, "3,5"));//2 points, 2nd is pretty close to query point From 274971c3e331b68e5f7ea2669024215b8017ff7a Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Thu, 15 Jun 2017 15:53:48 -0700 Subject: [PATCH 086/131] SOLR-10832: Fixed VersionInfo.getMaxVersionFromIndex when using PointsField with indexed="true" --- solr/CHANGES.txt | 2 + .../org/apache/solr/update/UpdateLog.java | 22 ++- .../org/apache/solr/update/VersionInfo.java | 78 +++++++---- .../collection1/conf/schema-version-dv.xml | 3 +- .../conf/schema-version-indexed.xml | 3 +- .../apache/solr/update/VersionInfoTest.java | 130 ++++++++++++------ 6 files changed, 156 insertions(+), 82 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a9ab28ac062..3c276729a0e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -386,6 +386,8 @@ Bug Fixes Trie numeric fields should throw SolrException(BAD_REQUEST) for malformed docValues range queries. (hossman, Tomás Fernández Löbbe) +* SOLR-10832: Fixed VersionInfo.getMaxVersionFromIndex when using PointsField with indexed="true" (hossman) + Optimizations ---------------------- * SOLR-10634: JSON Facet API: When a field/terms facet will retrieve all buckets (i.e. limit:-1) diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index bb7f5f5a4f2..c84a9a6f4ae 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -1999,19 +1999,17 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer { // this method is primarily used for unit testing and is not part of the public API for this class Long getMaxVersionFromIndex() { - if (maxVersionFromIndex == null && versionInfo != null) { - RefCounted newestSearcher = (uhandler != null && uhandler.core != null) - ? uhandler.core.getRealtimeSearcher() : null; - if (newestSearcher == null) - throw new IllegalStateException("No searcher available to lookup max version from index!"); - - try { - maxVersionFromIndex = seedBucketsWithHighestVersion(newestSearcher.get(), versionInfo); - } finally { - newestSearcher.decref(); - } + RefCounted newestSearcher = (uhandler != null && uhandler.core != null) + ? uhandler.core.getRealtimeSearcher() : null; + if (newestSearcher == null) + throw new IllegalStateException("No searcher available to lookup max version from index!"); + + try { + seedBucketsWithHighestVersion(newestSearcher.get()); + return getCurrentMaxVersion(); + } finally { + newestSearcher.decref(); } - return maxVersionFromIndex; } /** diff --git a/solr/core/src/java/org/apache/solr/update/VersionInfo.java b/solr/core/src/java/org/apache/solr/update/VersionInfo.java index 67b40420cf9..7697be4faf5 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionInfo.java +++ b/solr/core/src/java/org/apache/solr/update/VersionInfo.java @@ -24,6 +24,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PointValues; import org.apache.lucene.index.Terms; import org.apache.solr.legacy.LegacyNumericUtils; import org.apache.lucene.queries.function.FunctionValues; @@ -228,37 +229,32 @@ public class VersionInfo { */ public Long getMaxVersionFromIndex(IndexSearcher searcher) throws IOException { - String versionFieldName = versionField.getName(); + final String versionFieldName = versionField.getName(); log.debug("Refreshing highest value of {} for {} version buckets from index", versionFieldName, buckets.length); - long maxVersionInIndex = 0L; - // if indexed, then we have terms to get the max from if (versionField.indexed()) { - LeafReader leafReader = SlowCompositeReaderWrapper.wrap(searcher.getIndexReader()); - Terms versionTerms = leafReader.terms(versionFieldName); - Long max = (versionTerms != null) ? LegacyNumericUtils.getMaxLong(versionTerms) : null; - if (max != null) { - maxVersionInIndex = max.longValue(); - log.debug("Found MAX value {} from Terms for {} in index", maxVersionInIndex, versionFieldName); + if (versionField.getType().isPointField()) { + return getMaxVersionFromIndexedPoints(searcher); } else { - log.debug("No terms found for {}, cannot seed version bucket highest value from index", versionFieldName); - } - } else { - ValueSource vs = versionField.getType().getValueSource(versionField, null); - Map funcContext = ValueSource.newContext(searcher); - vs.createWeight(funcContext, searcher); - // TODO: multi-thread this - for (LeafReaderContext ctx : searcher.getTopReaderContext().leaves()) { - int maxDoc = ctx.reader().maxDoc(); - FunctionValues fv = vs.getValues(funcContext, ctx); - for (int doc = 0; doc < maxDoc; doc++) { - long v = fv.longVal(doc); - maxVersionInIndex = Math.max(v, maxVersionInIndex); - } + return getMaxVersionFromIndexedTerms(searcher); + } + } + // else: not indexed, use docvalues via value source ... + + long maxVersionInIndex = 0L; + ValueSource vs = versionField.getType().getValueSource(versionField, null); + Map funcContext = ValueSource.newContext(searcher); + vs.createWeight(funcContext, searcher); + // TODO: multi-thread this + for (LeafReaderContext ctx : searcher.getTopReaderContext().leaves()) { + int maxDoc = ctx.reader().maxDoc(); + FunctionValues fv = vs.getValues(funcContext, ctx); + for (int doc = 0; doc < maxDoc; doc++) { + long v = fv.longVal(doc); + maxVersionInIndex = Math.max(v, maxVersionInIndex); } } - return maxVersionInIndex; } @@ -271,4 +267,38 @@ public class VersionInfo { } } } + + private long getMaxVersionFromIndexedTerms(IndexSearcher searcher) throws IOException { + assert ! versionField.getType().isPointField(); + + final String versionFieldName = versionField.getName(); + final LeafReader leafReader = SlowCompositeReaderWrapper.wrap(searcher.getIndexReader()); + final Terms versionTerms = leafReader.terms(versionFieldName); + final Long max = (versionTerms != null) ? LegacyNumericUtils.getMaxLong(versionTerms) : null; + if (null != max) { + log.debug("Found MAX value {} from Terms for {} in index", max, versionFieldName); + return max.longValue(); + } + return 0L; + } + + private long getMaxVersionFromIndexedPoints(IndexSearcher searcher) throws IOException { + assert versionField.getType().isPointField(); + + final String versionFieldName = versionField.getName(); + final byte[] maxBytes = PointValues.getMaxPackedValue(searcher.getIndexReader(), versionFieldName); + if (null == maxBytes) { + return 0L; + } + final Object maxObj = versionField.getType().toObject(versionField, new BytesRef(maxBytes)); + if (null == maxObj || ! ( maxObj instanceof Number) ) { + // HACK: aparently nothing asserts that the FieldType is numeric (let alone a Long???) + log.error("Unable to convert MAX byte[] from Points for {} in index", versionFieldName); + return 0L; + } + + final long max = ((Number)maxObj).longValue(); + log.debug("Found MAX value {} from Points for {} in index", max, versionFieldName); + return max; + } } diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml b/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml index 773136a8b45..593b2a27881 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + @@ -24,6 +24,7 @@ id + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml b/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml index c77d9ec0b04..06d6b656abe 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + @@ -24,6 +24,7 @@ id + diff --git a/solr/core/src/test/org/apache/solr/update/VersionInfoTest.java b/solr/core/src/test/org/apache/solr/update/VersionInfoTest.java index e8a85bdcebe..de68a5073da 100644 --- a/solr/core/src/test/org/apache/solr/update/VersionInfoTest.java +++ b/solr/core/src/test/org/apache/solr/update/VersionInfoTest.java @@ -21,6 +21,7 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.util.Hash; import org.apache.solr.core.CoreContainer; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.SchemaField; import org.junit.Test; public class VersionInfoTest extends SolrTestCaseJ4 { @@ -28,8 +29,13 @@ public class VersionInfoTest extends SolrTestCaseJ4 { @Test public void testMaxIndexedVersionFromIndex() throws Exception { initCore("solrconfig-tlog.xml", "schema-version-indexed.xml"); - try { - testMaxVersionLogic(req()); + try (SolrQueryRequest r = req()) { + SchemaField v = r.getCore().getUpdateHandler().getUpdateLog().getVersionInfo().getVersionField(); + assertNotNull(v); + assertTrue(v.indexed()); + assertFalse(v.hasDocValues()); + + testMaxVersionLogic(r); } finally { deleteCore(); } @@ -38,8 +44,13 @@ public class VersionInfoTest extends SolrTestCaseJ4 { @Test public void testMaxDocValuesVersionFromIndex() throws Exception { initCore("solrconfig-tlog.xml","schema-version-dv.xml"); - try { - testMaxVersionLogic(req()); + try (SolrQueryRequest r = req()) { + SchemaField v = r.getCore().getUpdateHandler().getUpdateLog().getVersionInfo().getVersionField(); + assertNotNull(v); + assertFalse(v.indexed()); + assertTrue(v.hasDocValues()); + + testMaxVersionLogic(r); } finally { deleteCore(); } @@ -55,76 +66,107 @@ public class VersionInfoTest extends SolrTestCaseJ4 { // index the first doc String docId = Integer.toString(1); + BytesRef idBytes = new BytesRef(docId); assertU(adoc("id", docId)); assertU(commit()); - // max from index should not be 0 or null - Long maxVersionFromIndex = ulog.getMaxVersionFromIndex(); - assertNotNull(maxVersionFromIndex); - assertTrue(maxVersionFromIndex != 0L); + // max from the ulog should not be 0 or null + Long maxVersionFromUlog = ulog.getMaxVersionFromIndex(); + assertNotNull(maxVersionFromUlog); + assertTrue(maxVersionFromUlog != 0L); - // version from index should be less than or equal the version of the first doc indexed VersionInfo vInfo = ulog.getVersionInfo(); - Long version = vInfo.getVersionFromIndex(new BytesRef(docId)); + try (SolrQueryRequest newReq = req()) { + // max version direct from the index should not be null, and should match what ulog reports + // (since doc is committed) + Long vInfoMax = vInfo.getMaxVersionFromIndex(newReq.getSearcher()); + assertNotNull(vInfoMax); + assertEquals(maxVersionFromUlog, vInfoMax); + } + + // max version from ulog (and index) should be exactly the same as our single committed doc + Long version = vInfo.getVersionFromIndex(idBytes); assertNotNull("version info should not be null for test doc: "+docId, version); - assertTrue("max version from index should be less than or equal to the version of first doc added, diff: "+ - (version - maxVersionFromIndex), maxVersionFromIndex <= version); + assertEquals(maxVersionFromUlog, version); - BytesRef idBytes = new BytesRef(docId); int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0); VersionBucket bucket = vInfo.bucket(bucketHash); - assertTrue(bucket.highest == version.longValue()); + assertEquals(bucket.highest, version.longValue()); - // send 2nd doc ... + // send 2nd doc ... BUT DO NOT COMMIT docId = Integer.toString(2); - assertU(adoc("id", docId)); - assertU(commit()); - - maxVersionFromIndex = ulog.getMaxVersionFromIndex(); - assertNotNull(maxVersionFromIndex); - assertTrue(maxVersionFromIndex != 0L); - - vInfo = ulog.getVersionInfo(); - version = vInfo.getVersionFromIndex(new BytesRef(docId)); - assertNotNull("version info should not be null for test doc: "+docId, version); - assertTrue("max version from index should be less than version of last doc added, diff: "+ - (version - maxVersionFromIndex), maxVersionFromIndex < version); - idBytes = new BytesRef(docId); - bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0); - bucket = vInfo.bucket(bucketHash); - assertTrue(bucket.highest == version.longValue()); + assertU(adoc("id", docId)); + + try (SolrQueryRequest newReq = req()) { + // max version direct from the index should not be null, and should still match what ulog + // previously reported (since new doc is un-committed) + Long vInfoMax = vInfo.getMaxVersionFromIndex(newReq.getSearcher()); + assertNotNull(vInfoMax); + assertEquals(maxVersionFromUlog, vInfoMax); + } + + maxVersionFromUlog = ulog.getMaxVersionFromIndex(); + assertNotNull(maxVersionFromUlog); + assertTrue("max version in ulog should have increased since our last committed doc: " + + version + " ?< " + maxVersionFromUlog, + version < maxVersionFromUlog.longValue()); + + version = vInfo.getVersionFromIndex(idBytes); + assertNull("version info should be null for uncommited test doc: "+docId, version); Long versionFromTLog = ulog.lookupVersion(idBytes); - Long versionFromIndex = vInfo.getVersionFromIndex(idBytes); + assertNotNull("version from tlog should be non-null for uncommited test doc: "+docId, versionFromTLog); + + // now commit that 2nd doc + assertU(commit()); + try (SolrQueryRequest newReq = req()) { + // max version direct from the index should match the new doc we just committed + Long vInfoMax = vInfo.getMaxVersionFromIndex(newReq.getSearcher()); + assertEquals(versionFromTLog, vInfoMax); + } + assertEquals("committing doc should not have changed version from ulog", + versionFromTLog, ulog.lookupVersion(idBytes)); + Long versionFromIndex = version = vInfo.getVersionFromIndex(idBytes); + assertNotNull("version from index should be non-null for commited test doc: "+docId, versionFromIndex); assertEquals("version from tlog and version from index should be the same", - versionFromTLog, versionFromIndex); + versionFromTLog, versionFromIndex); + + bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0); + bucket = vInfo.bucket(bucketHash); + assertEquals(bucket.highest, version.longValue()); // reload the core, which should reset the max CoreContainer coreContainer = req.getCore().getCoreContainer(); coreContainer.reload(req.getCore().getName()); - maxVersionFromIndex = ulog.getMaxVersionFromIndex(); - assertEquals("max version from index should be equal to version of last doc added after reload", - maxVersionFromIndex, version); + maxVersionFromUlog = ulog.getMaxVersionFromIndex(); + assertEquals("after reload, max version from ulog should be equal to version of last doc added", + maxVersionFromUlog, versionFromIndex); // one more doc after reload docId = Integer.toString(3); + idBytes = new BytesRef(docId); assertU(adoc("id", docId)); assertU(commit()); - maxVersionFromIndex = ulog.getMaxVersionFromIndex(); - assertNotNull(maxVersionFromIndex); - assertTrue(maxVersionFromIndex != 0L); + maxVersionFromUlog = ulog.getMaxVersionFromIndex(); + assertNotNull(maxVersionFromUlog); + assertTrue(maxVersionFromUlog != 0L); vInfo = ulog.getVersionInfo(); - version = vInfo.getVersionFromIndex(new BytesRef(docId)); + try (SolrQueryRequest newReq = req()) { + // max version direct from the index should not be null, and should match what ulog reports + // (since doc is committed) + Long vInfoMax = vInfo.getMaxVersionFromIndex(newReq.getSearcher()); + assertNotNull(vInfoMax); + assertEquals(maxVersionFromUlog, vInfoMax); + } + version = vInfo.getVersionFromIndex(idBytes); assertNotNull("version info should not be null for test doc: "+docId, version); - assertTrue("max version from index should be less than version of last doc added, diff: "+ - (version - maxVersionFromIndex), maxVersionFromIndex < version); + assertEquals(maxVersionFromUlog, version); - idBytes = new BytesRef(docId); bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0); bucket = vInfo.bucket(bucketHash); - assertTrue(bucket.highest == version.longValue()); + assertEquals(bucket.highest, version.longValue()); } } From 4eea44103eab800daddabc99eb82f450700108aa Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Tue, 13 Jun 2017 15:57:14 -0500 Subject: [PATCH 087/131] Ref Guide: Add callouts to the security.json example --- .../src/basic-authentication-plugin.adoc | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc index 057d9dfd72d..f7282160f83 100644 --- a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc @@ -36,26 +36,26 @@ An example `security.json` showing both sections is shown below to show how thes [source,json] ---- { -"authentication":{ - "blockUnknown": true, +"authentication":{ <1> + "blockUnknown": true, <2> "class":"solr.BasicAuthPlugin", - "credentials":{"solr":"IV0EHq1OnNrj6gvRCwvFwTrZ1+z1oBbnQdiVC3otuq0= Ndd7LKvVBAaZIF0QAVi1ekCfAJXr1GGfLtRUXhgrF8c="} + "credentials":{"solr":"IV0EHq1OnNrj6gvRCwvFwTrZ1+z1oBbnQdiVC3otuq0= Ndd7LKvVBAaZIF0QAVi1ekCfAJXr1GGfLtRUXhgrF8c="} <3> }, "authorization":{ "class":"solr.RuleBasedAuthorizationPlugin", "permissions":[{"name":"security-edit", - "role":"admin"}], - "user-role":{"solr":"admin"} + "role":"admin"}], <4> + "user-role":{"solr":"admin"} <5> }} ---- There are several things defined in this file: -* Basic authentication and rule-based authorization plugins are enabled. -* A user called 'solr', with a password `'SolrRocks'` has been defined. -* The parameter `"blockUnknown": true` means that unauthenticated requests are not allowed to pass through. -* The 'admin' role has been defined, and it has permission to edit security settings. -* The 'solr' user has been defined to the 'admin' role. +<1> Basic authentication and rule-based authorization plugins are enabled. +<2> A user called 'solr', with a password `'SolrRocks'` has been defined. +<3> The parameter `"blockUnknown":true` means that unauthenticated requests are not allowed to pass through. +<4> The 'admin' role has been defined, and it has permission to edit security settings. +<5> The 'solr' user has been defined to the 'admin' role. Save your settings to a file called `security.json` locally. If you are using Solr in standalone mode, you should put this file in `$SOLR_HOME`. From bf26608f6d2b8c1281975dfb25c3cd2110ea5260 Mon Sep 17 00:00:00 2001 From: Cassandra Targett Date: Thu, 15 Jun 2017 18:03:03 -0700 Subject: [PATCH 088/131] SOLR-10892: Change easy tables to description lists --- .../src/blockjoin-faceting.adoc | 19 +- .../src/collapse-and-expand-results.adoc | 74 ++++--- solr/solr-ref-guide/src/configsets-api.adoc | 30 +-- .../cross-data-center-replication-cdcr.adoc | 49 ++--- solr/solr-ref-guide/src/de-duplication.adoc | 41 ++-- .../src/defining-core-properties.adoc | 46 +++-- solr/solr-ref-guide/src/defining-fields.adoc | 18 +- .../detecting-languages-during-indexing.adoc | 95 ++++++--- .../src/distributed-requests.adoc | 39 ++-- ...field-type-definitions-and-properties.adoc | 38 ++-- .../src/hadoop-authentication-plugin.adoc | 42 ++-- .../solr-ref-guide/src/index-replication.adoc | 180 ++++++++++++------ .../src/indexconfig-in-solrconfig.adoc | 13 +- .../src/initparams-in-solrconfig.adoc | 16 +- .../src/kerberos-authentication-plugin.adoc | 58 +++--- .../src/making-and-restoring-backups.adoc | 98 +++++----- .../src/mbean-request-handler.adoc | 19 +- solr/solr-ref-guide/src/morelikethis.adoc | 72 ++++--- .../src/near-real-time-searching.adoc | 34 ++-- .../src/parameter-reference.adoc | 45 ++--- solr/solr-ref-guide/src/query-screen.adoc | 64 +++++-- .../src/requestdispatcher-in-solrconfig.adoc | 15 +- ...rs-and-searchcomponents-in-solrconfig.adoc | 14 +- solr/solr-ref-guide/src/response-writers.adoc | 93 +++++---- .../solr-ref-guide/src/result-clustering.adoc | 81 ++++---- solr/solr-ref-guide/src/result-grouping.adoc | 77 +++++--- .../src/rule-based-authorization-plugin.adoc | 38 ++-- .../src/running-solr-on-hdfs.adoc | 91 ++++----- solr/solr-ref-guide/src/spatial-search.adoc | 142 ++++++++------ .../src/the-query-elevation-component.adoc | 17 +- .../src/the-stats-component.adoc | 69 ++++--- .../src/the-term-vector-component.adoc | 55 +++--- .../src/the-terms-component.adoc | 115 +++++------ .../src/update-request-processors.adoc | 6 +- .../src/updatehandlers-in-solrconfig.adoc | 56 +++--- .../src/updating-parts-of-documents.adoc | 54 ++---- .../uploading-data-with-index-handlers.adoc | 126 ++++++++---- ...data-with-solr-cell-using-apache-tika.adoc | 110 +++++++---- solr/solr-ref-guide/src/v2-api.adoc | 24 +-- .../src/velocity-response-writer.adoc | 101 +++++----- 40 files changed, 1332 insertions(+), 1042 deletions(-) diff --git a/solr/solr-ref-guide/src/blockjoin-faceting.adoc b/solr/solr-ref-guide/src/blockjoin-faceting.adoc index bf33aca1961..1a89a570071 100644 --- a/solr/solr-ref-guide/src/blockjoin-faceting.adoc +++ b/solr/solr-ref-guide/src/blockjoin-faceting.adoc @@ -102,14 +102,15 @@ Queries are constructed the same way as for a <>. ++ +At most only one of the `min`, `max`, or `sort` (see below) parameters may be specified. ++ +If none are specified, the group head document of each group will be selected based on the highest scoring document in that group. The default is none. -At most only one of the min, max, or sort (see below) parameters may be specified. - -If none are specified, the group head document of each group will be selected based on the highest scoring document in that group. |none -|sort a| +sort:: Selects the group head document for each group based on which document comes first according to the specified <>. ++ +At most only one of the `min`, `max`, (see above) or `sort` parameters may be specified. ++ +If none are specified, the group head document of each group will be selected based on the highest scoring document in that group. The default is none. -At most only one of the min, max, (see above) or sort parameters may be specified. +nullPolicy:: +There are three available null policies: ++ +* `ignore`: removes documents with a null value in the collapse field. This is the default. +* `expand`: treats each document with a null value in the collapse field as a separate group. +* `collapse`: collapses all documents with a null value into a single group using either highest score, or minimum/maximum. ++ +The default is `ignore`. -If none are specified, the group head document of each group will be selected based on the highest scoring document in that group. |none -|nullPolicy a| -There are three null policies: +hint:: +Currently there is only one hint available: `top_fc`, which stands for top level FieldCache. ++ +The `top_fc` hint is only available when collapsing on String fields. `top_fc` usually provides the best query time speed but takes the longest to warm on startup or following a commit. `top_fc` will also result in having the collapsed field cached in memory twice if it's used for faceting or sorting. For very high cardinality (high distinct count) fields, `top_fc` may not fare so well. ++ +The default is none. -* *ignore*: removes documents with a null value in the collapse field. This is the default. -* *expand*: treats each document with a null value in the collapse field as a separate group. -* *collapse*: collapses all documents with a null value into a single group using either highest score, or minimum/maximum. +size:: +Sets the initial size of the collapse data structures when collapsing on a *numeric field only*. ++ +The data structures used for collapsing grow dynamically when collapsing on numeric fields. Setting the size above the number of results expected in the result set will eliminate the resizing cost. ++ +The default is 100,000. - |ignore -|hint |Currently there is only one hint available: `top_fc`, which stands for top level FieldCache. The `top_fc` hint is only available when collapsing on String fields. `top_fc` usually provides the best query time speed but takes the longest to warm on startup or following a commit. `top_fc` will also result in having the collapsed field cached in memory twice if it's used for faceting or sorting. For very high cardinality (high distinct count) fields, `top_fc` may not fare so well. |none -|size |Sets the initial size of the collapse data structures when collapsing on a *numeric field only*. The data structures used for collapsing grow dynamically when collapsing on numeric fields. Setting the size above the number of results expected in the result set will eliminate the resizing cost. |100,000 -|=== -*Sample Syntax:* +=== Sample Syntax Collapse on `group_field` selecting the document in each group with the highest scoring document: @@ -137,13 +148,14 @@ Inside the expanded section there is a _map_ with each group head pointing to th The ExpandComponent has the following parameters: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +expand.sort:: +Orders the documents within the expanded groups. The default is `score desc`. -[cols="20,60,20",options="header"] -|=== -|Parameter |Description |Default -|expand.sort |Orders the documents within the expanded groups |score desc -|expand.rows |The number of rows to display in each group |5 -|expand.q |Overrides the main q parameter, determines which documents to include in the main group. |main q -|expand.fq |Overrides main fq's, determines which documents to include in the main group. |main fq's -|=== +expand.rows:: +The number of rows to display in each group. The default is 5 rows. + +expand.q:: +Overrides the main query (`q`), determines which documents to include in the main group. The default is to use the main query. + +expand.fq:: +Overrides main filter queries (`fq`), determines which documents to include in the main group. The default is to use the main filter queries. diff --git a/solr/solr-ref-guide/src/configsets-api.adoc b/solr/solr-ref-guide/src/configsets-api.adoc index 2bd50ffc4ba..603e08e097e 100644 --- a/solr/solr-ref-guide/src/configsets-api.adoc +++ b/solr/solr-ref-guide/src/configsets-api.adoc @@ -46,15 +46,13 @@ Create a ConfigSet, based on an existing ConfigSet. [[ConfigSetsAPI-Input]] === Input -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +The following parameters are supported when creating a ConfigSet. -[cols="25,10,10,10,45",options="header"] -|=== -|Key |Type |Required |Default |Description -|name |String |Yes | |ConfigSet to be created -|baseConfigSet |String |Yes | |ConfigSet to copy as a base -|configSetProp._name=value_ |String |No | |ConfigSet property from base to override -|=== +name:: The ConfigSet to be created. This parameter is required. + +baseConfigSet:: The ConfigSet to copy as a base. This parameter is required. + +configSetProp._name_=_value_:: Any ConfigSet property from base to override. [[ConfigSetsAPI-Output]] === Output @@ -101,13 +99,7 @@ Delete a ConfigSet *Query Parameters* -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed - -[cols="20,15,10,15,40",options="header"] -|=== -|Key |Type |Required |Default |Description -|name |String |Yes | |ConfigSet to be deleted -|=== +name:: The ConfigSet to be deleted. This parameter is required. [[ConfigSetsAPI-Output.1]] === Output @@ -184,13 +176,7 @@ Upload a ConfigSet, sent in as a zipped file. Please note that a ConfigSet is up [[ConfigSetsAPI-Input.3]] === Input -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed - -[cols="20,15,10,15,40",options="header"] -|=== -|Key |Type |Required |Default |Description -|name |String |Yes | |ConfigSet to be created -|=== +name:: The ConfigSet to be created when the upload is complete. This parameter is required. The body of the request should contain a zipped config set. diff --git a/solr/solr-ref-guide/src/cross-data-center-replication-cdcr.adoc b/solr/solr-ref-guide/src/cross-data-center-replication-cdcr.adoc index 3bd482b623f..67729557a5a 100644 --- a/solr/solr-ref-guide/src/cross-data-center-replication-cdcr.adoc +++ b/solr/solr-ref-guide/src/cross-data-center-replication-cdcr.adoc @@ -252,15 +252,15 @@ The configuration details, defaults and options are as follows: CDCR can be configured to forward update requests to one or more replicas. A replica is defined with a “replica” list as follows: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed -[cols="20,10,15,55",options="header"] -|=== -|Parameter |Required |Default |Description -|zkHost |Yes |none |The host address for ZooKeeper of the target SolrCloud. Usually this is a comma-separated list of addresses to each node in the target ZooKeeper ensemble. -|Source |Yes |none |The name of the collection on the Source SolrCloud to be replicated. -|Target |Yes |none |The name of the collection on the target SolrCloud to which updates will be forwarded. -|=== +`zkHost`:: +The host address for ZooKeeper of the target SolrCloud. Usually this is a comma-separated list of addresses to each node in the target ZooKeeper ensemble. This parameter is required. + +`Source`:: +The name of the collection on the Source SolrCloud to be replicated. This parameter is required. + +`Target`:: +The name of the collection on the target SolrCloud to which updates will be forwarded. This parameter is required. ==== The Replicator Element @@ -268,39 +268,28 @@ The CDC Replicator is the component in charge of forwarding updates to the repli The replicator uses a fixed thread pool to forward updates to multiple replicas in parallel. If more than one replica is configured, one thread will forward a batch of updates from one replica at a time in a round-robin fashion. The replicator can be configured with a “replicator” list as follows: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`threadPoolSize`:: +The number of threads to use for forwarding updates. One thread per replica is recommended. The default is `2`. -[cols="20,10,15,55",options="header"] -|=== -|Parameter |Required |Default |Description -|threadPoolSize |No |2 |The number of threads to use for forwarding updates. One thread per replica is recommended. -|schedule |No |10 |The delay in milliseconds for the monitoring the update log(s). -|batchSize |No |128 |The number of updates to send in one batch. The optimal size depends on the size of the documents. Large batches of large documents can increase your memory usage significantly. -|=== +`schedule`:: +The delay in milliseconds for the monitoring the update log(s). The default is `10`. + +`batchSize`:: +The number of updates to send in one batch. The optimal size depends on the size of the documents. Large batches of large documents can increase your memory usage significantly. The default is `128`. ==== The updateLogSynchronizer Element Expert: Non-leader nodes need to synchronize their update logs with their leader node from time to time in order to clean deprecated transaction log files. By default, such a synchronization process is performed every minute. The schedule of the synchronization can be modified with a “updateLogSynchronizer” list as follows: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed - -[cols="20,10,15,55",options="header"] -|=== -|Parameter |Required |Default |Description -|schedule |No |60000 |The delay in milliseconds for synchronizing the updates log. -|=== +`schedule`:: + The delay in milliseconds for synchronizing the updates log. The default is `60000`. ==== The Buffer Element CDCR is configured by default to buffer any new incoming updates. When buffering updates, the updates log will store all the updates indefinitely. Replicas do not need to buffer updates, and it is recommended to disable buffer on the target SolrCloud. The buffer can be disabled at startup with a “buffer” list and the parameter “defaultState” as follows: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed - -[cols="20,10,15,55",options="header"] -|=== -|Parameter |Required |Default |Description -|defaultState |No |enabled |The state of the buffer at startup. -|=== +`defaultState`:: +The state of the buffer at startup. The default is `enabled`. == CDCR API diff --git a/solr/solr-ref-guide/src/de-duplication.adoc b/solr/solr-ref-guide/src/de-duplication.adoc index 8f4d01a2c9d..3e9cd46a141 100644 --- a/solr/solr-ref-guide/src/de-duplication.adoc +++ b/solr/solr-ref-guide/src/de-duplication.adoc @@ -20,17 +20,12 @@ If duplicate, or near-duplicate documents are a concern in your index, de-duplication may be worth implementing. -Preventing duplicate or near duplicate documents from entering an index or tagging documents with a signature/fingerprint for duplicate field collapsing can be efficiently achieved with a low collision or fuzzy hash algorithm. Solr natively supports de-duplication techniques of this type via the `Signature` class and allows for the easy addition of new hash/signature implementations. A Signature can be implemented several ways: +Preventing duplicate or near duplicate documents from entering an index or tagging documents with a signature/fingerprint for duplicate field collapsing can be efficiently achieved with a low collision or fuzzy hash algorithm. Solr natively supports de-duplication techniques of this type via the `Signature` class and allows for the easy addition of new hash/signature implementations. A Signature can be implemented in a few ways: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +* MD5Signature: 128-bit hash used for exact duplicate detection. +* Lookup3Signature: 64-bit hash used for exact duplicate detection. This is much faster than MD5 and smaller to index. +* http://wiki.apache.org/solr/TextProfileSignature[TextProfileSignature]: Fuzzy hashing implementation from Apache Nutch for near duplicate detection. It's tunable but works best on longer text. -[cols="30,70",options="header"] -|=== -|Method |Description -|MD5Signature |128-bit hash used for exact duplicate detection. -|Lookup3Signature |64-bit hash used for exact duplicate detection. This is much faster than MD5 and smaller to index. -|http://wiki.apache.org/solr/TextProfileSignature[TextProfileSignature] |Fuzzy hashing implementation from Apache Nutch for near duplicate detection. It's tunable but works best on longer text. -|=== Other, more sophisticated algorithms for fuzzy/near hashing can be added later. @@ -68,23 +63,27 @@ The `SignatureUpdateProcessorFactory` has to be registered in `solrconfig.xml` a The `SignatureUpdateProcessorFactory` takes several properties: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed - -[cols="20,30,50",options="header"] -|=== -|Parameter |Default |Description -|signatureClass |`org.apache.solr.update.processor.Lookup3Signature` a| -A Signature implementation for generating a signature hash. The full classpath of the implementation must be specified. The available options are described above, the associated classpaths to use are: +signatureClass:: +A Signature implementation for generating a signature hash. The default is `org.apache.solr.update.processor.Lookup3Signature`. ++ +The full classpath of the implementation must be specified. The available options are described above, the associated classpaths to use are: * `org.apache.solr.update.processor.Lookup3Signature` * `org.apache.solr.update.processor.MD5Signature` * `org.apache.solr.update.process.TextProfileSignature` -|fields |all fields |The fields to use to generate the signature hash in a comma separated list. By default, all fields on the document will be used. -|signatureField |signatureField |The name of the field used to hold the fingerprint/signature. The field should be defined in schema.xml. -|enabled |true |Enable/disable de-duplication processing. -|overwriteDupes |true |If true, when a document exists that already matches this signature, it will be overwritten. -|=== +fields:: +The fields to use to generate the signature hash in a comma separated list. By default, all fields on the document will be used. + +signatureField:: +The name of the field used to hold the fingerprint/signature. The field should be defined in `schema.xml`. The default is `signatureField`. + +enabled:: +Set to *false* to disable de-duplication processing. The default is *true*. + +overwriteDupes:: +If true, the default, when a document exists that already matches this signature, it will be overwritten. + [[De-Duplication-Inschema.xml]] === In schema.xml diff --git a/solr/solr-ref-guide/src/defining-core-properties.adoc b/solr/solr-ref-guide/src/defining-core-properties.adoc index 93a3d3ea3c7..a533098609e 100644 --- a/solr/solr-ref-guide/src/defining-core-properties.adoc +++ b/solr/solr-ref-guide/src/defining-core-properties.adoc @@ -70,26 +70,32 @@ The minimal `core.properties` file is an empty file, in which case all of the pr Java properties files allow the hash (`#`) or bang (`!`) characters to specify comment-to-end-of-line. -This table defines the recognized properties: +The following properties are available: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`name`:: The name of the SolrCore. You'll use this name to reference the SolrCore when running commands with the CoreAdminHandler. -[cols="25,75",options="header"] -|=== -|Property |Description -|`name` |The name of the SolrCore. You'll use this name to reference the SolrCore when running commands with the CoreAdminHandler. -|`config` |The configuration file name for a given core. The default is `solrconfig.xml`. -|`schema` |The schema file name for a given core. The default is `schema.xml` but please note that if you are using a "managed schema" (the default behavior) then any value for this property which does not match the effective `managedSchemaResourceName` will be read once, backed up, and converted for managed schema use. See <> for details. -|`dataDir` |The core's data directory (where indexes are stored) as either an absolute pathname, or a path relative to the value of `instanceDir`. This is `data` by default. -|`configSet` |The name of a defined configset, if desired, to use to configure the core (see the <> for more details). -|`properties` |The name of the properties file for this core. The value can be an absolute pathname or a path relative to the value of `instanceDir`. -|`transient` |If *true*, the core can be unloaded if Solr reaches the `transientCacheSize`. The default if not specified is *false*. Cores are unloaded in order of least recently used first. _Setting to *true* is not recommended in SolrCloud mode._ -|`loadOnStartup` |If *true*, the default if it is not specified, the core will loaded when Solr starts. _Setting to *false* is not recommended in SolrCloud mode._ -|`coreNodeName` |Used only in SolrCloud, this is a unique identifier for the node hosting this replica. By default a coreNodeName is generated automatically, but setting this attribute explicitly allows you to manually assign a new core to replace an existing replica. For example: when replacing a machine that has had a hardware failure by restoring from backups on a new machine with a new hostname or port.. -|`ulogDir` |The absolute or relative directory for the update log for this core (SolrCloud). -|`shard` |The shard to assign this core to (SolrCloud). -|`collection` |The name of the collection this core is part of (SolrCloud). -|`roles` |Future param for SolrCloud or a way for users to mark nodes for their own use. -|=== +`config`:: The configuration file name for a given core. The default is `solrconfig.xml`. -Additional "user defined" properties may be specified for use as variables. For more information on how to define local properties, see the section <>. +`schema`:: The schema file name for a given core. The default is `schema.xml` but please note that if you are using a "managed schema" (the default behavior) then any value for this property which does not match the effective `managedSchemaResourceName` will be read once, backed up, and converted for managed schema use. See <> for more details. + +`dataDir`:: The core's data directory (where indexes are stored) as either an absolute pathname, or a path relative to the value of `instanceDir`. This is `data` by default. + +`configSet`:: The name of a defined configset, if desired, to use to configure the core (see the section <> for more details). + +`properties`:: The name of the properties file for this core. The value can be an absolute pathname or a path relative to the value of `instanceDir`. + +`transient`:: If *true*, the core can be unloaded if Solr reaches the `transientCacheSize`. The default if not specified is *false*. Cores are unloaded in order of least recently used first. _Setting this to *true* is not recommended in SolrCloud mode._ + +`loadOnStartup`:: If *true*, the default if it is not specified, the core will loaded when Solr starts. _Setting this to *false* is not recommended in SolrCloud mode._ + +`coreNodeName`:: Used only in SolrCloud, this is a unique identifier for the node hosting this replica. By default a `coreNodeName` is generated automatically, but setting this attribute explicitly allows you to manually assign a new core to replace an existing replica. For example, this can be useful when replacing a machine that has had a hardware failure by restoring from backups on a new machine with a new hostname or port. + +`ulogDir`:: The absolute or relative directory for the update log for this core (SolrCloud). + +`shard`:: The shard to assign this core to (SolrCloud). + +`collection`:: The name of the collection this core is part of (SolrCloud). + +`roles`:: Future parameter for SolrCloud or a way for users to mark nodes for their own use. + +Additional user-defined properties may be specified for use as variables. For more information on how to define local properties, see the section <>. diff --git a/solr/solr-ref-guide/src/defining-fields.adoc b/solr/solr-ref-guide/src/defining-fields.adoc index 4ef3a5d9e39..8e6de9c4269 100644 --- a/solr/solr-ref-guide/src/defining-fields.adoc +++ b/solr/solr-ref-guide/src/defining-fields.adoc @@ -33,15 +33,16 @@ The following example defines a field named `price` with a type named `float` an [[DefiningFields-FieldProperties]] == Field Properties -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +Field definitions can have the following properties: -[cols="30,70",options="header"] -|=== -|Property |Description -|name |The name of the field. Field names should consist of alphanumeric or underscore characters only and not start with a digit. This is not currently strictly enforced, but other field names will not have first class support from all components and back compatibility is not guaranteed. Names with both leading and trailing underscores (e.g., `\_version_`) are reserved. Every field must have a `name`. -|type |The name of the `fieldType` for this field. This will be found in the `name` attribute on the `fieldType` definition. Every field must have a `type`. -|default |A default value that will be added automatically to any document that does not have a value in this field when it is indexed. If this property is not specified, there is no default. -|=== +`name`:: +The name of the field. Field names should consist of alphanumeric or underscore characters only and not start with a digit. This is not currently strictly enforced, but other field names will not have first class support from all components and back compatibility is not guaranteed. Names with both leading and trailing underscores (e.g., `\_version_`) are reserved. Every field must have a `name`. + +`type`:: +The name of the `fieldType` for this field. This will be found in the `name` attribute on the `fieldType` definition. Every field must have a `type`. + +`default`:: +A default value that will be added automatically to any document that does not have a value in this field when it is indexed. If this property is not specified, there is no default. [[DefiningFields-OptionalFieldTypeOverrideProperties]] == Optional Field Type Override Properties @@ -70,4 +71,3 @@ Fields can have many of the same properties as field types. Properties from the |=== // TODO: SOLR-10655 END - diff --git a/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc b/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc index b73fdf77777..4003f1ac914 100644 --- a/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc +++ b/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc @@ -71,28 +71,75 @@ Here is an example of a minimal LangDetect `langid` configuration in `solrconfig As previously mentioned, both implementations of the `langid` UpdateRequestProcessor take the same parameters. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`langid`:: +When `true`, the default, enables language detection. -[cols="30,10,10,10,40",options="header"] -|=== -|Parameter |Type |Default |Required |Description -|langid |Boolean |true |no |Enables and disables language detection. -|langid.fl |string |none |yes |A comma- or space-delimited list of fields to be processed by `langid`. -|langid.langField |string |none |yes |Specifies the field for the returned language code. -|langid.langsField |multivalued string |none |no |Specifies the field for a list of returned language codes. If you use `langid.map.individual`, each detected language will be added to this field. -|langid.overwrite |Boolean |false |no |Specifies whether the content of the `langField` and `langsField` fields will be overwritten if they already contain values. -|langid.lcmap |string |none |false |A space-separated list specifying colon delimited language code mappings to apply to the detected languages. For example, you might use this to map Chinese, Japanese, and Korean to a common `cjk` code, and map both American and British English to a single `en` code by using `langid.lcmap=ja:cjk zh:cjk ko:cjk en_GB:en en_US:en`. This affects both the values put into the `langField` and `langsField` fields, as well as the field suffixes when using `langid.map`, unless overridden by `langid.map.lcmap` -|langid.threshold |float |0.5 |no |Specifies a threshold value between 0 and 1 that the language identification score must reach before `langid` accepts it. With longer text fields, a high threshold such at 0.8 will give good results. For shorter text fields, you may need to lower the threshold for language identification, though you will be risking somewhat lower quality results. We recommend experimenting with your data to tune your results. -|langid.whitelist |string |none |no |Specifies a list of allowed language identification codes. Use this in combination with `langid.map` to ensure that you only index documents into fields that are in your schema. -|langid.map |Boolean |false |no |Enables field name mapping. If true, Solr will map field names for all fields listed in `langid.fl`. -|langid.map.fl |string |none |no |A comma-separated list of fields for `langid.map` that is different than the fields specified in `langid.fl`. -|langid.map.keepOrig |Boolean |false |no |If true, Solr will copy the field during the field name mapping process, leaving the original field in place. -|langid.map.individual |Boolean |false |no |If true, Solr will detect and map languages for each field individually. -|langid.map.individual.fl |string |none |no |A comma-separated list of fields for use with `langid.map.individual` that is different than the fields specified in `langid.fl`. -|langid.fallbackFields |string |none |no |If no language is detected that meets the `langid.threshold` score, or if the detected language is not on the `langid.whitelist`, this field specifies language codes to be used as fallback values. If no appropriate fallback languages are found, Solr will use the language code specified in `langid.fallback`. -|langid.fallback |string |none |no |Specifies a language code to use if no language is detected or specified in `langid.fallbackFields`. -|langid.map.lcmap |string |determined by `langid.lcmap` |no |A space-separated list specifying colon delimited language code mappings to use when mapping field names. For example, you might use this to make Chinese, Japanese, and Korean language fields use a common `*_cjk` suffix, and map both American and British English fields to a single `*_en` by using `langid.map.lcmap=ja:cjk zh:cjk ko:cjk en_GB:en en_US:en`. -|langid.map.pattern |Java regular expression |none |no |By default, fields are mapped as _. To change this pattern, you can specify a Java regular expression in this parameter. -|langid.map.replace |Java replace |none |no |By default, fields are mapped as _. To change this pattern, you can specify a Java replace in this parameter. -|langid.enforceSchema |Boolean |true |no |If false, the `langid` processor does not validate field names against your schema. This may be useful if you plan to rename or delete fields later in the UpdateChain. -|=== +`langid.fl`:: +A comma- or space-delimited list of fields to be processed by `langid`. This parameter is required. + +`langid.langField`:: +Specifies the field for the returned language code. This parameter is required. + +`langid.langsField`:: +Specifies the field for a list of returned language codes. If you use `langid.map.individual`, each detected language will be added to this field. + +`langid.overwrite`:: +Specifies whether the content of the `langField` and `langsField` fields will be overwritten if they already contain values. The default is `false`. + +`langid.lcmap`:: +A space-separated list specifying colon delimited language code mappings to apply to the detected languages. ++ +For example, you might use this to map Chinese, Japanese, and Korean to a common `cjk` code, and map both American and British English to a single `en` code by using `langid.lcmap=ja:cjk zh:cjk ko:cjk en_GB:en en_US:en`. ++ +This affects both the values put into the `langField` and `langsField` fields, as well as the field suffixes when using `langid.map`, unless overridden by `langid.map.lcmap`. + +`langid.threshold`:: +Specifies a threshold value between 0 and 1 that the language identification score must reach before `langid` accepts it. ++ +With longer text fields, a high threshold such as `0.8` will give good results. For shorter text fields, you may need to lower the threshold for language identification, though you will be risking somewhat lower quality results. We recommend experimenting with your data to tune your results. ++ +The default is `0.5`. + +`langid.whitelist`:: +Specifies a list of allowed language identification codes. Use this in combination with `langid.map` to ensure that you only index documents into fields that are in your schema. + +`langid.map`:: +Enables field name mapping. If `true`, Solr will map field names for all fields listed in `langid.fl`. The default is `false`. + +`langid.map.fl`:: +A comma-separated list of fields for `langid.map` that is different than the fields specified in `langid.fl`. + +`langid.map.keepOrig`:: +If `true`, Solr will copy the field during the field name mapping process, leaving the original field in place. The default is `false`. + +`langid.map.individual`:: +If `true`, Solr will detect and map languages for each field individually. The default is `false`. + +`langid.map.individual.fl`:: +A comma-separated list of fields for use with `langid.map.individual` that is different than the fields specified in `langid.fl`. + +`langid.fallback`:: +Specifies a language code to use if no language is detected or specified in `langid.fallbackFields`. + +`langid.fallbackFields`:: +If no language is detected that meets the `langid.threshold` score, or if the detected language is not on the `langid.whitelist`, this field specifies language codes to be used as fallback values. ++ +If no appropriate fallback languages are found, Solr will use the language code specified in `langid.fallback`. + +`langid.map.lcmap`:: +A space-separated list specifying colon-delimited language code mappings to use when mapping field names. ++ +For example, you might use this to make Chinese, Japanese, and Korean language fields use a common `*_cjk` suffix, and map both American and British English fields to a single `*_en` by using `langid.map.lcmap=ja:cjk zh:cjk ko:cjk en_GB:en en_US:en`. ++ +A list defined with this parameter will override any configuration set with `langid.lcmap`. + +`langid.map.pattern`:: +By default, fields are mapped as _. To change this pattern, you can specify a Java regular expression in this parameter. + +`langid.map.replace`:: +By default, fields are mapped as `_`. To change this pattern, you can specify a Java replace in this parameter. + +`langid.enforceSchema`:: +If `false`, the `langid` processor does not validate field names against your schema. This may be useful if you plan to rename or delete fields later in the UpdateChain. ++ +The default is `true`. diff --git a/solr/solr-ref-guide/src/distributed-requests.adoc b/solr/solr-ref-guide/src/distributed-requests.adoc index 75f023c1c0c..b89878fa0c5 100644 --- a/solr/solr-ref-guide/src/distributed-requests.adoc +++ b/solr/solr-ref-guide/src/distributed-requests.adoc @@ -91,21 +91,32 @@ To configure the standard handler, provide a configuration like this in `solrcon The parameters that can be specified are as follows: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`socketTimeout`:: +The amount of time in ms that a socket is allowed to wait. The default is `0`, where the operating system's default will be used. -[cols="20,15,65",options="header"] -|=== -|Parameter |Default |Explanation -|`socketTimeout` |0 (use OS default) |The amount of time in ms that a socket is allowed to wait. -|`connTimeout` |0 (use OS default) |The amount of time in ms that is accepted for binding / connecting a socket -|`maxConnectionsPerHost` |20 |The maximum number of concurrent connections that is made to each individual shard in a distributed search. -|`maxConnections` |`10000` |The total maximum number of concurrent connections in distributed searches. -|`corePoolSize` |0 |The retained lowest limit on the number of threads used in coordinating distributed search. -|`maximumPoolSize` |Integer.MAX_VALUE |The maximum number of threads used for coordinating distributed search. -|`maxThreadIdleTime` |5 seconds |The amount of time to wait for before threads are scaled back in response to a reduction in load. -|`sizeOfQueue` |-1 |If specified, the thread pool will use a backing queue instead of a direct handoff buffer. High throughput systems will want to configure this to be a direct hand off (with -1). Systems that desire better latency will want to configure a reasonable size of queue to handle variations in requests. -|`fairnessPolicy` |false |Chooses the JVM specifics dealing with fair policy queuing, if enabled distributed searches will be handled in a First in First out fashion at a cost to throughput. If disabled throughput will be favored over latency. -|=== +`connTimeout`:: +The amount of time in ms that is accepted for binding / connecting a socket. The default is `0`, where the operating system's default will be used. + +`maxConnectionsPerHost`:: +The maximum number of concurrent connections that is made to each individual shard in a distributed search. The default is `20`. + +`maxConnections`:: +The total maximum number of concurrent connections in distributed searches. The default is `10000` + +`corePoolSize`:: +The retained lowest limit on the number of threads used in coordinating distributed search. The default is `0`. + +`maximumPoolSize`:: +The maximum number of threads used for coordinating distributed search. The default is `Integer.MAX_VALUE`. + +`maxThreadIdleTime`:: +The amount of time in seconds to wait for before threads are scaled back in response to a reduction in load. The default is `5`. + +`sizeOfQueue`:: +If specified, the thread pool will use a backing queue instead of a direct handoff buffer. High throughput systems will want to configure this to be a direct hand off (with `-1`). Systems that desire better latency will want to configure a reasonable size of queue to handle variations in requests. The default is `-1`. + +`fairnessPolicy`:: +Chooses the JVM specifics dealing with fair policy queuing, if enabled distributed searches will be handled in a First in First out fashion at a cost to throughput. If disabled throughput will be favored over latency. The default is `false`. [[DistributedRequests-ConfiguringstatsCache_DistributedIDF_]] == Configuring statsCache (Distributed IDF) diff --git a/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc b/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc index 12d291306cf..89b8e9062ad 100644 --- a/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc +++ b/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc @@ -75,19 +75,33 @@ The properties that can be specified for a given field type fall into three majo === General Properties -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +These are the general properties for fields + +`name`:: +The name of the fieldType. This value gets used in field definitions, in the "type" attribute. It is strongly recommended that names consist of alphanumeric or underscore characters only and not start with a digit. This is not currently strictly enforced. + +`class`:: +The class name that gets used to store and index the data for this type. Note that you may prefix included class names with "solr." and Solr will automatically figure out which packages to search for the class - so `solr.TextField` will work. ++ +If you are using a third-party class, you will probably need to have a fully qualified class name. The fully qualified equivalent for `solr.TextField` is `org.apache.solr.schema.TextField`. + +`positionIncrementGap`:: +For multivalued fields, specifies a distance between multiple values, which prevents spurious phrase matches. + +`autoGeneratePhraseQueries`:: For text fields. If `true`, Solr automatically generates phrase queries for adjacent terms. If `false`, terms must be enclosed in double-quotes to be treated as phrases. + +`enableGraphQueries`:: +For text fields, applicable when querying with <>. Use `true` (the default) for field types with query analyzers including graph-aware filters, e.g., <> and <>. ++ +Use `false` for field types with query analyzers including filters that can match docs when some tokens are missing, e.g., <>. + +[[FieldTypeDefinitionsandProperties-docValuesFormat]] +`docValuesFormat`:: +Defines a custom `DocValuesFormat` to use for fields of this type. This requires that a schema-aware codec, such as the `SchemaCodecFactory` has been configured in solrconfig.xml. + +`postingsFormat`:: +Defines a custom `PostingsFormat` to use for fields of this type. This requires that a schema-aware codec, such as the `SchemaCodecFactory` has been configured in solrconfig.xml. -[cols="30,40,30",options="header"] -|=== -|Property |Description |Values -|name |The name of the fieldType. This value gets used in field definitions, in the "type" attribute. It is strongly recommended that names consist of alphanumeric or underscore characters only and not start with a digit. This is not currently strictly enforced. | -|class |The class name that gets used to store and index the data for this type. Note that you may prefix included class names with "solr." and Solr will automatically figure out which packages to search for the class - so `solr.TextField` will work. If you are using a third-party class, you will probably need to have a fully qualified class name. The fully qualified equivalent for `solr.TextField` is `org.apache.solr.schema.TextField`. | -|positionIncrementGap |For multivalued fields, specifies a distance between multiple values, which prevents spurious phrase matches |integer -|autoGeneratePhraseQueries |For text fields. If true, Solr automatically generates phrase queries for adjacent terms. If false, terms must be enclosed in double-quotes to be treated as phrases. |true or false -|enableGraphQueries |For text fields, applicable when querying with <>. Use `true` (the default) for field types with query analyzers including graph-aware filters, e.g. <> and <>. Use `false` for field types with query analyzers including filters that can match docs when some tokens are missing, e.g., <>. |true or false -|[[FieldTypeDefinitionsandProperties-docValuesFormat]]docValuesFormat |Defines a custom `DocValuesFormat` to use for fields of this type. This requires that a schema-aware codec, such as the `SchemaCodecFactory` has been configured in solrconfig.xml. |n/a -|postingsFormat |Defines a custom `PostingsFormat` to use for fields of this type. This requires that a schema-aware codec, such as the `SchemaCodecFactory` has been configured in solrconfig.xml. |n/a -|=== [NOTE] ==== diff --git a/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc b/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc index 2ac541a57af..1c17fbca029 100644 --- a/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc @@ -41,21 +41,35 @@ For most SolrCloud or standalone Solr setups, the `HadoopAuthPlugin` should suff [[HadoopAuthenticationPlugin-PluginConfiguration]] == Plugin Configuration -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`class`:: +Should be either `solr.HadoopAuthPlugin` or `solr.ConfigurableInternodeAuthHadoopPlugin`. This parameter is required. + +`type`:: +The type of authentication scheme to be configured. See https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[configuration] options. This parameter is required. + +`sysPropPrefix`:: +The prefix to be used to define the Java system property for configuring the authentication mechanism. This property is required. ++ +The name of the Java system property is defined by appending the configuration parameter name to this prefix value. For example, if the prefix is `solr` then the Java system property `solr.kerberos.principal` defines the value of configuration parameter `kerberos.principal`. + +`authConfigs`:: +Configuration parameters required by the authentication scheme defined by the type property. This property is required. For more details, see https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[Hadoop configuration] options. + +`defaultConfigs`:: +Default values for the configuration parameters specified by the `authConfigs` property. The default values are specified as a collection of key-value pairs (i.e., `"property-name": "default_value"`). + +`enableDelegationToken`:: +If `true`, the delegation tokens functionality will be enabled. + +`initKerberosZk`:: +For enabling initialization of kerberos before connecting to ZooKeeper (if applicable). + +`proxyUserConfigs`:: +Configures proxy users for the underlying Hadoop authentication mechanism. This configuration is expressed as a collection of key-value pairs (i.e., `"property-name": "default_value"`). + +`clientBuilderFactory`:: No | +The `HttpClientBuilderFactory` implementation used for the Solr internal communication. Only applicable for `ConfigurableInternodeAuthHadoopPlugin`. -[cols="20,15,65",options="header"] -|=== -|Parameter Name |Required |Description -|class |Yes |Should be either `solr.HadoopAuthPlugin` or `solr.ConfigurableInternodeAuthHadoopPlugin`. -|type |Yes |The type of authentication scheme to be configured. See https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[configuration] options. -|sysPropPrefix |Yes |The prefix to be used to define the Java system property for configuring the authentication mechanism. The name of the Java system property is defined by appending the configuration parameter name to this prefix value. For example, if the prefix is 'solr' then the Java system property 'solr.kerberos.principal' defines the value of configuration parameter 'kerberos.principal'. -|authConfigs |Yes |Configuration parameters required by the authentication scheme defined by the type property. For more details, see https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[Hadoop configuration] options. -|defaultConfigs |No |Default values for the configuration parameters specified by the `authConfigs` property. The default values are specified as a collection of key-value pairs (i.e., `property-name:default_value`). -|enableDelegationToken |No |Enable (or disable) the delegation tokens functionality. -|initKerberosZk |No |For enabling initialization of kerberos before connecting to ZooKeeper (if applicable). -|proxyUserConfigs |No |Configures proxy users for the underlying Hadoop authentication mechanism. This configuration is expressed as a collection of key-value pairs (i.e., `property-name:value`). -|clientBuilderFactory |No |The `HttpClientBuilderFactory` implementation used for the Solr internal communication. Only applicable for `ConfigurableInternodeAuthHadoopPlugin`. -|=== [[HadoopAuthenticationPlugin-ExampleConfigurations]] == Example Configurations diff --git a/solr/solr-ref-guide/src/index-replication.adoc b/solr/solr-ref-guide/src/index-replication.adoc index df8e9c60371..774b78cde75 100644 --- a/solr/solr-ref-guide/src/index-replication.adoc +++ b/solr/solr-ref-guide/src/index-replication.adoc @@ -51,21 +51,33 @@ When using SolrCloud, the `ReplicationHandler` must be available via the `/repli The table below defines the key terms associated with Solr replication. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +Index:: +A Lucene index is a directory of files. These files make up the searchable and returnable data of a Solr Core. + +Distribution:: +The copying of an index from the master server to all slaves. The distribution process takes advantage of Lucene's index file structure. + +Inserts and Deletes:: +As inserts and deletes occur in the index, the directory remains unchanged. Documents are always inserted into newly created files. Documents that are deleted are not removed from the files. They are flagged in the file, deletable, and are not removed from the files until the index is optimized. + +Master and Slave:: +A Solr replication master is a single node which receives all updates initially and keeps everything organized. Solr replication slave nodes receive no updates directly, instead all changes (such as inserts, updates, deletes, etc.) are made against the single master node. Changes made on the master are distributed to all the slave nodes which service all query requests from the clients. + +Update:: +An update is a single change request against a single Solr instance. It may be a request to delete a document, add a new document, change a document, delete all documents matching a query, etc. Updates are handled synchronously within an individual Solr instance. + +Optimization:: +A process that compacts the index and merges segments in order to improve query performance. Optimization should only be run on the master nodes. An optimized index may give query performance gains compared to an index that has become fragmented over a period of time with many updates. Distributing an optimized index requires a much longer time than the distribution of new segments to an un-optimized index. + +Segments:: +A self contained subset of an index consisting of some documents and data structures related to the inverted index of terms in those documents. + +mergeFactor:: +A parameter that controls the number of segments in an index. For example, when mergeFactor is set to 3, Solr will fill one segment with documents until the limit maxBufferedDocs is met, then it will start a new segment. When the number of segments specified by mergeFactor is reached (in this example, 3) then Solr will merge all the segments into a single index file, then begin writing new documents to a new segment. + +Snapshot:: +A directory containing hard links to the data files of an index. Snapshots are distributed from the master nodes when the slaves pull them, "smart copying" any segments the slave node does not have in snapshot directory that contains the hard links to the most recent index data files. -[cols="30,70",options="header"] -|=== -|Term |Definition -|Index |A Lucene index is a directory of files. These files make up the searchable and returnable data of a Solr Core. -|Distribution |The copying of an index from the master server to all slaves. The distribution process takes advantage of Lucene's index file structure. -|Inserts and Deletes |As inserts and deletes occur in the index, the directory remains unchanged. Documents are always inserted into newly created files. Documents that are deleted are not removed from the files. They are flagged in the file, deletable, and are not removed from the files until the index is optimized. -|Master and Slave |A Solr replication master is a single node which receives all updates initially and keeps everything organized. Solr replication slave nodes receive no updates directly, instead all changes (such as inserts, updates, deletes, etc.) are made against the single master node. Changes made on the master are distributed to all the slave nodes which service all query requests from the clients. -|Update |An update is a single change request against a single Solr instance. It may be a request to delete a document, add a new document, change a document, delete all documents matching a query, etc. Updates are handled synchronously within an individual Solr instance. -|Optimization |A process that compacts the index and merges segments in order to improve query performance. Optimization should only be run on the master nodes. An optimized index may give query performance gains compared to an index that has become fragmented over a period of time with many updates. Distributing an optimized index requires a much longer time than the distribution of new segments to an un-optimized index. -|Segments |A self contained subset of an index consisting of some documents and data structures related to the inverted index of terms in those documents. -|mergeFactor |A parameter that controls the number of segments in an index. For example, when mergeFactor is set to 3, Solr will fill one segment with documents until the limit maxBufferedDocs is met, then it will start a new segment. When the number of segments specified by mergeFactor is reached (in this example, 3) then Solr will merge all the segments into a single index file, then begin writing new documents to a new segment. -|Snapshot |A directory containing hard links to the data files of an index. Snapshots are distributed from the master nodes when the slaves pull them, "smart copying" any segments the slave node does not have in snapshot directory that contains the hard links to the most recent index data files. -|=== [[IndexReplication-ConfiguringtheReplicationHandler]] == Configuring the ReplicationHandler @@ -80,17 +92,20 @@ In addition to `ReplicationHandler` configuration options specific to the master Before running a replication, you should set the following parameters on initialization of the handler: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`replicateAfter`:: +String specifying action after which replication should occur. Valid values are commit, optimize, or startup. There can be multiple values for this parameter. If you use "startup", you need to have a "commit" and/or "optimize" entry also if you want to trigger replication on future commits or optimizes. -[cols="30,70",options="header"] -|=== -|Name |Description -|replicateAfter |String specifying action after which replication should occur. Valid values are commit, optimize, or startup. There can be multiple values for this parameter. If you use "startup", you need to have a "commit" and/or "optimize" entry also if you want to trigger replication on future commits or optimizes. -|backupAfter |String specifying action after which a backup should occur. Valid values are commit, optimize, or startup. There can be multiple values for this parameter. It is not required for replication, it just makes a backup. -|maxNumberOfBackups |Integer specifying how many backups to keep. This can be used to delete all but the most recent N backups. -|confFiles |The configuration files to replicate, separated by a comma. -|commitReserveDuration |If your commits are very frequent and your network is slow, you can tweak this parameter to increase the amount of time taken to download 5Mb from the master to a slave. The default is 10 seconds. -|=== +`backupAfter` +String specifying action after which a backup should occur. Valid values are commit, optimize, or startup. There can be multiple values for this parameter. It is not required for replication, it just makes a backup. + +`maxNumberOfBackups` +Integer specifying how many backups to keep. This can be used to delete all but the most recent N backups. + +`confFiles`:: +The configuration files to replicate, separated by a comma. + +`commitReserveDuration`:: +If your commits are very frequent and your network is slow, you can tweak this parameter to increase the amount of time taken to download 5Mb from the master to a slave. The default is 10 seconds. The example below shows a possible 'master' configuration for the `ReplicationHandler`, including a fixed number of backups and an invariant setting for the `maxWriteMBPerSec` request parameter to prevent slaves from saturating its network interface @@ -203,17 +218,13 @@ Here is an example of a ReplicationHandler configuration for a repeater: When a commit or optimize operation is performed on the master, the RequestHandler reads the list of file names which are associated with each commit point. This relies on the `replicateAfter` parameter in the configuration to decide which types of events should trigger replication. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +These operations are supported: -[cols="30,70",options="header"] -|=== -|Setting on the Master |Description -|commit |Triggers replication whenever a commit is performed on the master index. -|optimize |Triggers replication whenever the master index is optimized. -|startup |Triggers replication whenever the master index starts up. -|=== +* `commit`: Triggers replication whenever a commit is performed on the master index. +* `optimize`: Triggers replication whenever the master index is optimized. +* `startup`: Triggers replication whenever the master index starts up. -The replicateAfter parameter can accept multiple arguments. For example: +The `replicateAfter` parameter can accept multiple arguments. For example: [source,xml] ---- @@ -262,36 +273,87 @@ To correct this problem, the slave then copies all the index files from master t You can use the HTTP commands below to control the ReplicationHandler's operations. -[width="100%",options="header",] -|=== -|Command |Description -|http://_master_host:port_/solr/_core_name_/replication?command=enablereplication |Enables replication on the master for all its slaves. -|http://_master_host:port_/solr/_core_name_/replication?command=disablereplication |Disables replication on the master for all its slaves. -|http://_host:port_/solr/_core_name_/replication?command=indexversion |Returns the version of the latest replicatable index on the specified master or slave. -|http://_slave_host:port_/solr/_core_name_/replication?command=fetchindex |Forces the specified slave to fetch a copy of the index from its master. If you like, you can pass an extra attribute such as masterUrl or compression (or any other parameter which is specified in the `` tag) to do a one time replication from a master. This obviates the need for hard-coding the master in the slave. -|http://_slave_host:port_/solr/_core_name_/replication?command=abortfetch |Aborts copying an index from a master to the specified slave. -|http://_slave_host:port_/solr/_core_name_/replication?command=enablepoll |Enables the specified slave to poll for changes on the master. -|http://_slave_host:port_/solr/_core_name_/replication?command=disablepoll |Disables the specified slave from polling for changes on the master. -|http://_slave_host:port_/solr/_core_name_/replication?command=details |Retrieves configuration details and current status. -|http://_host:port_/solr/_core_name_/replication?command=filelist&generation=<_generation-number_> |Retrieves a list of Lucene files present in the specified host's index. You can discover the generation number of the index by running the `indexversion` command. -|http://_master_host:port_/solr/_core_name_/replication?command=backup a| -Creates a backup on master if there are committed index data in the server; otherwise, does nothing. This command is useful for making periodic backups. +`enablereplication`:: +Enable replication on the "master" for all its slaves. ++ +[source,bash] +http://_master_host:port_/solr/_core_name_/replication?command=enablereplication -supported request parameters: +`disablereplication`:: +Disable replication on the master for all its slaves. ++ +[source,bash] +http://_master_host:port_/solr/_core_name_/replication?command=disablereplication -* `numberToKeep:` request parameter can be used with the backup command unless the `maxNumberOfBackups` initialization parameter has been specified on the handler – in which case `maxNumberOfBackups` is always used and attempts to use the `numberToKeep` request parameter will cause an error. -* `name` : (optional) Backup name . The snapshot will be created in a directory called snapshot. within the data directory of the core . By default the name is generated using date in `yyyyMMddHHmmssSSS` format. If `location` parameter is passed , that would be used instead of the data directory -* `location`: Backup location +`indexversion`:: +Return the version of the latest replicatable index on the specified master or slave. ++ +[source,bash] +http://_host:port_/solr/_core_name_/replication?command=indexversion -|http://_master_host:port_ /solr/_core_name_/replication?command=deletebackup a| -Delete any backup created using the `backup` command . +`fetchindex`:: +Force the specified slave to fetch a copy of the index from its master. ++ +[source.bash] +http://_slave_host:port_/solr/_core_name_/replication?command=fetchindex ++ +If you like, you can pass an extra attribute such as `masterUrl` or `compression` (or any other parameter which is specified in the `` tag) to do a one time replication from a master. This obviates the need for hard-coding the master in the slave. -Request parameters: +`abortfetch`:: +Abort copying an index from a master to the specified slave. ++ +[source,bash] +http://_slave_host:port_/solr/_core_name_/replication?command=abortfetch -* name: The name of the snapshot . A snapshot with the name snapshot. must exist .If not, an error is thrown -* location: Location where the snapshot is created +`enablepoll`:: +Enable the specified slave to poll for changes on the master. ++ +[source,bash] +http://_slave_host:port_/solr/_core_name_/replication?command=enablepoll + +`disablepoll`:: +Disable the specified slave from polling for changes on the master. ++ +[source,bash] +http://_slave_host:port_/solr/_core_name_/replication?command=disablepoll + +`details`:: +Retrieve configuration details and current status. ++ +[source,bash] +http://_slave_host:port_/solr/_core_name_/replication?command=details + +`filelist`:: +Retrieve a list of Lucene files present in the specified host's index. ++ +[source,bash] +http://_host:port_/solr/_core_name_/replication?command=filelist&generation=<_generation-number_> ++ +You can discover the generation number of the index by running the `indexversion` command. + +`backup`:: +Create a backup on master if there are committed index data in the server; otherwise, does nothing. ++ +[source,bash] +http://_master_host:port_/solr/_core_name_/replication?command=backup ++ +This command is useful for making periodic backups. There are several supported request parameters: ++ +* `numberToKeep:`: This can be used with the backup command unless the `maxNumberOfBackups` initialization parameter has been specified on the handler – in which case `maxNumberOfBackups` is always used and attempts to use the `numberToKeep` request parameter will cause an error. +* `name`: (optional) Backup name. The snapshot will be created in a directory called `snapshot.` within the data directory of the core. By default the name is generated using date in `yyyyMMddHHmmssSSS` format. If `location` parameter is passed, that would be used instead of the data directory +* `location`: Backup location. + +`deletebackup`:: +Delete any backup created using the `backup` command. ++ +[source,bash] +http://_master_host:port_ /solr/_core_name_/replication?command=deletebackup ++ +There are two supported parameters: + +* `name`: The name of the snapshot. A snapshot with the name `snapshot._name_` must exist. If not, an error is thrown. +* `location`: Location where the snapshot is created. -|=== [[IndexReplication-DistributionandOptimization]] == Distribution and Optimization @@ -302,7 +364,9 @@ The time required to optimize a master index can vary dramatically. A small inde Distributing a newly optimized index may take only a few minutes or up to an hour or more, again depending on the size of the index and the performance capabilities of network connections and disks. During optimization the machine is under load and does not process queries very well. Given a schedule of updates being driven a few times an hour to the slaves, we cannot run an optimize with every committed snapshot. -Copying an optimized index means that the *entire* index will need to be transferred during the next snappull. This is a large expense, but not nearly as huge as running the optimize everywhere. Consider this example: on a three-slave one-master configuration, distributing a newly-optimized index takes approximately 80 seconds _total_. Rolling the change across a tier would require approximately ten minutes per machine (or machine group). If this optimize were rolled across the query tier, and if each slave node being optimized were disabled and not receiving queries, a rollout would take at least twenty minutes and potentially as long as an hour and a half. Additionally, the files would need to be synchronized so that the _following_ the optimize, snappull would not think that the independently optimized files were different in any way. This would also leave the door open to independent corruption of indexes instead of each being a perfect copy of the master. +Copying an optimized index means that the *entire* index will need to be transferred during the next `snappull`. This is a large expense, but not nearly as huge as running the optimize everywhere. + +Consider this example: on a three-slave one-master configuration, distributing a newly-optimized index takes approximately 80 seconds _total_. Rolling the change across a tier would require approximately ten minutes per machine (or machine group). If this optimize were rolled across the query tier, and if each slave node being optimized were disabled and not receiving queries, a rollout would take at least twenty minutes and potentially as long as an hour and a half. Additionally, the files would need to be synchronized so that the _following_ the optimize, `snappull` would not think that the independently optimized files were different in any way. This would also leave the door open to independent corruption of indexes instead of each being a perfect copy of the master. Optimizing on the master allows for a straight-forward optimization operation. No query slaves need to be taken out of service. The optimized index can be distributed in the background as queries are being normally serviced. The optimization can occur at any time convenient to the application providing index updates. diff --git a/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc b/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc index ce3250355d2..63ab26dda8a 100644 --- a/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc @@ -192,15 +192,12 @@ The maximum time to wait for a write lock on an IndexWriter. The default is 1000 There are a few other parameters that may be important to configure for your implementation. These settings affect how or when updates are made to an index. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`reopenReaders`:: Controls if IndexReaders will be re-opened, instead of closed and then opened, which is often less efficient. The default is true. + +`deletionPolicy`:: Controls how commits are retained in case of rollback. The default is `SolrDeletionPolicy`, which has sub-parameters for the maximum number of commits to keep (`maxCommitsToKeep`), the maximum number of optimized commits to keep (`maxOptimizedCommitsToKeep`), and the maximum age of any commit to keep (`maxCommitAge`), which supports `DateMathParser` syntax. + +`infoStream`:: The InfoStream setting instructs the underlying Lucene classes to write detailed debug information from the indexing process as Solr log messages. -[cols="30,70",options="header"] -|=== -|Setting |Description -|reopenReaders |Controls if IndexReaders will be re-opened, instead of closed and then opened, which is often less efficient. The default is true. -|deletionPolicy |Controls how commits are retained in case of rollback. The default is `SolrDeletionPolicy`, which has sub-parameters for the maximum number of commits to keep (`maxCommitsToKeep`), the maximum number of optimized commits to keep (`maxOptimizedCommitsToKeep`), and the maximum age of any commit to keep (`maxCommitAge`), which supports `DateMathParser` syntax. -|infoStream |The InfoStream setting instructs the underlying Lucene classes to write detailed debug information from the indexing process as Solr log messages. -|=== [source,xml] ---- diff --git a/solr/solr-ref-guide/src/initparams-in-solrconfig.adoc b/solr/solr-ref-guide/src/initparams-in-solrconfig.adoc index 126e96b045b..ac409ff66ce 100644 --- a/solr/solr-ref-guide/src/initparams-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/initparams-in-solrconfig.adoc @@ -44,22 +44,16 @@ This sets the default search field ("df") to be "_text_" for all of the request The syntax and semantics are similar to that of a `` . The following are the attributes -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`path`:: +A comma-separated list of paths which will use the parameters. Wildcards can be used in paths to define nested paths, as described below. -[cols="30,70",options="header"] -|=== -|Property |Description -|path |A comma-separated list of paths which will use the parameters. Wildcards can be used in paths to define nested paths, as described below. -|name a| +`name`:: The name of this set of parameters. The name can be used directly in a requestHandler definition if a path is not explicitly named. If you give your `` a name, you can refer to the params in a `` that is not defined as a path. - ++ For example, if an `` section has the name "myParams", you can call the name when defining your request handler: - ++ [source,xml] ----- ----- -|=== [[InitParamsinSolrConfig-Wildcards]] == Wildcards diff --git a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc index 396243349ce..da963166f37 100644 --- a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc @@ -232,19 +232,26 @@ The main properties we are concerned with are the `keyTab` and `principal` prope While starting up Solr, the following host-specific parameters need to be passed. These parameters can be passed at the command line with the `bin/solr` start command (see <> for details on how to pass system parameters) or defined in `bin/solr.in.sh` or `bin/solr.in.cmd` as appropriate for your operating system. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`solr.kerberos.name.rules`:: +Used to map Kerberos principals to short names. Default value is `DEFAULT`. Example of a name rule: `RULE:[1:$1@$0](.\*EXAMPLE.COM)s/@.*//`. + +`solr.kerberos.cookie.domain`:: Used to issue cookies and should have the hostname of the Solr node. This parameter is required. + +`solr.kerberos.cookie.portaware`:: +When set to `true`, cookies are differentiated based on host and port, as opposed to standard cookies which are not port aware. This should be set if more than one Solr node is hosted on the same host. The default is `false`. + +`solr.kerberos.principal`:: +The service principal. This parameter is required. + +`solr.kerberos.keytab`:: +Keytab file path containing service principal credentials. This parameter is required. + +`solr.kerberos.jaas.appname`:: +The app name (section name) within the JAAS configuration file which is required for internode communication. Default is `Client`, which is used for ZooKeeper authentication as well. If different users are used for ZooKeeper and Solr, they will need to have separate sections in the JAAS configuration file. + +`java.security.auth.login.config`:: +Path to the JAAS configuration file for configuring a Solr client for internode communication. This parameter is required. -[cols="35,10,55",options="header"] -|=== -|Parameter Name |Required |Description -|`solr.kerberos.name.rules` |No |Used to map Kerberos principals to short names. Default value is `DEFAULT`. Example of a name rule: `RULE:[1:$1@$0](.\*EXAMPLE.COM)s/@.*//` -|`solr.kerberos.cookie.domain` |Yes |Used to issue cookies and should have the hostname of the Solr node. -|`solr.kerberos.cookie.portaware` |No |When set to true, cookies are differentiated based on host and port, as opposed to standard cookies which are not port aware. This should be set if more than one Solr node is hosted on the same host. The default is false. -|`solr.kerberos.principal` |Yes |The service principal. -|`solr.kerberos.keytab` |Yes |Keytab file path containing service principal credentials. -|`solr.kerberos.jaas.appname` |No |The app name (section name) within the JAAS configuration file which is required for internode communication. Default is `Client`, which is used for ZooKeeper authentication as well. If different users are used for ZooKeeper and Solr, they will need to have separate sections in the JAAS configuration file. -|`java.security.auth.login.config` |Yes |Path to the JAAS configuration file for configuring a Solr client for internode communication. -|=== Here is an example that could be added to `bin/solr.in.sh`. Make sure to change this example to use the right hostname and the keytab file path. @@ -279,18 +286,23 @@ There are a few use cases for Solr where this might be helpful: To enable delegation tokens, several parameters must be defined. These parameters can be passed at the command line with the `bin/solr` start command (see <> for details on how to pass system parameters) or defined in `bin/solr.in.sh` or `bin/solr.in.cmd` as appropriate for your operating system. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`solr.kerberos.delegation.token.enabled`:: +This is `false` by default, set to `true` to enable delegation tokens. This parameter is required if you want to enable tokens. -[cols="40,10,50",options="header"] -|=== -|Parameter Name |Required |Description -|`solr.kerberos.delegation.token.enabled` |Yes, to enable tokens |False by default, set to true to enable delegation tokens. -|`solr.kerberos.delegation.token.kind` |No |Type of delegation tokens. By default this is `solr-dt`. Likely this does not need to change. No other option is available at this time. -|`solr.kerberos.delegation.token.validity` |No |Time, in seconds, for which delegation tokens are valid. The default is 36000 seconds. -|`solr.kerberos.delegation.token.signer.secret.provider` |No |Where delegation token information is stored internally. The default is `zookeeper` which must be the location for delegation tokens to work across Solr servers (when running in SolrCloud mode). No other option is available at this time. -|`solr.kerberos.delegation.token.signer.secret.provider.zookeper.path` |No |The ZooKeeper path where the secret provider information is stored. This is in the form of the path + /security/token. The path can include the chroot or the chroot can be omitted if you are not using it. This example includes the chroot: `server1:9983,server2:9983,server3:9983/solr/security/token`. -|`solr.kerberos.delegation.token.secret.manager.znode.working.path` |No |The ZooKeeper path where token information is stored. This is in the form of the path + /security/zkdtsm. The path can include the chroot or the chroot can be omitted if you are not using it. This example includes the chroot: `server1:9983,server2:9983,server3:9983/solr/security/zkdtsm`. -|=== +`solr.kerberos.delegation.token.kind`:: +The type of delegation tokens. By default this is `solr-dt`. Likely this does not need to change. No other option is available at this time. + +`solr.kerberos.delegation.token.validity`:: +Time, in seconds, for which delegation tokens are valid. The default is 36000 seconds. + +`solr.kerberos.delegation.token.signer.secret.provider`:: +Where delegation token information is stored internally. The default is `zookeeper` which must be the location for delegation tokens to work across Solr servers (when running in SolrCloud mode). No other option is available at this time. + +`solr.kerberos.delegation.token.signer.secret.provider.zookeper.path`:: +The ZooKeeper path where the secret provider information is stored. This is in the form of the path + /security/token. The path can include the chroot or the chroot can be omitted if you are not using it. This example includes the chroot: `server1:9983,server2:9983,server3:9983/solr/security/token`. + +`solr.kerberos.delegation.token.secret.manager.znode.working.path`:: +The ZooKeeper path where token information is stored. This is in the form of the path + /security/zkdtsm. The path can include the chroot or the chroot can be omitted if you are not using it. This example includes the chroot: `server1:9983,server2:9983,server3:9983/solr/security/zkdtsm`. [[KerberosAuthenticationPlugin-StartSolr]] === Start Solr diff --git a/solr/solr-ref-guide/src/making-and-restoring-backups.adoc b/solr/solr-ref-guide/src/making-and-restoring-backups.adoc index 3d7a17de6b5..6f3383c1b45 100644 --- a/solr/solr-ref-guide/src/making-and-restoring-backups.adoc +++ b/solr/solr-ref-guide/src/making-and-restoring-backups.adoc @@ -37,7 +37,7 @@ Backups and restoration uses Solr's replication handler. Out of the box, Solr in === Backup API -The backup API requires sending a command to the `/replication` handler to back up the system. +The `backup` API requires sending a command to the `/replication` handler to back up the system. You can trigger a back-up with an HTTP command like this (replace "gettingstarted" with the name of the core you are working with): @@ -47,27 +47,28 @@ You can trigger a back-up with an HTTP command like this (replace "gettingstarte http://localhost:8983/solr/gettingstarted/replication?command=backup ---- -The backup command is an asynchronous call, and it will represent data from the latest index commit point. All indexing and search operations will continue to be executed against the index as usual. +The `backup` command is an asynchronous call, and it will represent data from the latest index commit point. All indexing and search operations will continue to be executed against the index as usual. Only one backup call can be made against a core at any point in time. While an ongoing backup operation is happening subsequent calls for restoring will throw an exception. The backup request can also take the following additional parameters: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`location`:: +The path where the backup will be created. If the path is not absolute then the backup path will be relative to Solr's instance directory. +|name |The snapshot will be created in a directory called `snapshot.`. If a name is not specified then the directory name would have the following format: `snapshot.`. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|location |The path where the backup will be created. If the path is not absolute then the backup path will be relative to Solr's instance directory. -|name |The snapshot will be created in a directory called `snapshot.`. If a name is not specified then the directory name would have the following format: `snapshot.` -|numberToKeep |The number of backups to keep. If `maxNumberOfBackups` has been specified on the replication handler in `solrconfig.xml`, `maxNumberOfBackups` is always used and attempts to use `numberToKeep` will cause an error. Also, this parameter is not taken into consideration if the backup name is specified. More information about `maxNumberOfBackups` can be found in the section <>. -|repository |The name of the repository to be used for the backup. If no repository is specified then the local filesystem repository will be used automatically. -|commitName |The name of the commit which was used while taking a snapshot using the CREATESNAPSHOT command. -|=== +`numberToKeep`:: +The number of backups to keep. If `maxNumberOfBackups` has been specified on the replication handler in `solrconfig.xml`, `maxNumberOfBackups` is always used and attempts to use `numberToKeep` will cause an error. Also, this parameter is not taken into consideration if the backup name is specified. More information about `maxNumberOfBackups` can be found in the section <>. + +`repository`:: +The name of the repository to be used for the backup. If no repository is specified then the local filesystem repository will be used automatically. + +`commitName`:: +The name of the commit which was used while taking a snapshot using the CREATESNAPSHOT command. === Backup Status -The backup operation can be monitored to see if it has completed by sending the `details` command to the `/replication` handler, as in this example: +The `backup` operation can be monitored to see if it has completed by sending the `details` command to the `/replication` handler, as in this example: .Status API Example [source,text] @@ -103,25 +104,24 @@ http://localhost:8983/solr/gettingstarted/replication?command=restore&name=backu This will restore the named index snapshot into the current core. Searches will start reflecting the snapshot data once the restore is complete. -The restore request can also take these additional parameters: +The `restore` request can take these additional parameters: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`location`:: +The location of the backup snapshot file. If not specified, it looks for backups in Solr's data directory. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|location |The location of the backup snapshot file. If not specified, it looks for backups in Solr's data directory. -|name |The name of the backed up index snapshot to be restored. If the name is not provided it looks for backups with `snapshot.` format in the location directory. It picks the latest timestamp backup in that case. -|repository |The name of the repository to be used for the backup. If no repository is specified then the local filesystem repository will be used automatically. -|=== +`name`:: +The name of the backed up index snapshot to be restored. If the name is not provided it looks for backups with `snapshot.` format in the location directory. It picks the latest timestamp backup in that case. -The restore command is an asynchronous call. Once the restore is complete the data reflected will be of the backed up index which was restored. +`repository`:: +The name of the repository to be used for the backup. If no repository is specified then the local filesystem repository will be used automatically. -Only one restore call can can be made against a core at one point in time. While an ongoing restore operation is happening subsequent calls for restoring will throw an exception. +The `restore` command is an asynchronous call. Once the restore is complete the data reflected will be of the backed up index which was restored. + +Only one `restore` call can can be made against a core at one point in time. While an ongoing restore operation is happening subsequent calls for restoring will throw an exception. === Restore Status API -You can also check the status of a restore operation by sending the `restorestatus` command to the `/replication` handler, as in this example: +You can also check the status of a `restore` operation by sending the `restorestatus` command to the `/replication` handler, as in this example: .Status API Example [source,text] @@ -158,21 +158,18 @@ You can trigger a snapshot command with an HTTP command like this (replace "tech http://localhost:8983/solr/admin/cores?action=CREATESNAPSHOT&core=techproducts&commitName=commit1 ---- -The list snapshot request parameters are: +The `CREATESNAPSHOT` request parameters are: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`commitName`:: +The name to store the snapshot as. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|commitName |Specify the commit name to store the snapshot as -|core |name of the core to perform the snapshot on -|async |Request ID to track this action which will be processed asynchronously -|=== +`core`:: The name of the core to perform the snapshot on. + +`async`:: Request ID to track this action which will be processed asynchronously. === List Snapshot API -The list snapshot functionality lists all the taken snapshots for a particular core. +The `LISTSNAPSHOTS` command lists all the taken snapshots for a particular core. You can trigger a list snapshot command with an HTTP command like this (replace "techproducts" with the name of the core you are working with): @@ -184,20 +181,17 @@ http://localhost:8983/solr/admin/cores?action=LISTSNAPSHOTS&core=techproducts&co The list snapshot request parameters are: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +core:: +The name of the core to whose snapshots we want to list. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|core |name of the core to whose snapshots we want to list -|async |Request ID to track this action which will be processed asynchronously -|=== +async:: +Request ID to track this action which will be processed asynchronously. === Delete Snapshot API -The delete snapshot functionality deletes a particular snapshot for a particular core. +The `DELETESNAPSHOT` command deletes a snapshot for a particular core. -You can trigger a delete snapshot command with an HTTP command like this (replace "techproducts" with the name of the core you are working with): +You can trigger a delete snapshot with an HTTP command like this (replace "techproducts" with the name of the core you are working with): .Delete Snapshot API Example [source,text] @@ -207,13 +201,15 @@ http://localhost:8983/solr/admin/cores?action=DELETESNAPSHOT&core=techproducts&c The delete snapshot request parameters are: -[width="100%",options="header",] -|=== -|Parameter |Description -|commitName |Specify the commit name to be deleted -|core |name of the core whose snapshot we want to delete -|async |Request ID to track this action which will be processed asynchronously -|=== +`commitName`:: +Specify the commit name to be deleted + +`core`:: +The name of the core whose snapshot we want to delete + +`async`:: +Request ID to track this action which will be processed asynchronously + == Backup/Restore Storage Repositories diff --git a/solr/solr-ref-guide/src/mbean-request-handler.adoc b/solr/solr-ref-guide/src/mbean-request-handler.adoc index 6900cf0b0bb..eebd082e9b3 100644 --- a/solr/solr-ref-guide/src/mbean-request-handler.adoc +++ b/solr/solr-ref-guide/src/mbean-request-handler.adoc @@ -22,16 +22,17 @@ The MBean Request Handler offers programmatic access to the information provided The MBean Request Handler accepts the following parameters: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`key`:: +Restricts results by object key. -[cols="10,20,10,60",options="header"] -|=== -|Parameter |Type |Default |Description -|key |multivalued |all |Restricts results by object key. -|cat |multivalued |all |Restricts results by category name. -|stats |boolean |false |Specifies whether statistics are returned with results. You can override the `stats` parameter on a per-field basis. -|wt |multivalued |xml |The output format. This operates the same as the <>. -|=== +`cat`:: +Restricts results by category name. + +`stats`:: +Specifies whether statistics are returned with results. You can override the `stats` parameter on a per-field basis. The default is `false`. + +`wt`:: +The output format. This operates the same as the <>. The default is `xml`. [[MBeanRequestHandler-Examples]] == Examples diff --git a/solr/solr-ref-guide/src/morelikethis.adoc b/solr/solr-ref-guide/src/morelikethis.adoc index ec6129eeb29..e0756cbbc55 100644 --- a/solr/solr-ref-guide/src/morelikethis.adoc +++ b/solr/solr-ref-guide/src/morelikethis.adoc @@ -47,51 +47,63 @@ The next phase filters terms from the original document using thresholds defined The table below summarizes the `MoreLikeThis` parameters supported by Lucene/Solr. These parameters can be used with any of the three possible MoreLikeThis approaches. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`mlt.fl`:: +Specifies the fields to use for similarity. If possible, these should have stored `termVectors`. + +`mlt.mintf`:: +Specifies the Minimum Term Frequency, the frequency below which terms will be ignored in the source document. + +`mlt.mindf`:: +Specifies the Minimum Document Frequency, the frequency at which words will be ignored which do not occur in at least this many documents. + +`mlt.maxdf`:: +Specifies the Maximum Document Frequency, the frequency at which words will be ignored which occur in more than this many documents. + +`mlt.minwl`:: +Sets the minimum word length below which words will be ignored. + +`mlt.maxwl`:: +Sets the maximum word length above which words will be ignored. + +`mlt.maxqt`:: +Sets the maximum number of query terms that will be included in any generated query. + +`mlt.maxntp`:: +Sets the maximum number of tokens to parse in each example document field that is not stored with TermVector support. + +`mlt.boost`:: +Specifies if the query will be boosted by the interesting term relevance. It can be either "true" or "false". + +`mlt.qf`:: +Query fields and their boosts using the same format as that used by the <>. These fields must also be specified in `mlt.fl`. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|mlt.fl |Specifies the fields to use for similarity. If possible, these should have stored `termVectors`. -|mlt.mintf |Specifies the Minimum Term Frequency, the frequency below which terms will be ignored in the source document. -|mlt.mindf |Specifies the Minimum Document Frequency, the frequency at which words will be ignored which do not occur in at least this many documents. -|mlt.maxdf |Specifies the Maximum Document Frequency, the frequency at which words will be ignored which occur in more than this many documents. -|mlt.minwl |Sets the minimum word length below which words will be ignored. -|mlt.maxwl |Sets the maximum word length above which words will be ignored. -|mlt.maxqt |Sets the maximum number of query terms that will be included in any generated query. -|mlt.maxntp |Sets the maximum number of tokens to parse in each example document field that is not stored with TermVector support. -|mlt.boost |Specifies if the query will be boosted by the interesting term relevance. It can be either "true" or "false". -|mlt.qf |Query fields and their boosts using the same format as that used by the <>. These fields must also be specified in `mlt.fl`. -|=== [[MoreLikeThis-ParametersfortheMoreLikeThisComponent]] == Parameters for the MoreLikeThisComponent Using MoreLikeThis as a search component returns similar documents for each document in the response set. In addition to the common parameters, these additional options are available: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`mlt`:: +If set to `true`, activates the `MoreLikeThis` component and enables Solr to return `MoreLikeThis` results. + +`mlt.count`:: +Specifies the number of similar documents to be returned for each result. The default value is 5. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|mlt |If set to true, activates the `MoreLikeThis` component and enables Solr to return `MoreLikeThis` results. -|mlt.count |Specifies the number of similar documents to be returned for each result. The default value is 5. -|=== [[MoreLikeThis-ParametersfortheMoreLikeThisHandler]] == Parameters for the MoreLikeThisHandler The table below summarizes parameters accessible through the `MoreLikeThisHandler`. It supports faceting, paging, and filtering using common query parameters, but does not work well with alternate query parsers. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`mlt.match.include`:: +Specifies whether or not the response should include the matched document. If set to false, the response will look like a normal select response. + +`mlt.match.offset`:: +Specifies an offset into the main query search results to locate the document on which the `MoreLikeThis` query should operate. By default, the query operates on the first result for the q parameter. + +`mlt.interestingTerms`:: +Controls how the `MoreLikeThis` component presents the "interesting" terms (the top TF/IDF terms) for the query. Supports three settings. The setting list lists the terms. The setting none lists no terms. The setting details lists the terms along with the boost value used for each term. Unless `mlt.boost=true`, all terms will have `boost=1.0`. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|mlt.match.include |Specifies whether or not the response should include the matched document. If set to false, the response will look like a normal select response. -|mlt.match.offset |Specifies an offset into the main query search results to locate the document on which the `MoreLikeThis` query should operate. By default, the query operates on the first result for the q parameter. -|mlt.interestingTerms |Controls how the `MoreLikeThis` component presents the "interesting" terms (the top TF/IDF terms) for the query. Supports three settings. The setting list lists the terms. The setting none lists no terms. The setting details lists the terms along with the boost value used for each term. Unless `mlt.boost=true`, all terms will have `boost=1.0`. -|=== [[MoreLikeThis-MoreLikeThisQueryParser]] == More Like This Query Parser diff --git a/solr/solr-ref-guide/src/near-real-time-searching.adoc b/solr/solr-ref-guide/src/near-real-time-searching.adoc index 8d87d5477e9..fe0e44988bd 100644 --- a/solr/solr-ref-guide/src/near-real-time-searching.adoc +++ b/solr/solr-ref-guide/src/near-real-time-searching.adoc @@ -37,14 +37,11 @@ An *optimize* is like a *hard commit* except that it forces all of the index seg Soft commit takes uses two parameters: `maxDocs` and `maxTime`. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`maxDocs`:: +Integer. Defines the number of documents to queue before pushing them to the index. It works in conjunction with the `update_handler_autosoftcommit_max_time` parameter in that if either limit is reached, the documents will be pushed to the index. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|`maxDocs` |Integer. Defines the number of documents to queue before pushing them to the index. It works in conjunction with the `update_handler_autosoftcommit_max_time` parameter in that if either limit is reached, the documents will be pushed to the index. -|`maxTime` |The number of milliseconds to wait before pushing documents to the index. It works in conjunction with the `update_handler_autosoftcommit_max_docs` parameter in that if either limit is reached, the documents will be pushed to the index. -|=== +`maxTime`:: +The number of milliseconds to wait before pushing documents to the index. It works in conjunction with the `update_handler_autosoftcommit_max_docs` parameter in that if either limit is reached, the documents will be pushed to the index. Use `maxDocs` and `maxTime` judiciously to fine-tune your commit strategies. @@ -78,17 +75,20 @@ It's better to use `maxTime` rather than `maxDocs` to modify an `autoSoftCommit` [[NearRealTimeSearching-OptionalAttributesforcommitandoptimize]] === Optional Attributes for commit and optimize -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`waitSearcher`:: +Block until a new searcher is opened and registered as the main query searcher, making the changes visible. Default is `true`. -[cols="20,20,60",options="header"] -|=== -|Parameter |Valid Attributes |Description -|`waitSearcher` |true, false |Block until a new searcher is opened and registered as the main query searcher, making the changes visible. Default is true. -|`OpenSearcher` |true, false |Open a new searcher making all documents indexed so far visible for searching. Default is true. -|`softCommit` |true, false |Perform a soft commit. This will refresh the view of the index faster, but without guarantees that the document is stably stored. Default is false. -|`expungeDeletes` |true, false |Valid for `commit` only. This parameter purges deleted data from segments. The default is false. -|`maxSegments` |integer |Valid for `optimize` only. Optimize down to at most this number of segments. The default is 1. -|=== +`OpenSearcher`:: +Open a new searcher making all documents indexed so far visible for searching. Default is `true`. + +`softCommit`:: +Perform a soft commit. This will refresh the view of the index faster, but without guarantees that the document is stably stored. Default is `false`. + +`expungeDeletes`:: +Valid for `commit` only. This parameter purges deleted data from segments. The default is `false`. + +`maxSegments`:: +Valid for `optimize` only. Optimize down to at most this number of segments. The default is `1`. Example of `commit` and `optimize` with optional attributes: diff --git a/solr/solr-ref-guide/src/parameter-reference.adoc b/solr/solr-ref-guide/src/parameter-reference.adoc index 7f395d574fd..a511bedd1b8 100644 --- a/solr/solr-ref-guide/src/parameter-reference.adoc +++ b/solr/solr-ref-guide/src/parameter-reference.adoc @@ -20,48 +20,39 @@ == Cluster Parameters -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`numShards`:: +Defaults to `1`. The number of shards to hash documents to. There must be one leader per shard and each leader can have _N_ replicas. -[cols="20,20,60"] -|=== -|`numShards` |Defaults to 1 |The number of shards to hash documents to. There must be one leader per shard and each leader can have N replicas. -|=== == SolrCloud Instance Parameters These are set in `solr.xml`, but by default the `host` and `hostContext` parameters are set up to also work with system properties. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`host`:: +Defaults to the first local host address found. If the wrong host address is found automatically, you can override the host address with this parameter. -[cols="20,20,60"] -|=== -|`host` |Defaults to the first local host address found |If the wrong host address is found automatically, you can override the host address with this parameter. -|`hostPort` |Defaults to the port specified via `bin/solr -p `, or `8983` if not specified. |The port that Solr is running on. This value is only used when `-DzkRun` is specified without a value (see below), to calculate the default port on which embedded ZooKeeper will run. **I**n the `solr.xml` shipped with Solr, the `hostPort` system property is not referenced, and so is ignored. If you want to run Solr on a non-default port, use `bin/solr -p ` rather than specifying `-DhostPort`. -|`hostContext` |Defaults to `solr` |The context path for the Solr web application. -|=== +`hostPort`:: +Defaults to the port specified via `bin/solr -p `, or `8983` if not specified. The port that Solr is running on. This value is only used when `-DzkRun` is specified without a value (see below), to calculate the default port on which embedded ZooKeeper will run. In the `solr.xml` shipped with Solr, the `hostPort` system property is not referenced, and so is ignored. If you want to run Solr on a non-default port, use `bin/solr -p ` rather than specifying `-DhostPort`. + +`hostContext`:: +Defaults to `solr`. The context path for the Solr web application. == SolrCloud Instance ZooKeeper Parameters -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`zkRun`:: +Defaults to `localhost:`. Causes Solr to run an embedded version of ZooKeeper. Set to the address of ZooKeeper on this node; this allows us to know who you are in the list of addresses in the `zkHost` connect string. Use `-DzkRun` (with no value) to get the default value. -[cols="20,20,60"] -|=== -|`zkRun` |Defaults to `localhost:` |Causes Solr to run an embedded version of ZooKeeper. Set to the address of ZooKeeper on this node; this allows us to know who you are in the list of addresses in the `zkHost` connect string. Use `-DzkRun` (with no value) to get the default value. -|`zkHost` |No default |The host address for ZooKeeper. Usually this is a comma-separated list of addresses to each node in your ZooKeeper ensemble. -|`zkClientTimeout` |Defaults to 15000 |The time a client is allowed to not talk to ZooKeeper before its session expires. -|=== +`zkHost`:: +The host address for ZooKeeper. Usually this is a comma-separated list of addresses to each node in your ZooKeeper ensemble. + +`zkClientTimeout`:: +Defaults to 15000. The time a client is allowed to not talk to ZooKeeper before its session expires. `zkRun` and `zkHost` are set up using system properties. `zkClientTimeout` is set up in `solr.xml` by default, but can also be set using a system property. == SolrCloud Core Parameters -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed - -[cols="20,20,60"] -|=== -|`shard` |Defaults to being automatically assigned based on numShards |Specifies which shard this core acts as a replica of. -|=== - -`shard` can be specified in the <> for each core. +`shard`:: +Defaults to being automatically assigned based on numShards. Specifies which shard this core acts as a replica of. `shard` can be specified in the <> for each core. Additional cloud related parameters are discussed in <> diff --git a/solr/solr-ref-guide/src/query-screen.adoc b/solr/solr-ref-guide/src/query-screen.adoc index 23c2334c746..e089f4ff824 100644 --- a/solr/solr-ref-guide/src/query-screen.adoc +++ b/solr/solr-ref-guide/src/query-screen.adoc @@ -33,26 +33,48 @@ The response has at least two sections, but may have several more depending on t The `response` includes the documents that matched the query, in `doc` sub-sections. The fields return depend on the parameters of the query (and the defaults of the request handler used). The number of results is also included in this section. -This screen allows you to experiment with different query options, and inspect how your documents were indexed. The query parameters available on the form are some basic options that most users want to have available, but there are dozens more available which could be simply added to the basic request by hand (if opened in a browser). The table below explains the parameters available: +This screen allows you to experiment with different query options, and inspect how your documents were indexed. The query parameters available on the form are some basic options that most users want to have available, but there are dozens more available which could be simply added to the basic request by hand (if opened in a browser). The following parameters are available: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +Request-handler (qt):: +Specifies the query handler for the request. If a query handler is not specified, Solr processes the response with the standard query handler. -[cols="20,80",options="header"] -|=== -|Field |Description -|Request-handler (qt) |Specifies the query handler for the request. If a query handler is not specified, Solr processes the response with the standard query handler. -|q |The query event. See <> for an explanation of this parameter. -|fq |The filter queries. See <> for more information on this parameter. -|sort |Sorts the response to a query in either ascending or descending order based on the response's score or another specified characteristic. -|start, rows |`start` is the offset into the query result starting at which documents should be returned. The default value is 0, meaning that the query should return results starting with the first document that matches. This field accepts the same syntax as the start query parameter, which is described in <>. `rows` is the number of rows to return. -|fl |Defines the fields to return for each document. You can explicitly list the stored fields, <>, and <> you want to have returned by separating them with either a comma or a space. -|wt |Specifies the Response Writer to be used to format the query response. Defaults to XML if not specified. -|indent |Click this button to request that the Response Writer use indentation to make the responses more readable. -|debugQuery |Click this button to augment the query response with debugging information, including "explain info" for each document returned. This debugging information is intended to be intelligible to the administrator or programmer. -|dismax |Click this button to enable the Dismax query parser. See <> for further information. -|edismax |Click this button to enable the Extended query parser. See <> for further information. -|hl |Click this button to enable highlighting in the query response. See <> for more information. -|facet |Enables faceting, the arrangement of search results into categories based on indexed terms. See <> for more information. -|spatial |Click to enable using location data for use in spatial or geospatial searches. See <> for more information. -|spellcheck |Click this button to enable the Spellchecker, which provides inline query suggestions based on other, similar, terms. See <> for more information. -|=== +q:: +The query event. See <> for an explanation of this parameter. + +fq:: +The filter queries. See <> for more information on this parameter. + +sort:: +Sorts the response to a query in either ascending or descending order based on the response's score or another specified characteristic. + +start, rows:: +`start` is the offset into the query result starting at which documents should be returned. The default value is 0, meaning that the query should return results starting with the first document that matches. This field accepts the same syntax as the start query parameter, which is described in <>. `rows` is the number of rows to return. + +fl:: +Defines the fields to return for each document. You can explicitly list the stored fields, <>, and <> you want to have returned by separating them with either a comma or a space. + +wt:: +Specifies the Response Writer to be used to format the query response. Defaults to XML if not specified. + +indent:: +Click this button to request that the Response Writer use indentation to make the responses more readable. + +debugQuery:: +Click this button to augment the query response with debugging information, including "explain info" for each document returned. This debugging information is intended to be intelligible to the administrator or programmer. + +dismax:: +Click this button to enable the Dismax query parser. See <> for further information. + +edismax:: +Click this button to enable the Extended query parser. See <> for further information. + +hl:: Click this button to enable highlighting in the query response. See <> for more information. + +facet:: +Enables faceting, the arrangement of search results into categories based on indexed terms. See <> for more information. + +spatial:: +Click to enable using location data for use in spatial or geospatial searches. See <> for more information. + +spellcheck:: +Click this button to enable the Spellchecker, which provides inline query suggestions based on other, similar, terms. See <> for more information. diff --git a/solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adoc b/solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adoc index 2883f9af368..430b4041389 100644 --- a/solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adoc @@ -71,15 +71,14 @@ The `` element controls HTTP cache control headers. Do not confuse This element allows for three attributes and one sub-element. The attributes of the `` element control whether a 304 response to a GET request is allowed, and if so, what sort of response it should be. When an HTTP client application issues a GET, it may optionally specify that a 304 response is acceptable if the resource has not been modified since the last time it was fetched. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`never304`:: +If present with the value `true`, then a GET request will never respond with a 304 code, even if the requested resource has not been modified. When this attribute is set to true, the next two attributes are ignored. Setting this to true is handy for development, as the 304 response can be confusing when tinkering with Solr responses through a web browser or other client that supports cache headers. -[cols="20,80",options="header"] -|=== -|Parameter |Description -|never304 |If present with the value `true`, then a GET request will never respond with a 304 code, even if the requested resource has not been modified. When this attribute is set to true, the next two attributes are ignored. Setting this to true is handy for development, as the 304 response can be confusing when tinkering with Solr responses through a web browser or other client that supports cache headers. -|lastModFrom |This attribute may be set to either `openTime` (the default) or `dirLastMod`. The value `openTime` indicates that last modification times, as compared to the If-Modified-Since header sent by the client, should be calculated relative to the time the Searcher started. Use `dirLastMod` if you want times to exactly correspond to when the index was last updated on disk. -|etagSeed |This value of this attribute is sent as the value of the `ETag` header. Changing this value can be helpful to force clients to re-fetch content even when the indexes have not changed---for example, when you've made some changes to the configuration. -|=== +`lastModFrom`:: +This attribute may be set to either `openTime` (the default) or `dirLastMod`. The value `openTime` indicates that last modification times, as compared to the If-Modified-Since header sent by the client, should be calculated relative to the time the Searcher started. Use `dirLastMod` if you want times to exactly correspond to when the index was last updated on disk. + +`etagSeed`:: +This value of this attribute is sent as the value of the `ETag` header. Changing this value can be helpful to force clients to re-fetch content even when the indexes have not changed---for example, when you've made some changes to the configuration. [source,xml] ---- diff --git a/solr/solr-ref-guide/src/requesthandlers-and-searchcomponents-in-solrconfig.adoc b/solr/solr-ref-guide/src/requesthandlers-and-searchcomponents-in-solrconfig.adoc index 15ddef66c0a..46d9c9ebece 100644 --- a/solr/solr-ref-guide/src/requesthandlers-and-searchcomponents-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/requesthandlers-and-searchcomponents-in-solrconfig.adoc @@ -127,13 +127,13 @@ There are several default search components that work with all SearchHandlers wi [cols="20,40,40",options="header"] |=== |Component Name |Class Name |More Information -|query |solr.QueryComponent |Described in the section <>. -|facet |solr.FacetComponent |Described in the section <>. -|mlt |solr.MoreLikeThisComponent |Described in the section <>. -|highlight |solr.HighlightComponent |Described in the section <>. -|stats |solr.StatsComponent |Described in the section <>. -|debug |solr.DebugComponent |Described in the section on <>. -|expand |solr.ExpandComponent |Described in the section <>. +|query |`solr.QueryComponent` |Described in the section <>. +|facet |`solr.FacetComponent` |Described in the section <>. +|mlt |`solr.MoreLikeThisComponent` |Described in the section <>. +|highlight |`solr.HighlightComponent` |Described in the section <>. +|stats |`solr.StatsComponent` |Described in the section <>. +|debug |`solr.DebugComponent` |Described in the section on <>. +|expand |`solr.ExpandComponent` |Described in the section <>. |=== If you register a new search component with one of these default names, the newly defined component will be used instead of the default. diff --git a/solr/solr-ref-guide/src/response-writers.adoc b/solr/solr-ref-guide/src/response-writers.adoc index 8d113705ab1..4f33effde0a 100644 --- a/solr/solr-ref-guide/src/response-writers.adoc +++ b/solr/solr-ref-guide/src/response-writers.adoc @@ -19,27 +19,26 @@ // specific language governing permissions and limitations // under the License. -A Response Writer generates the formatted response of a search. Solr supports a variety of Response Writers to ensure that query responses can be parsed by the appropriate language or application. +A Response Writer generates the formatted response of a search. -The `wt` parameter selects the Response Writer to be used. The table below lists the most common settings for the `wt` parameter. +Solr supports a variety of Response Writers to ensure that query responses can be parsed by the appropriate language or application. + +The `wt` parameter selects the Response Writer to be used. The list below describe shows the most common settings for the `wt` parameter, with links to further sections that discuss them in more detail. + +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> -[width="100%",options="header",] -|=== -|`wt` Parameter Setting |Response Writer Selected -|csv |<> -|geojson |<> -|javabin |<> -|json |<> -|php |<> -|phps |<> -|python |<> -|ruby |<> -|smile |<> -|velocity |<> -|xlsx |<> -|xml |<> -|xslt |<> -|=== [[ResponseWriters-TheStandardXMLResponseWriter]] == The Standard XML Response Writer @@ -55,13 +54,7 @@ The behavior of the XML Response Writer can be driven by the following query par The `version` parameter determines the XML protocol used in the response. Clients are strongly encouraged to _always_ specify the protocol version, so as to ensure that the format of the response they receive does not change unexpectedly if the Solr server is upgraded and a new default format is introduced. -Currently supported version values are: - -[width="100%",options="header",] -|=== -|XML Version |Notes -|2.2 |The format of the responseHeader changed to use the same `` structure as the rest of the response. -|=== +The only currently supported version value is `2.2`. The format of the `responseHeader` changed to use the same `` structure as the rest of the response. The default value is the latest supported. @@ -173,17 +166,35 @@ The default mime type for the JSON writer is `application/json`, however this ca This parameter controls the output format of NamedLists, where order is more important than access by name. NamedList is currently used for field faceting data. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +The `json.nl` parameter takes the following values: + -[cols="20,40,40",options="header"] -|=== |json.nl Parameter setting |Example output for `NamedList("a"=1, "bar"="foo", null=3, null=null)` |Description -|flat _(the default)_ |`["a",1, "bar","foo", null,3, null,null]` |NamedList is represented as a flat array, alternating names and values. -|map |`{"a":1, "bar":"foo", "":3, "":null}` |NamedList is represented as a JSON object. Although this is the simplest mapping, a NamedList can have optional keys, repeated keys, and preserves order. Using a JSON object (essentially a map or hash) for a NamedList results in the loss of some information. -|arrarr |`[["a",1], ["bar","foo"], [null,3], [null,null]]` |NamedList is represented as an array of two element arrays. -|arrmap |[`{"a":1}, {"b":2}, 3, null]` |NamedList is represented as an array of JSON objects. -|arrntv |`[{"name":"a","type":"int","value":1}, {"name":"bar","type":"str","value":"foo"}, {"name":null,"type":"int","value":3}, {"name":null,"type":"null","value":null}]` |NamedList is represented as an array of Name Type Value JSON objects. -|=== + +`flat`:: +The default. NamedList is represented as a flat array, alternating names and values. ++ +With input of `NamedList("a"=1, "bar"="foo", null=3, null=null)`, the output would be `["a",1, "bar","foo", null,3, null,null]`. + +`map`:: +NamedList is represented as a JSON object. Although this is the simplest mapping, a NamedList can have optional keys, repeated keys, and preserves order. Using a JSON object (essentially a map or hash) for a NamedList results in the loss of some information. ++ +With input of `NamedList("a"=1, "bar"="foo", null=3, null=null)`, the output would be `{"a":1, "bar":"foo", "":3, "":null}`. + +arrarr:: +NamedList is represented as an array of two element arrays. ++ +With input of `NamedList("a"=1, "bar"="foo", null=3, null=null)`, the output would be `[["a",1], ["bar","foo"], [null,3], [null,null]]`. + +arrmap:: +NamedList is represented as an array of JSON objects. ++ +With input of `NamedList("a"=1, "bar"="foo", null=3, null=null)`, the output would be `[{"a":1}, {"b":2}, 3, null]`. + +arrntv:: +NamedList is represented as an array of Name Type Value JSON objects. ++ +With input of `NamedList("a"=1, "bar"="foo", null=3, null=null)`, the output would be `[{"name":"a","type":"int","value":1}, {"name":"bar","type":"str","value":"foo"}, {"name":null,"type":"int","value":3}, {"name":null,"type":"null","value":null}]`. [[ResponseWriters-json.wrf]] ==== json.wrf @@ -278,11 +289,11 @@ These parameters specify the CSV format that will be returned. You can accept th [width="50%",options="header",] |=== |Parameter |Default Value -|csv.encapsulator |" +|csv.encapsulator |`"` |csv.escape |None -|csv.separator |, -|csv.header |Defaults to true. If false, Solr does not print the column headers -|csv.newline |\n +|csv.separator |`,` +|csv.header |Defaults to `true`. If `false`, Solr does not print the column headers. +|csv.newline |`\n` |csv.null |Defaults to a zero length string. Use this parameter when a document has no value for a particular field. |=== @@ -295,8 +306,8 @@ These parameters specify how multi-valued fields are encoded. Per-field override |=== |Parameter |Default Value |csv.mv.encapsulator |None -|csv.mv.escape |\ -|csv.mv.separator |Defaults to the `csv.separator` value +|csv.mv.escape |`\` +|csv.mv.separator |Defaults to the `csv.separator` value. |=== [[ResponseWriters-Example]] diff --git a/solr/solr-ref-guide/src/result-clustering.adoc b/solr/solr-ref-guide/src/result-clustering.adoc index c4fa048366b..db9a43ce608 100644 --- a/solr/solr-ref-guide/src/result-clustering.adoc +++ b/solr/solr-ref-guide/src/result-clustering.adoc @@ -209,31 +209,34 @@ An example configuration could look as shown below. [[ResultClustering-ConfigurationParametersoftheClusteringComponent]] === Configuration Parameters of the Clustering Component -The table below summarizes parameters of each clustering engine or the entire clustering component (depending where they are declared). +The following parameters of each clustering engine or the entire clustering component (depending where they are declared) are available. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`clustering`:: +When `true`, clustering component is enabled. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|`clustering` |When `true`, clustering component is enabled. -|`clustering.engine` |Declares which clustering engine to use. If not present, the first declared engine will become the default one. -|`clustering.results` |When `true`, the component will perform clustering of search results (this should be enabled). -|`clustering.collection` |When `true`, the component will perform clustering of the whole document index (this section does not cover full-index clustering). -|=== +`clustering.engine`:: +Declares which clustering engine to use. If not present, the first declared engine will become the default one. + +`clustering.results`:: +When `true`, the component will perform clustering of search results (this should be enabled). + +`clustering.collection`:: +When `true`, the component will perform clustering of the whole document index (this section does not cover full-index clustering). At the engine declaration level, the following parameters are supported. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`carrot.algorithm`:: +The algorithm class. + +`carrot.resourcesDir`:: +Algorithm-specific resources and configuration files (stop words, other lexical resources, default settings). By default points to `conf/clustering/carrot2/` + +`carrot.outputSubClusters`:: +If `true` and the algorithm supports hierarchical clustering, sub-clusters will also be emitted. Default value: true. + +`carrot.numDescriptions`:: +Maximum number of per-cluster labels to return (if the algorithm assigns more than one label to a cluster). -[cols="30,70",options="header"] -|=== -|Parameter |Description -|`carrot.algorithm` |The algorithm class. -|`carrot.resourcesDir` |Algorithm-specific resources and configuration files (stop words, other lexical resources, default settings). By default points to `conf/clustering/carrot2/` -|`carrot.outputSubClusters` |If `true` and the algorithm supports hierarchical clustering, sub-clusters will also be emitted. Default value: true. -|`carrot.numDescriptions` |Maximum number of per-cluster labels to return (if the algorithm assigns more than one label to a cluster). -|=== The `carrot.algorithm` parameter should contain a fully qualified class name of an algorithm supported by the http://project.carrot2.org[Carrot2] framework. Currently, the following algorithms are available: @@ -255,30 +258,27 @@ The question of which algorithm to choose depends on the amount of traffic (STC The clustering engine can apply clustering to the full content of (stored) fields or it can run an internal highlighter pass to extract context-snippets before clustering. Highlighting is recommended when the logical snippet field contains a lot of content (this would affect clustering performance). Highlighting can also increase the quality of clustering because the content passed to the algorithm will be more focused around the query (it will be query-specific context). The following parameters control the internal highlighter. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`carrot.produceSummary`:: +When `true` the clustering component will run a highlighter pass on the content of logical fields pointed to by `carrot.title` and `carrot.snippet`. Otherwise full content of those fields will be clustered. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|`carrot.produceSummary` |When `true` the clustering component will run a highlighter pass on the content of logical fields pointed to by `carrot.title` and `carrot.snippet`. Otherwise full content of those fields will be clustered. -|`carrot.fragSize` |The size, in characters, of the snippets (aka fragments) created by the highlighter. If not specified, the default highlighting fragsize (`hl.fragsize`) will be used. -|`carrot.summarySnippets` |The number of summary snippets to generate for clustering. If not specified, the default highlighting snippet count (`hl.snippets`) will be used. -|=== +`carrot.fragSize`:: +The size, in characters, of the snippets (aka fragments) created by the highlighter. If not specified, the default highlighting fragsize (`hl.fragsize`) will be used. + +`carrot.summarySnippets`:: The number of summary snippets to generate for clustering. If not specified, the default highlighting snippet count (`hl.snippets`) will be used. [[ResultClustering-LogicaltoDocumentFieldMapping]] === Logical to Document Field Mapping As already mentioned in <>, the clustering component clusters "documents" consisting of logical parts that need to be mapped onto physical schema of data stored in Solr. The field mapping attributes provide a connection between fields and logical document parts. Note that the content of title and snippet fields must be *stored* so that it can be retrieved at search time. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`carrot.title`:: +The field (alternatively comma- or space-separated list of fields) that should be mapped to the logical document's title. The clustering algorithms typically give more weight to the content of the title field compared to the content (snippet). For best results, the field should contain concise, noise-free content. If there is no clear title in your data, you can leave this parameter blank. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|`carrot.title` |The field (alternatively comma- or space-separated list of fields) that should be mapped to the logical document's title. The clustering algorithms typically give more weight to the content of the title field compared to the content (snippet). For best results, the field should contain concise, noise-free content. If there is no clear title in your data, you can leave this parameter blank. -|`carrot.snippet` |The field (alternatively comma- or space-separated list of fields) that should be mapped to the logical document's main content. If this mapping points to very large content fields the performance of clustering may drop significantly. An alternative then is to use query-context snippets for clustering instead of full field content. See the description of the `carrot.produceSummary` parameter for details. -|`carrot.url` |The field that should be mapped to the logical document's content URL. Leave blank if not required. -|=== +`carrot.snippet`:: +The field (alternatively comma- or space-separated list of fields) that should be mapped to the logical document's main content. If this mapping points to very large content fields the performance of clustering may drop significantly. An alternative then is to use query-context snippets for clustering instead of full field content. See the description of the `carrot.produceSummary` parameter for details. + +`carrot.url`:: +The field that should be mapped to the logical document's content URL. Leave blank if not required. [[ResultClustering-ClusteringMultilingualContent]] === Clustering Multilingual Content @@ -287,14 +287,11 @@ The field mapping specification can include a `carrot.lang` parameter, which def The language hint makes it easier for clustering algorithms to separate documents from different languages on input and to pick the right language resources for clustering. If you do have multi-lingual query results (or query results in a language different than English), it is strongly advised to map the language field appropriately. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`carrot.lang`:: +The field that stores ISO 639-1 code of the language of the document's text fields. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|`carrot.lang` |The field that stores ISO 639-1 code of the language of the document's text fields. -|`carrot.lcmap` |A mapping of arbitrary strings into ISO 639 two-letter codes used by `carrot.lang`. The syntax of this parameter is the same as `langid.map.lcmap`, for example: `langid.map.lcmap=japanese:ja polish:pl english:en` -|=== +`carrot.lcmap`:: +A mapping of arbitrary strings into ISO 639 two-letter codes used by `carrot.lang`. The syntax of this parameter is the same as `langid.map.lcmap`, for example: `langid.map.lcmap=japanese:ja polish:pl english:en` The default language can also be set using Carrot2-specific algorithm attributes (in this case the http://doc.carrot2.org/#section.attribute.lingo.MultilingualClustering.defaultLanguage[MultilingualClustering.defaultLanguage] attribute). diff --git a/solr/solr-ref-guide/src/result-grouping.adoc b/solr/solr-ref-guide/src/result-grouping.adoc index 72a79a1abe4..89b3c339e56 100644 --- a/solr/solr-ref-guide/src/result-grouping.adoc +++ b/solr/solr-ref-guide/src/result-grouping.adoc @@ -59,44 +59,65 @@ If you ask Solr to group these documents by "product_range", then the total amou Result Grouping takes the following request parameters. Any number of these request parameters can be included in a single request: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`group`:: +If `true`, query results will be grouped. -[cols="20,20,60",options="header"] -|=== -|Parameter |Type |Description -|group |Boolean |If true, query results will be grouped. -|group.field |string |The name of the field by which to group results. The field must be single-valued, and either be indexed or a field type that has a value source and works in a function query, such as `ExternalFileField`. It must also be a string-based field, such as `StrField` or `TextField` -|group.func |query a| +`group.field`:: +The name of the field by which to group results. The field must be single-valued, and either be indexed or a field type that has a value source and works in a function query, such as `ExternalFileField`. It must also be a string-based field, such as `StrField` or `TextField` + +`group.func`:: Group based on the unique values of a function query. - ++ NOTE: This option does not work with <>. -|group.query |query |Return a single group of documents that match the given query. -|rows |integer |The number of groups to return. The default value is 10. -|start |integer |Specifies an initial offset for the list of groups. -|group.limit |integer |Specifies the number of results to return for each group. The default value is 1. -|group.offset |integer |Specifies an initial offset for the document list of each group. -|sort |sortspec |Specifies how Solr sorts the groups relative to each other. For example, `sort=popularity desc` will cause the groups to be sorted according to the highest popularity document in each group. The default value is `score desc`. -|group.sort |sortspec |Specifies how Solr sorts documents within each group. The default behavior if `group.sort` is not specified is to use the same effective value as the `sort` parameter. -|group.format |grouped/simple |If this parameter is set to `simple`, the grouped documents are presented in a single flat list, and the `start` and `rows` parameters affect the numbers of documents instead of groups. -|group.main |Boolean |If true, the result of the first field grouping command is used as the main result list in the response, using `group.format=simple`. -|group.ngroups |Boolean a| -If true, Solr includes the number of groups that have matched the query in the results. The default value is false. +`group.query`:: +Return a single group of documents that match the given query. +`rows`:: +The number of groups to return. The default value is `10`. + +`start`:: +Specifies an initial offset for the list of groups. + +`group.limit`:: +Specifies the number of results to return for each group. The default value is `1`. + +`group.offset`:: +Specifies an initial offset for the document list of each group. + +`sort`:: +Specifies how Solr sorts the groups relative to each other. For example, `sort=popularity desc` will cause the groups to be sorted according to the highest popularity document in each group. The default value is `score desc`. + +`group.sort`:: +Specifies how Solr sorts documents within each group. The default behavior if `group.sort` is not specified is to use the same effective value as the `sort` parameter. + +`group.format`:: +If this parameter is set to `simple`, the grouped documents are presented in a single flat list, and the `start` and `rows` parameters affect the numbers of documents instead of groups. An alternate value for this parameter is `grouped`. + +`group.main`:: +If `true`, the result of the first field grouping command is used as the main result list in the response, using `group.format=simple`. + +`group.ngroups`:: +If `true`, Solr includes the number of groups that have matched the query in the results. The default value is false. ++ See below for <> when using sharded indexes -|group.truncate |Boolean |If true, facet counts are based on the most relevant document of each group matching the query. The default value is false. -|group.facet |Boolean a| -Determines whether to compute grouped facets for the field facets specified in facet.field parameters. Grouped facets are computed based on the first specified group. As with normal field faceting, fields shouldn't be tokenized (otherwise counts are computed for each token). Grouped faceting supports single and multivalued fields. Default is false. +`group.truncate`:: +If `true`, facet counts are based on the most relevant document of each group matching the query. The default value is `false`. -*Warning*: There can be a heavy performance cost to this option. +`group.facet`:: +Determines whether to compute grouped facets for the field facets specified in facet.field parameters. Grouped facets are computed based on the first specified group. As with normal field faceting, fields shouldn't be tokenized (otherwise counts are computed for each token). Grouped faceting supports single and multivalued fields. Default is `false`. ++ +WARNING: There can be a heavy performance cost to this option. ++ +See below for <> when using sharded indexes. -See below for <> when using sharded indexes +`group.cache.percent`:: +Setting this parameter to a number greater than 0 enables caching for result grouping. Result Grouping executes two searches; this option caches the second search. The default value is `0`. The maximum value is `100`. ++ +Testing has shown that group caching only improves search time with Boolean, wildcard, and fuzzy queries. For simple queries like term or "match all" queries, group caching degrades performance. -|group.cache.percent |integer between 0 and 100 |Setting this parameter to a number greater than 0 enables caching for result grouping. Result Grouping executes two searches; this option caches the second search. The default value is 0. Testing has shown that group caching only improves search time with Boolean, wildcard, and fuzzy queries. For simple queries like term or "match all" queries, group caching degrades performance. -|=== - -Any number of group commands (`group.field`, `group.func`, `group.query`) may be specified in a single request. +Any number of group commands (e.g., `group.field`, `group.func`, `group.query`, etc.) may be specified in a single request. [[ResultGrouping-Examples]] == Examples diff --git a/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc b/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc index df377819de5..5b548f2e77d 100644 --- a/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc +++ b/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc @@ -132,35 +132,35 @@ Permissions need to be created if they are not on the list of pre-defined permis Several properties can be used to define your custom permission. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`name`:: +The name of the permission. This is required only if it is a predefined permission. -[cols="30,70",options="header"] -|=== -|Property |Description -|name |The name of the permission. This is required only if it is a predefined permission. -|collection a| +`collection`:: The collection or collections the permission will apply to. ++ +When the path that will be allowed is collection-specific, such as when setting permissions to allow use of the Schema API, omitting the collection property will allow the defined path and/or method for all collections. However, when the path is one that is non-collection-specific, such as the Collections API, the collection value must be `null`. The default value is * (all collections). -When the path that will be allowed is collection-specific, such as when setting permissions to allow useof the Schema API, omitting the collection property will allow the defined path and/or method for all collections. However, when the path is one that is non-collection-specific, such as the Collections API, the collection value must be `null`. The default value is * (all collections). +`path`:: +A request handler name, such as `/update` or `/select`. A wild card is supported, to allow for all paths as appropriate (such as, `/update/*`). -|path |A request handler name, such as `/update` or `/select`. A wild card is supported, to allow for all paths as appropriate (such as, `/update/*`). -|method |HTTP methods that are allowed for this permission. You could allow only GET requests, or have a role that allows PUT and POST requests. The method values that are allowed for this property are GET, POST, PUT,DELETEand HEAD. -|params a| +`method`:: HTTP methods that are allowed for this permission. You could allow only GET requests, or have a role that allows PUT and POST requests. The method values that are allowed for this property are GET, POST, PUT,DELETE and HEAD. + +`params`:: The names and values of request parameters. This property can be omitted if all request parameters are to be matched, but will restrict access only to the values provided if defined. - ++ For example, this property could be used to limit the actions a role is allowed to perform with the Collections API. If the role should only be allowed to perform the LIST or CLUSTERSTATUS requests, you would define this as follows: - ++ [source,json] ---- "params": { "action": ["LIST", "CLUSTERSTATUS"] } ---- - ++ The value of the parameter can be a simple string or it could be a regular expression. Use the prefix `REGEX:` to use a regular expression match instead of a string identity match - ++ If the commands LIST and CLUSTERSTATUS are case insensitive, the above example should be as follows - ++ [source,json] ---- "params": { @@ -168,9 +168,11 @@ If the commands LIST and CLUSTERSTATUS are case insensitive, the above example s } ---- -|before |This property allows ordering of permissions. The value of this property is the index of the permission that this new permission should be placed before in `security.json`. The index is automatically assigned in the order they are created -|role |The name of the role(s) to give this permission. This name will be used to map user IDs to the role to grant these permissions. The value can be wildcard such as (`*`), which means that any user is OK, but no user is NOT OK. -|=== +`before`:: +This property allows ordering of permissions. The value of this property is the index of the permission that this new permission should be placed before in `security.json`. The index is automatically assigned in the order they are created. + +`role`:: +The name of the role(s) to give this permission. This name will be used to map user IDs to the role to grant these permissions. The value can be wildcard such as (`*`), which means that any user is OK, but no user is NOT OK. The following creates a new permission named "collection-mgr" that is allowed to create and list collections. The permission will be placed before the "read" permission. Note also that we have defined "collection as `null`, this is because requests to the Collections API are never collection-specific. diff --git a/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc b/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc index e23431cdfb9..9f8e2dc300b 100644 --- a/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc +++ b/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc @@ -103,73 +103,59 @@ The `HdfsDirectoryFactory` has a number of settings that are defined as part of [[RunningSolronHDFS-SolrHDFSSettings]] === Solr HDFS Settings -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed - -[cols="20,30,10,40",options="header"] -|=== -|Parameter |Example Value |Default |Description -|`solr.hdfs.home` |`hdfs://host:port/path/solr` |N/A |A root location in HDFS for Solr to write collection data to. Rather than specifying an HDFS location for the data directory or update log directory, use this to specify one root location and have everything automatically created within this HDFS location. -|=== +`solr.hdfs.home`:: +A root location in HDFS for Solr to write collection data to. Rather than specifying an HDFS location for the data directory or update log directory, use this to specify one root location and have everything automatically created within this HDFS location. The structure of this parameter is `hdfs://host:port/path/solr`. [[RunningSolronHDFS-BlockCacheSettings]] === Block Cache Settings -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`solr.hdfs.blockcache.enabled`:: +Enable the blockcache. The default is `true`. -[cols="30,10,60",options="header"] -|=== -|Parameter |Default |Description -|`solr.hdfs.blockcache.enabled` |true |Enable the blockcache -|`solr.hdfs.blockcache.read.enabled` |true |Enable the read cache -|`solr.hdfs.blockcache.direct.memory.allocation` |true |Enable direct memory allocation. If this is false, heap is used -|`solr.hdfs.blockcache.slab.count` |1 |Number of memory slabs to allocate. Each slab is 128 MB in size. -|`solr.hdfs.blockcache.global` |true |Enable/Disable using one global cache for all SolrCores. The settings used will be from the first HdfsDirectoryFactory created. -|=== +`solr.hdfs.blockcache.read.enabled`:: +Enable the read cache. The default is `true`. + +`solr.hdfs.blockcache.direct.memory.allocation`:: +Enable direct memory allocation. If this is `false`, heap is used. The default is `true`. + +`solr.hdfs.blockcache.slab.count`:: +Number of memory slabs to allocate. Each slab is 128 MB in size. The default is `1`. + +`solr.hdfs.blockcache.global`:: +Enable/Disable using one global cache for all SolrCores. The settings used will be from the first HdfsDirectoryFactory created. The default is `true`. [[RunningSolronHDFS-NRTCachingDirectorySettings]] === NRTCachingDirectory Settings -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`solr.hdfs.nrtcachingdirectory.enable`:: true | +Enable the use of NRTCachingDirectory. The default is `true`. -[cols="30,10,60",options="header"] -|=== -|Parameter |Default |Description -|`solr.hdfs.nrtcachingdirectory.enable` |true |Enable the use of NRTCachingDirectory -|`solr.hdfs.nrtcachingdirectory.maxmergesizemb` |16 |NRTCachingDirectory max segment size for merges -|`solr.hdfs.nrtcachingdirectory.maxcachedmb` |192 |NRTCachingDirectory max cache size -|=== +`solr.hdfs.nrtcachingdirectory.maxmergesizemb`:: +NRTCachingDirectory max segment size for merges. The default is `16`. + +`solr.hdfs.nrtcachingdirectory.maxcachedmb`:: +NRTCachingDirectory max cache size. The default is `192`. [[RunningSolronHDFS-HDFSClientConfigurationSettings]] === HDFS Client Configuration Settings -solr.hdfs.confdir pass the location of HDFS client configuration files - needed for HDFS HA for example. - -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed - -[cols="30,10,60",options="header"] -|=== -|Parameter |Default |Description -|`solr.hdfs.confdir` |N/A |Pass the location of HDFS client configuration files - needed for HDFS HA for example. -|=== +`solr.hdfs.confdir`:: +Pass the location of HDFS client configuration files - needed for HDFS HA for example. [[RunningSolronHDFS-KerberosAuthenticationSettings]] === Kerberos Authentication Settings Hadoop can be configured to use the Kerberos protocol to verify user identity when trying to access core services like HDFS. If your HDFS directories are protected using Kerberos, then you need to configure Solr's HdfsDirectoryFactory to authenticate using Kerberos in order to read and write to HDFS. To enable Kerberos authentication from Solr, you need to set the following parameters: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`solr.hdfs.security.kerberos.enabled`:: false |Set to `true` to enable Kerberos authentication. The default is `false`. -[cols="30,10,60",options="header"] -|=== -|Parameter |Default |Description -|`solr.hdfs.security.kerberos.enabled` |false |Set to true to enable Kerberos authentication -|`solr.hdfs.security.kerberos.keytabfile` |N/A a| +`solr.hdfs.security.kerberos.keytabfile`:: A keytab file contains pairs of Kerberos principals and encrypted keys which allows for password-less authentication when Solr attempts to authenticate with secure Hadoop. - ++ This file will need to be present on all Solr servers at the same path provided in this parameter. -|`solr.hdfs.security.kerberos.principal` |N/A |The Kerberos principal that Solr should use to authenticate to secure Hadoop; the format of a typical Kerberos V5 principal is: `primary/instance@realm` -|=== +`solr.hdfs.security.kerberos.principal`:: +The Kerberos principal that Solr should use to authenticate to secure Hadoop; the format of a typical Kerberos V5 principal is: `primary/instance@realm`. [[RunningSolronHDFS-Example]] == Example @@ -210,20 +196,19 @@ One benefit to running Solr in HDFS is the ability to automatically add new repl Collections created using `autoAddReplicas=true` on a shared file system have automatic addition of replicas enabled. The following settings can be used to override the defaults in the `` section of `solr.xml`. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`autoReplicaFailoverWorkLoopDelay`:: +The time (in ms) between clusterstate inspections by the Overseer to detect and possibly act on creation of a replacement replica. The default is `10000`. -[cols="40,10,50",options="header"] -|=== -|Param |Default |Description -|autoReplicaFailoverWorkLoopDelay |10000 |The time (in ms) between clusterstate inspections by the Overseer to detect and possibly act on creation of a replacement replica. -|autoReplicaFailoverWaitAfterExpiration |30000 |The minimum time (in ms) to wait for initiating replacement of a replica after first noticing it not being live. This is important to prevent false positives while stoping or starting the cluster. -|autoReplicaFailoverBadNodeExpiration |60000 |The delay (in ms) after which a replica marked as down would be unmarked. -|=== +`autoReplicaFailoverWaitAfterExpiration`:: +The minimum time (in ms) to wait for initiating replacement of a replica after first noticing it not being live. This is important to prevent false positives while stoping or starting the cluster. The default is `30000`. + +`autoReplicaFailoverBadNodeExpiration`:: +The delay (in ms) after which a replica marked as down would be unmarked. The default is `60000`. [[RunningSolronHDFS-TemporarilydisableautoAddReplicasfortheentirecluster]] -=== Temporarily disable autoAddReplicas for the entire cluster +=== Temporarily Disable autoAddReplicas for the Entire Cluster -When doing offline maintenance on the cluster and for various other use cases where an admin would like to temporarily disable auto addition of replicas, the following APIs will disable and re-enable autoAddReplicas for **all collections in the cluster**: +When doing offline maintenance on the cluster and for various other use cases where an admin would like to temporarily disable auto addition of replicas, the following APIs will disable and re-enable autoAddReplicas for *all collections in the cluster*: Disable auto addition of replicas cluster wide by setting the cluster property `autoAddReplicas` to `false`: diff --git a/solr/solr-ref-guide/src/spatial-search.adoc b/solr/solr-ref-guide/src/spatial-search.adoc index 69d130514a3..8b56c022f23 100644 --- a/solr/solr-ref-guide/src/spatial-search.adoc +++ b/solr/solr-ref-guide/src/spatial-search.adoc @@ -66,44 +66,46 @@ If you'd rather use a standard industry format, Solr supports WKT and GeoJSON. H There are two spatial Solr "query parsers" for geospatial search: `geofilt` and `bbox`. They take the following parameters: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`d`:: +The radial distance, usually in kilometers. RPT & BBoxField can set other units via the setting `distanceUnits`. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|d |the radial distance, usually in kilometers. (RPT & BBoxField can set other units via the setting `distanceUnits`) -|pt |the center point using the format "lat,lon" if latitude & longitude. Otherwise, "x,y" for PointType or "x y" for RPT field types. -|sfield |a spatial indexed field -|score a| +`pt`:: +The center point using the format "lat,lon" if latitude & longitude. Otherwise, "x,y" for PointType or "x y" for RPT field types. + +`sfield`:: +A spatial indexed field. + +`score`:: (Advanced option; not supported by LatLonType (deprecated) or PointType) If the query is used in a scoring context (e.g. as the main query in `q`), this _<>_ determines what scores will be produced. Valid values are: -* `none` - A fixed score of 1.0. (the default) -* `kilometers` - distance in kilometers between the field value and the specified center point -* `miles` - distance in miles between the field value and the specified center point -* `degrees` - distance in degrees between the field value and the specified center point -* `distance` - distance between the field value and the specified center point in the `distanceUnits` configured for this field -* `recipDistance` - 1 / the distance - +* `none`: A fixed score of 1.0. (the default) +* `kilometers`: distance in kilometers between the field value and the specified center point +* `miles`: distance in miles between the field value and the specified center point +* `degrees`: distance in degrees between the field value and the specified center point +* `distance`: distance between the field value and the specified center point in the `distanceUnits` configured for this field +* `recipDistance`: 1 / the distance ++ [WARNING] ==== Don't use this for indexed non-point shapes (e.g. polygons). The results will be erroneous. And with RPT, it's only recommended for multi-valued point data, as the implementation doesn't scale very well and for single-valued fields, you should instead use a separate non-RPT field purely for distance sorting. ==== - ++ When used with `BBoxField`, additional options are supported: ++ +* `overlapRatio`: The relative overlap between the indexed shape & query shape. +* `area`: haversine based area of the overlapping shapes expressed in terms of the `distanceUnits` configured for this field +* `area2D`: cartesian coordinates based area of the overlapping shapes expressed in terms of the `distanceUnits` configured for this field -* `overlapRatio` - The relative overlap between the indexed shape & query shape. -* `area` - haversine based area of the overlapping shapes expressed in terms of the `distanceUnits` configured for this field -* `area2D` - cartesian coordinates based area of the overlapping shapes expressed in terms of the `distanceUnits` configured for this field +`filter`:: +(Advanced option; not supported by LatLonType (deprecated) or PointType). If you only want the query to score (with the above `score` local parameter), not filter, then set this local parameter to false. -|filter |(Advanced option; not supported by LatLonType (deprecated) or PointType). If you only want the query to score (with the above `score` local parameter), not filter, then set this local parameter to false. -|=== [[SpatialSearch-geofilt]] === geofilt The `geofilt` filter allows you to retrieve results based on the geospatial distance (AKA the "great circle distance") from a given point. Another way of looking at it is that it creates a circular shape filter. For example, to find all documents within five kilometers of a given lat/lon point, you could enter `&q=*:*&fq={!geofilt sfield=store}&pt=45.15,-93.85&d=5`. This filter returns all results within a circle of the given radius around the initial point: -image::images/spatial-search/circle.png[image] +image::images/spatial-search/circle.png[5KM radius] [[SpatialSearch-bbox]] @@ -117,8 +119,7 @@ Here's a sample query: The rectangular shape is faster to compute and so it's sometimes used as an alternative to `geofilt` when it's acceptable to return points outside of the radius. However, if the ideal goal is a circle but you want it to run faster, then instead consider using the RPT field and try a large `distErrPct` value like `0.1` (10% radius). This will return results outside the radius but it will do so somewhat uniformly around the shape. -image::images/spatial-search/bbox.png[image] - +image::images/spatial-search/bbox.png[Bounding box] [IMPORTANT] ==== @@ -148,7 +149,6 @@ If you know the filter query (be it spatial or not) is fairly unique and not lik LLPSF does not support Solr's "PostFilter". - [[SpatialSearch-DistanceSortingorBoosting_FunctionQueries_]] == Distance Sorting or Boosting (Function Queries) @@ -220,32 +220,51 @@ RPT _shares_ various features in common with `LatLonPointSpatialField`. Some are To use RPT, the field type must be registered and configured in `schema.xml`. There are many options for this field type. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`name`:: +The name of the field type. -[cols="30,70",options="header"] -|=== -|Setting |Description -|name |The name of the field type. -|class |This should be `solr.SpatialRecursivePrefixTreeFieldType`. But be aware that the Lucene spatial module includes some other so-called "spatial strategies" other than RPT, notably TermQueryPT*, BBox, PointVector*, and SerializedDV. Solr requires a field type to parallel these in order to use them. The asterisked ones have them. -|spatialContextFactory |This is a Java class name to an internal extension point governing support for shape definitions & parsing. If you require polygon support, set this to `JTS` – an alias for `org.locationtech.spatial4j.context.jts.JtsSpatialContextFactory`; otherwise it can be omitted. See important info below about JTS. (note: prior to Solr 6, the "org.locationtech.spatial4j" part was "com.spatial4j.core" and there used to be no convenience JTS alias) -|geo |If **true**, the default, latitude and longitude coordinates will be used and the mathematical model will generally be a sphere. If false, the coordinates will be generic X & Y on a 2D plane using Euclidean/Cartesian geometry. -|format |Defines the shape syntax/format to be used. Defaults to `WKT` but `GeoJSON` is another popular format. Spatial4j governs this feature and supports https://locationtech.github.io/spatial4j/apidocs/org/locationtech/spatial4j/io/package-frame.html[other formats]. If a given shape is parseable as "lat,lon" or "x y" then that is always supported. -|distanceUnits a| -This is used to specify the units for distance measurements used throughout the use of this field. This can be `degrees`, `kilometers` or `miles`. It is applied to nearly all distance measurements involving the field: `maxDistErr`, `distErr`, `d`, `geodist` and the `score` when score is `distance`, `area`, or `area2d`. However, it doesn't affect distances embedded in WKT strings, (eg: "`BUFFER(POINT(200 10),0.2)`"), which are still in degrees. +`class`:: +This should be `solr.SpatialRecursivePrefixTreeFieldType`. But be aware that the Lucene spatial module includes some other so-called "spatial strategies" other than RPT, notably TermQueryPT*, BBox, PointVector*, and SerializedDV. Solr requires a field type to parallel these in order to use them. The asterisked ones have them. -`distanceUnits` defaults to either "```kilometers```" if `geo` is "```true```", or "```degrees```" if `geo` is "```false```". +`spatialContextFactory`:: +This is a Java class name to an internal extension point governing support for shape definitions & parsing. If you require polygon support, set this to `JTS` – an alias for `org.locationtech.spatial4j.context.jts.JtsSpatialContextFactory`; otherwise it can be omitted. See important info below about JTS. (note: prior to Solr 6, the "org.locationtech.spatial4j" part was "com.spatial4j.core" and there used to be no convenience JTS alias) +`geo`:: +If `true`, the default, latitude and longitude coordinates will be used and the mathematical model will generally be a sphere. If `false`, the coordinates will be generic X & Y on a 2D plane using Euclidean/Cartesian geometry. + +`format`:: Defines the shape syntax/format to be used. Defaults to `WKT` but `GeoJSON` is another popular format. Spatial4j governs this feature and supports https://locationtech.github.io/spatial4j/apidocs/org/locationtech/spatial4j/io/package-frame.html[other formats]. If a given shape is parseable as "lat,lon" or "x y" then that is always supported. + +`distanceUnits`:: a| +This is used to specify the units for distance measurements used throughout the use of this field. This can be `degrees`, `kilometers` or `miles`. It is applied to nearly all distance measurements involving the field: `maxDistErr`, `distErr`, `d`, `geodist` and the `score` when score is `distance`, `area`, or `area2d`. However, it doesn't affect distances embedded in WKT strings, (e.g., `BUFFER(POINT(200 10),0.2)`), which are still in degrees. ++ +`distanceUnits` defaults to either `kilometers` if `geo` is true`, or `degrees` if `geo` is `false`. ++ `distanceUnits` replaces the `units` attribute; which is now deprecated and mutually exclusive with this attribute. -|distErrPct |Defines the default precision of non-point shapes (both index & query), as a fraction between 0.0 (fully precise) to 0.5. The closer this number is to zero, the more accurate the shape will be. However, more precise indexed shapes use more disk space and take longer to index. Bigger distErrPct values will make queries faster but less accurate. At query time this can be overridden in the query syntax, such as to 0.0 so as to not approximate the search shape. The default for the RPT field is 0.025. Note: For RPTWithGeometrySpatialField (see below), there's always complete accuracy with the serialized geometry and so this doesn't control accuracy so much as it controls the trade-off of how big the index should be. distErrPct defaults to 0.15 for that field. -|maxDistErr |Defines the highest level of detail required for indexed data. If left blank, the default is one meter – just a bit less than 0.000009 degrees. This setting is used internally to compute an appropriate maxLevels (see below). -|worldBounds |Defines the valid numerical ranges for x and y, in the format of `ENVELOPE(minX, maxX, maxY, minY)`. If `geo="true"`, the standard lat-lon world boundaries are assumed. If `geo=false`, you should define your boundaries. -|distCalculator |Defines the distance calculation algorithm. If `geo=true`, "haversine" is the default. If `geo=false`, "cartesian" will be the default. Other possible values are "lawOfCosines", "vincentySphere" and "cartesian^2". -|prefixTree |Defines the spatial grid implementation. Since a PrefixTree (such as RecursivePrefixTree) maps the world as a grid, each grid cell is decomposed to another set of grid cells at the next level. If `geo=true` then the default prefix tree is "```geohash```", otherwise it's "```quad```". Geohash has 32 children at each level, quad has 4. Geohash can only be used for `geo=true` as it's strictly geospatial. A third choice is "```packedQuad```", which is generally more efficient than plain "quad", provided there are many levels -- perhaps 20 or more. -|maxLevels |Sets the maximum grid depth for indexed data. Instead, it's usually more intuitive to compute an appropriate maxLevels by specifying `maxDistErr` . -|=== +`distErrPct`:: +Defines the default precision of non-point shapes (both index & query), as a fraction between `0.0` (fully precise) to `0.5`. The closer this number is to zero, the more accurate the shape will be. However, more precise indexed shapes use more disk space and take longer to index. ++ +Bigger `distErrPct` values will make queries faster but less accurate. At query time this can be overridden in the query syntax, such as to `0.0` so as to not approximate the search shape. The default for the RPT field is `0.025`. ++ +NOTE: For RPTWithGeometrySpatialField (see below), there's always complete accuracy with the serialized geometry and so this doesn't control accuracy so much as it controls the trade-off of how big the index should be. distErrPct defaults to 0.15 for that field. -*_And there are others:_* `normWrapLongitude` _,_ `datelineRule`, `validationRule`, `autoIndex`, `allowMultiOverlap`, `precisionModel`. For further info, see notes below about `spatialContextFactory` implementations referenced above, especially the link to the JTS based one. +`maxDistErr`:: Defines the highest level of detail required for indexed data. If left blank, the default is one meter – just a bit less than 0.000009 degrees. This setting is used internally to compute an appropriate maxLevels (see below). + +`worldBounds`:: +Defines the valid numerical ranges for x and y, in the format of `ENVELOPE(minX, maxX, maxY, minY)`. If `geo="true"`, the standard lat-lon world boundaries are assumed. If `geo=false`, you should define your boundaries. + +`distCalculator`:: +Defines the distance calculation algorithm. If `geo=true`, "haversine" is the default. If `geo=false`, "cartesian" will be the default. Other possible values are "lawOfCosines", "vincentySphere" and "cartesian^2". + +`prefixTree`:: Defines the spatial grid implementation. Since a PrefixTree (such as RecursivePrefixTree) maps the world as a grid, each grid cell is decomposed to another set of grid cells at the next level. ++ +If `geo=true` then the default prefix tree is `geohash`, otherwise it's `quad`. Geohash has 32 children at each level, quad has 4. Geohash can only be used for `geo=true` as it's strictly geospatial. ++ +A third choice is `packedQuad`, which is generally more efficient than `quad`, provided there are many levels -- perhaps 20 or more. + +`maxLevels`:: Sets the maximum grid depth for indexed data. Instead, it's usually more intuitive to compute an appropriate maxLevels by specifying `maxDistErr` . + +*_And there are others:_* `normWrapLongitude`, `datelineRule`, `validationRule`, `autoIndex`, `allowMultiOverlap`, `precisionModel`. For further info, see notes below about `spatialContextFactory` implementations referenced above, especially the link to the JTS based one. [[SpatialSearch-JTSandPolygons]] === JTS and Polygons @@ -304,23 +323,30 @@ The RPT field supports generating a 2D grid of facet counts for documents having The heatmap feature is accessed from Solr's faceting feature. As a part of faceting, it supports the `key` local parameter as well as excluding tagged filter queries, just like other types of faceting do. This allows multiple heatmaps to be returned on the same field with different filters. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`facet`:: +Set to `true` to enable faceting. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|facet |Set to `true` to enable faceting -|facet.heatmap |The field name of type RPT -|facet.heatmap.geom |The region to compute the heatmap on, specified using the rectangle-range syntax or WKT. It defaults to the world. ex: `["-180 -90" TO "180 90"]` -|facet.heatmap.gridLevel |A specific grid level, which determines how big each grid cell is. Defaults to being computed via distErrPct (or distErr) -|facet.heatmap.distErrPct |A fraction of the size of geom used to compute gridLevel. Defaults to 0.15. It's computed the same as a similarly named parameter for RPT. -|facet.heatmap.distErr |A cell error distance used to pick the grid level indirectly. It's computed the same as a similarly named parameter for RPT. -|facet.heatmap.format |The format, either `ints2D` (default) or `png`. -|=== +`facet.heatmap`:: +The field name of type RPT. + +`facet.heatmap.geom`:: +The region to compute the heatmap on, specified using the rectangle-range syntax or WKT. It defaults to the world. ex: `["-180 -90" TO "180 90"]`. + +`facet.heatmap.gridLevel`:: +A specific grid level, which determines how big each grid cell is. Defaults to being computed via `distErrPct` (or `distErr`). + +`facet.heatmap.distErrPct`:: +A fraction of the size of geom used to compute gridLevel. Defaults to 0.15. It's computed the same as a similarly named parameter for RPT. + +`facet.heatmap.distErr`:: +A cell error distance used to pick the grid level indirectly. It's computed the same as a similarly named parameter for RPT. + +`facet.heatmap.format`:: +The format, either `ints2D` (default) or `png`. [TIP] ==== -You'll experiment with different distErrPct values (probably 0.10 - 0.20) with various input geometries till the default size is what you're looking for. The specific details of how it's computed isn't important. For high-detail grids used in point-plotting (loosely one cell per pixel), set distErr to be the number of decimal-degrees of several pixels or so of the map being displayed. Also, you probably don't want to use a geohash based grid because the cell orientation between grid levels flip-flops between being square and rectangle. Quad is consistent and has more levels, albeit at the expense of a larger index. +You'll experiment with different `distErrPct` values (probably 0.10 - 0.20) with various input geometries till the default size is what you're looking for. The specific details of how it's computed isn't important. For high-detail grids used in point-plotting (loosely one cell per pixel), set `distErr` to be the number of decimal-degrees of several pixels or so of the map being displayed. Also, you probably don't want to use a geohash-based grid because the cell orientation between grid levels flip-flops between being square and rectangle. Quad is consistent and has more levels, albeit at the expense of a larger index. ==== Here's some sample output in JSON (with "..." inserted for brevity): diff --git a/solr/solr-ref-guide/src/the-query-elevation-component.adoc b/solr/solr-ref-guide/src/the-query-elevation-component.adoc index 9898a082216..dcd3c7e190f 100644 --- a/solr/solr-ref-guide/src/the-query-elevation-component.adoc +++ b/solr/solr-ref-guide/src/the-query-elevation-component.adoc @@ -61,17 +61,16 @@ Optionally, in the Query Elevation Component configuration you can also specify foo ---- -The Query Elevation Search Component takes the following arguments: +The Query Elevation Search Component takes the following parameters: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`queryFieldType`:: +Specifies which fieldType should be used to analyze the incoming text. For example, it may be appropriate to use a fieldType with a LowerCaseFilter. -[cols="30,70",options="header"] -|=== -|Argument |Description -|`queryFieldType` |Specifies which fieldType should be used to analyze the incoming text. For example, it may be appropriate to use a fieldType with a LowerCaseFilter. -|`config-file` |Path to the file that defines query elevation. This file must exist in `/conf/` or `/`. If the file exists in the /conf/ directory it will be loaded once at startup. If it exists in the data directory, it will be reloaded for each IndexReader. -|`forceElevation` |By default, this component respects the requested `sort` parameter: if the request asks to sort by date, it will order the results by date. If `forceElevation=true` (the default), results will first return the boosted docs, then order by date. -|=== +`config-file`:: +Path to the file that defines query elevation. This file must exist in `/conf/` or `/`. If the file exists in the `conf/` directory it will be loaded once at startup. If it exists in the `data/` directory, it will be reloaded for each IndexReader. + +`forceElevation`:: +By default, this component respects the requested `sort` parameter: if the request asks to sort by date, it will order the results by date. If `forceElevation=true` (the default), results will first return the boosted docs, then order by date. [[TheQueryElevationComponent-elevate.xml]] === elevate.xml diff --git a/solr/solr-ref-guide/src/the-stats-component.adoc b/solr/solr-ref-guide/src/the-stats-component.adoc index 96ba88c1fe3..a5eb334a1bf 100644 --- a/solr/solr-ref-guide/src/the-stats-component.adoc +++ b/solr/solr-ref-guide/src/the-stats-component.adoc @@ -32,18 +32,14 @@ bin/solr -e techproducts The Stats Component accepts the following parameters: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`stats`:: +If `true`, then invokes the Stats component. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|stats |If **true**, then invokes the Stats component. -|stats.field a| +`stats.field`:: Specifies a field for which statistics should be generated. This parameter may be invoked multiple times in a query in order to request statistics on multiple fields. - ++ <> may be used to indicate which subset of the supported statistics should be computed, and/or that statistics should be computed over the results of an arbitrary numeric function (or query) instead of a simple field name. See the examples below. -|=== [[TheStatsComponent-Example]] === Example @@ -96,26 +92,47 @@ The query below demonstrates computing stats against two different fields numeri [[TheStatsComponent-StatisticsSupported]] == Statistics Supported -The table below explains the statistics supported by the Stats component. Not all statistics are supported for all field types, and not all statistics are computed by default (See <> below for details) +The table below explains the statistics supported by the Stats component. Not all statistics are supported for all field types, and not all statistics are computed by default (see <> below for details) -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`min`:: +The minimum value of the field/function in all documents in the set. This statistic is computed for all field types and is computed by default. -[cols="10,10,50,20,10",options="header"] -|=== -|Local Param |Sample Input |Description |Supported Types |Computed by Default -|min |true |The minimum value of the field/function in all documents in the set. |All |Yes -|max |true |The maximum value of the field/function in all documents in the set. |All |Yes -|sum |true |The sum of all values of the field/function in all documents in the set. |Numeric & Date |Yes -|count |true |The number of values found in all documents in the set for this field/function. |All |Yes -|missing |true |The number of documents in the set which do not have a value for this field/function. |All |Yes -|sumOfSquares |true |Sum of all values squared (a by product of computing stddev) |Numeric & Date |Yes -|mean |true |The average `(v1 + v2 .... + vN)/N` |Numeric & Date |Yes -|stddev |true |Standard deviation, measuring how widely spread the values in the data set are. |Numeric & Date |Yes -|percentiles |"1,99,99.9" |A list of percentile values based on cut-off points specified by the param value. These values are an approximation, using the https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf[t-digest algorithm]. |Numeric |No -|distinctValues |true |The set of all distinct values for the field/function in all of the documents in the set. This calculation can be very expensive for fields that do not have a tiny cardinality. |All |No -|countDistinct |true |The exact number of distinct values in the field/function in all of the documents in the set. This calculation can be very expensive for fields that do not have a tiny cardinality. |All |No -|cardinality |"true" or"0.3" |A statistical approximation (currently using the https://en.wikipedia.org/wiki/HyperLogLog[HyperLogLog] algorithm) of the number of distinct values in the field/function in all of the documents in the set. This calculation is much more efficient then using the 'countDistinct' option, but may not be 100% accurate. Input for this option can be floating point number between 0.0 and 1.0 indicating how aggressively the algorithm should try to be accurate: 0.0 means use as little memory as possible; 1.0 means use as much memory as needed to be as accurate as possible. 'true' is supported as an alias for "0.3" |All |No -|=== +`max`:: +The maximum value of the field/function in all documents in the set. This statistic is computed for all field types and is computed by default. + +`sum`:: +The sum of all values of the field/function in all documents in the set. This statistic is computed for numeric and date field types and is computed by default. + +`count`:: +The number of values found in all documents in the set for this field/function. This statistic is computed for all field types and is computed by default. + +`missing`:: +The number of documents in the set which do not have a value for this field/function. This statistic is computed for all field types and is computed by default. + +`sumOfSquares`:: +Sum of all values squared (a by product of computing stddev). This statistic is computed for numeric and date field types and is computed by default. + +`mean`:: +The average `(v1 + v2 .... + vN)/N`. This statistic is computed for numeric and date field types and is computed by default. + +`stddev`:: +Standard deviation, measuring how widely spread the values in the data set are. This statistic is computed for numeric and date field types and is computed by default. + +`percentiles`:: +A list of percentile values based on cut-off points specified by the parameter value, such as `1,99,99.9`. These values are an approximation, using the https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf[t-digest algorithm]. This statistic is computed for numeric field types and is not computed by default. + +`distinctValues`:: +The set of all distinct values for the field/function in all of the documents in the set. This calculation can be very expensive for fields that do not have a tiny cardinality. This statistic is computed for all field types but is not computed by default. + +`countDistinct`:: +The exact number of distinct values in the field/function in all of the documents in the set. This calculation can be very expensive for fields that do not have a tiny cardinality. This statistic is computed for all field types but is not computed by default. + +`cardinality`:: +A statistical approximation (currently using the https://en.wikipedia.org/wiki/HyperLogLog[HyperLogLog] algorithm) of the number of distinct values in the field/function in all of the documents in the set. This calculation is much more efficient then using the `countDistinct` option, but may not be 100% accurate. ++ +Input for this option can be floating point number between `0.0` and `1.0` indicating how aggressively the algorithm should try to be accurate: `0.0` means use as little memory as possible; `1.0` means use as much memory as needed to be as accurate as possible. `true` is supported as an alias for `0.3`. ++ +This statistic is computed for all field types but is not computed by default. [[TheStatsComponent-LocalParameters]] == Local Parameters diff --git a/solr/solr-ref-guide/src/the-term-vector-component.adoc b/solr/solr-ref-guide/src/the-term-vector-component.adoc index fb92dc9db31..dd73d8640ec 100644 --- a/solr/solr-ref-guide/src/the-term-vector-component.adoc +++ b/solr/solr-ref-guide/src/the-term-vector-component.adoc @@ -127,37 +127,48 @@ The example below shows an invocation of this component using the above configur [[TheTermVectorComponent-RequestParameters]] === Request Parameters -The example below shows the available request parameters for this component: +The example below shows some of the available request parameters for this component: -`\http://localhost:8983/solr/techproducts/tvrh?q=includes:[* TO *]&rows=10&indent=true&tv=true&tv.tf=true&tv.df=true&tv.positions=true&tv.offsets=true&tv.payloads=true&tv.fl=includes` +[source,bash] +http://localhost:8983/solr/techproducts/tvrh?q=includes:[* TO *]&rows=10&indent=true&tv=true&tv.tf=true&tv.df=true&tv.positions=true&tv.offsets=true&tv.payloads=true&tv.fl=includes -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`tv`:: +If `true`, the Term Vector Component will run. -[cols="20,60,20",options="header"] -|=== -|Boolean Parameters |Description |Type -|tv |Should the component run or not |boolean -|tv.docIds |Returns term vectors for the specified list of Lucene document IDs (not the Solr Unique Key). |comma seperated integers -|tv.fl |Returns term vectors for the specified list of fields. If not specified, the `fl` parameter is used. |comma seperated list of field names -|tv.all |A shortcut that invokes all the boolean parameters listed below. |boolean -|tv.df |Returns the Document Frequency (DF) of the term in the collection. This can be computationally expensive. |boolean -|tv.offsets |Returns offset information for each term in the document. |boolean -|tv.positions |Returns position information. |boolean -|tv.payloads |Returns payload information. |boolean -|tv.tf |Returns document term frequency info per term in the document. |boolean -|tv.tf_idf a| -Calculates TF / DF (ie: TF * IDF) for each term. Please note that this is a _literal_ calculation of "Term Frequency multiplied by Inverse Document Frequency" and *not* a classical TF-IDF similarity measure. +`tv.docIds`:: +For a given comma-separated list of Lucene document IDs (*not* the Solr Unique Key), term vectors will be returned. -Requires the parameters `tv.tf` and `tv.df` to be "true". This can be computationally expensive. (The results are not shown in example output) +`tv.fl`:: +For a given comma-separated list of fields, term vectors will be returned. If not specified, the `fl` parameter is used. - |boolean -|=== +`tv.all`:: +If `true`, all the boolean parameters listed below (`tv.df`, `tv.offsets`, `tv.positions`, `tv.payloads`, `tv.tf` and `tv.tf_idf`) will be enabled. + +`tv.df`:: +If `true`, returns the Document Frequency (DF) of the term in the collection. This can be computationally expensive. + +`tv.offsets`:: +If `true`, returns offset information for each term in the document. + +`tv.positions`:: +If `true`, returns position information. + +`tv.payloads`:: +If `true`, returns payload information. + +`tv.tf`:: +If `true`, returns document term frequency info for each term in the document. + +`tv.tf_idf`:: a| +If `true`, calculates TF / DF (ie: TF * IDF) for each term. Please note that this is a _literal_ calculation of "Term Frequency multiplied by Inverse Document Frequency" and *not* a classical TF-IDF similarity measure. ++ +This parameter requires both `tv.tf` and `tv.df` to be "true". This can be computationally expensive. (The results are not shown in example output) To learn more about TermVector component output, see the Wiki page: http://wiki.apache.org/solr/TermVectorComponentExampleOptions -For schema requirements, see the Wiki page: http://wiki.apache.org/solr/FieldOptionsByUseCase +For schema requirements, see also the section <>. [[TheTermVectorComponent-SolrJandtheTermVectorComponent]] == SolrJ and the Term Vector Component -Neither the SolrQuery class nor the QueryResponse class offer specific method calls to set Term Vector Component parameters or get the "termVectors" output. However, there is a patch for it: https://issues.apache.org/jira/browse/SOLR-949[SOLR-949]. +Neither the `SolrQuery` class nor the `QueryResponse` class offer specific method calls to set Term Vector Component parameters or get the "termVectors" output. However, there is a patch for it: https://issues.apache.org/jira/browse/SOLR-949[SOLR-949]. diff --git a/solr/solr-ref-guide/src/the-terms-component.adoc b/solr/solr-ref-guide/src/the-terms-component.adoc index 346278b52a0..c8ec78234e8 100644 --- a/solr/solr-ref-guide/src/the-terms-component.adoc +++ b/solr/solr-ref-guide/src/the-terms-component.adoc @@ -53,89 +53,86 @@ You could add this component to another handler if you wanted to, and pass "term The parameters below allow you to control what terms are returned. You can also configure any of these with the request handler if you'd like to set them permanently. Or, you can add them to the query request. These parameters are: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed - -[cols="20,15,15,50",options="header"] -|=== -|Parameter |Required |Default |Description -|terms |No |false a| -If set to true, enables the Terms Component. By default, the Terms Component is off. - +`terms`:: +If set to `true`, enables the Terms Component. By default, the Terms Component is off (`false`). ++ Example: `terms=true` -|terms.fl |Yes |null a| -Specifies the field from which to retrieve terms. - +`terms.fl`:: +Specifies the field from which to retrieve terms. This parameter is required if `terms=true`. ++ Example: `terms.fl=title` -|terms.list |No |null a| +`terms.list`:: Fetches the document frequency for a comma delimited list of terms. Terms are always returned in index order. If `terms.ttf` is set to true, also returns their total term frequency. If multiple `terms.fl` are defined, these statistics will be returned for each term in each requested field. - ++ Example: `terms.list=termA,termB,termC` -|terms.limit |No |10 a| -Specifies the maximum number of terms to return. The default is 10. If the limit is set to a number less than 0, then no maximum limit is enforced. Although this is not required, either this parameter or `terms.upper` must be defined. - +`terms.limit`:: +Specifies the maximum number of terms to return. The default is `10`. If the limit is set to a number less than 0, then no maximum limit is enforced. Although this is not required, either this parameter or `terms.upper` must be defined. ++ Example: `terms.limit=20` -|terms.lower |No |empty string a| +`terms.lower`:: Specifies the term at which to start. If not specified, the empty string is used, causing Solr to start at the beginning of the field. - ++ Example: `terms.lower=orange` -|terms.lower.incl |No |true a| +`terms.lower.incl`:: If set to true, includes the lower-bound term (specified with `terms.lower` in the result set. - ++ Example: `terms.lower.incl=false` -|terms.mincount |No |null a| +`terms.mincount`:: Specifies the minimum document frequency to return in order for a term to be included in a query response. Results are inclusive of the mincount (that is, >= mincount). - ++ Example: `terms.mincount=5` -|terms.maxcount |No |null a| +`terms.maxcount`:: Specifies the maximum document frequency a term must have in order to be included in a query response. The default setting is -1, which sets no upper bound. Results are inclusive of the maxcount (that is, <= maxcount). - ++ Example: `terms.maxcount=25` -|terms.prefix |No |null a| +`terms.prefix`:: Restricts matches to terms that begin with the specified string. - ++ Example: `terms.prefix=inter` -|terms.raw |No |false a| +`terms.raw`:: If set to true, returns the raw characters of the indexed term, regardless of whether it is human-readable. For instance, the indexed form of numeric numbers is not human-readable. - ++ Example: `terms.raw=true` -|terms.regex |No |null a| +`terms.regex`:: Restricts matches to terms that match the regular expression. - ++ Example: `terms.regex=.*pedist` -|terms.regex.flag |No |null a| +`terms.regex.flag`:: Defines a Java regex flag to use when evaluating the regular expression defined with `terms.regex`. See http://docs.oracle.com/javase/tutorial/essential/regex/pattern.html for details of each flag. Valid options are: -* case_insensitive -* comments -* multiline -* literal -* dotall -* unicode_case -* canon_eq -* unix_lines - +* `case_insensitive` +* `comments` +* `multiline` +* `literal` +* `dotall` +* `unicode_case` +* `canon_eq` +* `unix_lines` ++ Example: `terms.regex.flag=case_insensitive` -|terms.stats |No |null |Include index statistics in the results. Currently returns only the *numDocs* for a collection. When combined with terms.list it provides enough information to compute idf for a list of terms. -|terms.sort |No |count a| -Defines how to sort the terms returned. Valid options are *count*, which sorts by the term frequency, with the highest term frequency first, or *index*, which sorts in index order. +`terms.stats`:: +Include index statistics in the results. Currently returns only the *numDocs* for a collection. When combined with `terms.list` it provides enough information to compute inverse document frequency (IDF) for a list of terms. +`terms.sort`:: +Defines how to sort the terms returned. Valid options are `count`, which sorts by the term frequency, with the highest term frequency first, or `index`, which sorts in index order. ++ Example: `terms.sort=index` -|terms.ttf |No |false a| +`terms.ttf`:: If set to true, returns both `df` (docFreq) and `ttf` (totalTermFreq) statistics for each requested term in `terms.list`. In this case, the response format is: - ++ [source,xml] ---- @@ -148,19 +145,19 @@ If set to true, returns both `df` (docFreq) and `ttf` (totalTermFreq) statistics ---- -|terms.upper |No |null a| +`terms.upper`:: Specifies the term to stop at. Although this parameter is not required, either this parameter or `terms.limit` must be defined. - ++ Example: `terms.upper=plum` -|terms.upper.incl |No |false a| +`terms.upper.incl`:: If set to true, the upper bound term is included in the result set. The default is false. - ++ Example: `terms.upper.incl=true` -|=== +The response to a terms request is a list of the terms and their document frequency values. -The output is a list of the terms and their document frequency values. See below for examples. +You may also be interested in the {solr-javadocs}/solr-core/org/apache/solr/handler/component/TermsComponent.html[TermsComponent javadoc]. [[TheTermsComponent-Examples]] == Examples @@ -296,16 +293,8 @@ Result: The TermsComponent also supports distributed indexes. For the `/terms` request handler, you must provide the following two parameters: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`shards`:: +Specifies the shards in your distributed indexing configuration. For more information about distributed indexing, see <>. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|shards |Specifies the shards in your distributed indexing configuration. For more information about distributed indexing, see <>. -|shards.qt |Specifies the request handler Solr uses for requests to shards. -|=== - -[[TheTermsComponent-MoreResources]] -== More Resources - -* {solr-javadocs}/solr-core/org/apache/solr/handler/component/TermsComponent.html[TermsComponent javadoc] +`shards.qt`:: +Specifies the request handler Solr uses for requests to shards. diff --git a/solr/solr-ref-guide/src/update-request-processors.adoc b/solr/solr-ref-guide/src/update-request-processors.adoc index d1f5c351e74..7942028f792 100644 --- a/solr/solr-ref-guide/src/update-request-processors.adoc +++ b/solr/solr-ref-guide/src/update-request-processors.adoc @@ -386,7 +386,7 @@ These Update processors do not need any configuration is your `solrconfig.xml` . The `TemplateUpdateProcessorFactory` can be used to add new fields to documents based on a template pattern. -Use the parameter `processor=Template` to use it. The template parameter `Template.field` (multivalued) define the field to add and the pattern. Templates may contain placeholders which refer to other fields in the document. You can have multiple `Template.field` parameters in a single request. +Use the parameter `processor=Template` to use it. The template parameter `Template.field` (multivalued) defines the field to add and the pattern. Templates may contain placeholders which refer to other fields in the document. You can have multiple `Template.field` parameters in a single request. For example: @@ -395,7 +395,7 @@ For example: processor=Template&Template.field=fullName:Mr. {firstName} {lastName} ---- -The above example would add a new field to the document called `fullName`. The fields `firstName and` `lastName` are supplied from the document fields. If either of them is missing, that part is replaced with an empty string. If those fields are multi-valued, only the first value is used. +The above example would add a new field to the document called `fullName`. The fields `firstName` and `lastName` are supplied from the document fields. If either of them is missing, that part is replaced with an empty string. If those fields are multi-valued, only the first value is used. ==== AtomicUpdateProcessorFactory @@ -414,4 +414,4 @@ The above parameters convert a normal `update` operation on * `field1` to an atomic `add` operation * `field2` to an atomic `set` operation * `field3` to an atomic `inc` operation -* `field4` to an atomic `remove` operation \ No newline at end of file +* `field4` to an atomic `remove` operation diff --git a/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc b/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc index 664bd8c8e38..040da8626db 100644 --- a/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc +++ b/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc @@ -46,17 +46,16 @@ For more information about Near Real Time operations, see <> for more information: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`numRecordsToKeep`:: +The number of update records to keep per log. The default is `100`. + +`maxNumLogsToKeep`:: +The maximum number of logs keep. The default is `10`. + +`numVersionBuckets`:: +The number of buckets used to keep track of max version values when checking for re-ordered updates; increase this value to reduce the cost of synchronizing access to version buckets during high-volume indexing, this requires `(8 bytes (long) * numVersionBuckets)` of heap space per Solr core. The default is `65536`. -[cols="25,10,10,55",options="header"] -|=== -|Setting Name |Type |Default |Description -|numRecordsToKeep |int |100 |The number of update records to keep per log -|maxNumLogsToKeep |int |10 |The maximum number of logs keep -|numVersionBuckets |int |65536 |The number of buckets used to keep track of max version values when checking for re-ordered updates; increase this value to reduce the cost of synchronizing access to version buckets during high-volume indexing, this requires (8 bytes (long) * numVersionBuckets) of heap space per Solr core. -|=== An example, to be included under `` in `solrconfig.xml`, employing the above advanced settings: diff --git a/solr/solr-ref-guide/src/updating-parts-of-documents.adoc b/solr/solr-ref-guide/src/updating-parts-of-documents.adoc index ecd9b4c96f6..fac3cac57b1 100644 --- a/solr/solr-ref-guide/src/updating-parts-of-documents.adoc +++ b/solr/solr-ref-guide/src/updating-parts-of-documents.adoc @@ -35,37 +35,22 @@ Solr supports several modifiers that atomically update values of a document. Thi To use atomic updates, add a modifier to the field that needs to be updated. The content can be updated, added to, or incrementally increased if a number. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed - -[cols="30,70",options="header"] -|=== -|Modifier |Usage -|set a| +`set`:: Set or replace the field value(s) with the specified value(s), or remove the values if 'null' or empty list is specified as the new value. ++ +May be specified as a single value, or as a list for multiValued fields. -May be specified as a single value, or as a list for multiValued fields +`add`:: +Adds the specified values to a multiValued field. May be specified as a single value, or as a list. -|add a| -Adds the specified values to a multiValued field. +`remove`:: +Removes (all occurrences of) the specified values from a multiValued field. May be specified as a single value, or as a list. -May be specified as a single value, or as a list. +`removeregex`:: +Removes all occurrences of the specified regex from a multiValued field. May be specified as a single value, or as a list. -|remove a| -Removes (all occurrences of) the specified values from a multiValued field. - -May be specified as a single value, or as a list. - -|removeregex a| -Removes all occurrences of the specified regex from a multiValued field. - -May be specified as a single value, or as a list. - -|inc a| -Increments a numeric value by a specific amount. - -Must be specified as a single numeric value. - -|=== +`inc`:: +Increments a numeric value by a specific amount. Must be specified as a single numeric value. [[UpdatingPartsofDocuments-FieldStorage]] === Field Storage @@ -130,20 +115,11 @@ An atomic update operation is performed using this approach only when the fields To use in-place updates, add a modifier to the field that needs to be updated. The content can be updated or incrementally increased. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`set`:: +Set or replace the field value(s) with the specified value(s). May be specified as a single value. -[cols="30,70",options="header"] -|=== -|Modifier |Usage -|set a| -Set or replace the field value(s) with the specified value(s). - -May be specified as a single value. -|inc a| -Increments a numeric value by a specific amount. - -Must be specified as a single numeric value. -|=== +`inc`:: +Increments a numeric value by a specific amount. Must be specified as a single numeric value. [[UpdatingPartsofDocuments-Example.1]] === Example diff --git a/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc b/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc index 8bad5f58062..6a9d350120a 100644 --- a/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc +++ b/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc @@ -74,18 +74,15 @@ For example: The add command supports some optional attributes which may be specified. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`commitWithin`:: +Add the document within the specified number of milliseconds. -[cols="30,70",options="header"] -|=== -|Optional Parameter |Parameter Description -|commitWithin=_number_ |Add the document within the specified number of milliseconds -|overwrite=_boolean_ |Default is true. Indicates if the unique key constraints should be checked to overwrite previous versions of the same document (see below) -|=== +`overwrite`:: +Default is `true`. Indicates if the unique key constraints should be checked to overwrite previous versions of the same document (see below). -If the document schema defines a unique key, then by default an `/update` operation to add a document will overwrite (ie: replace) any document in the index with the same unique key. If no unique key has been defined, indexing performance is somewhat faster, as no check has to be made for an existing documents to replace. +If the document schema defines a unique key, then by default an `/update` operation to add a document will overwrite (i.e., replace) any document in the index with the same unique key. If no unique key has been defined, indexing performance is somewhat faster, as no check has to be made for an existing documents to replace. -If you have a unique key field, but you feel confident that you can safely bypass the uniqueness check (eg: you build your indexes in batch, and your indexing code guarantees it never adds the same document more than once) you can specify the `overwrite="false"` option when adding your documents. +If you have a unique key field, but you feel confident that you can safely bypass the uniqueness check (e.g., you build your indexes in batch, and your indexing code guarantees it never adds the same document more than once) you can specify the `overwrite="false"` option when adding your documents. [[UploadingDatawithIndexHandlers-XMLUpdateCommands]] === XML Update Commands @@ -101,15 +98,12 @@ The `` operation requests Solr to merge internal data structures in or The `` and `` elements accept these optional attributes: -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`waitSearcher`:: +Default is `true`. Blocks until a new searcher is opened and registered as the main query searcher, making the changes visible. -[cols="30,70",options="header"] -|=== -|Optional Attribute |Description -|waitSearcher |Default is true. Blocks until a new searcher is opened and registered as the main query searcher, making the changes visible. -|expungeDeletes |(commit only) Default is false. Merges segments that have more than 10% deleted docs, expunging them in the process. -|maxSegments |(optimize only) Default is 1. Merges the segments down to no more than this number of segments. -|=== +`expungeDeletes`:: (commit only) Default is `false`. Merges segments that have more than 10% deleted docs, expunging them in the process. + +`maxSegments`:: (optimize only) Default is `1`. Merges the segments down to no more than this number of segments. Here are examples of and using optional attributes: @@ -426,29 +420,83 @@ The CSV handler allows the specification of many parameters in the URL in the fo The table below describes the parameters for the update handler. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`separator`:: +Character used as field separator; default is ",". This parameter is global; for per-field usage, see the `split` parameter. ++ +Example: `separator=%09` -[cols="20,40,20,20",options="header"] -|=== -|Parameter |Usage |Global (g) or Per Field (f) |Example -|separator |Character used as field separator; default is "," |g,(f: see split) |separator=%09 -|trim |If true, remove leading and trailing whitespace from values. Default=false. |g,f |f.isbn.trim=true trim=false -|header |Set to true if first line of input contains field names. These will be used if the *fieldnames* parameter is absent. |g | -|fieldnames |Comma separated list of field names to use when adding documents. |g |fieldnames=isbn,price,title -|literal. |A literal value for a specified field name. |g |literal.color=red -|skip |Comma separated list of field names to skip. |g |skip=uninteresting,shoesize -|skipLines |Number of lines to discard in the input stream before the CSV data starts, including the header, if present. Default=0. |g |skipLines=5 -|encapsulator |The character optionally used to surround values to preserve characters such as the CSV separator or whitespace. This standard CSV format handles the encapsulator itself appearing in an encapsulated value by doubling the encapsulator. |g,(f: see split) |encapsulator=" -|escape |The character used for escaping CSV separators or other reserved characters. If an escape is specified, the encapsulator is not used unless also explicitly specified since most formats use either encapsulation or escaping, not both |g |escape=\ -|keepEmpty |Keep and index zero length (empty) fields. Default=false. |g,f |f.price.keepEmpty=true -|map |Map one value to another. Format is value:replacement (which can be empty.) |g,f |map=left:right f.subject.map=history:bunk -|split |If true, split a field into multiple values by a separate parser. |f | -|overwrite |If true (the default), check for and overwrite duplicate documents, based on the uniqueKey field declared in the Solr schema. If you know the documents you are indexing do not contain any duplicates then you may see a considerable speed up setting this to false. |g | -|commit |Issues a commit after the data has been ingested. |g | -|commitWithin |Add the document within the specified number of milliseconds. |g |commitWithin=10000 -|rowid |Map the rowid (line number) to a field specified by the value of the parameter, for instance if your CSV doesn't have a unique key and you want to use the row id as such. |g |rowid=id -|rowidOffset |Add the given offset (as an int) to the rowid before adding it to the document. Default is 0 |g |rowidOffset=10 -|=== +`trim`:: +If `true`, remove leading and trailing whitespace from values. The default is `false`. This parameter can be either global or per-field. ++ +Examples: `f.isbn.trim=true` or `trim=false` + +`header`:: +Set to `true` if first line of input contains field names. These will be used if the `fieldnames` parameter is absent. This parameter is global. + +`fieldnames`:: +Comma-separated list of field names to use when adding documents. This parameter is global. ++ +Example: `fieldnames=isbn,price,title` + +`literal._field_name_`:: +A literal value for a specified field name. This parameter is global. ++ +Example: `literal.color=red` + +`skip`:: +Comma separated list of field names to skip. This parameter is global. ++ +Example: `skip=uninteresting,shoesize` + +`skipLines`:: +Number of lines to discard in the input stream before the CSV data starts, including the header, if present. Default=`0`. This parameter is global. ++ +Example: `skipLines=5` + +`encapsulator`:: The character optionally used to surround values to preserve characters such as the CSV separator or whitespace. This standard CSV format handles the encapsulator itself appearing in an encapsulated value by doubling the encapsulator. ++ +This parameter is global; for per-field usage, see `split`. ++ +Example: `encapsulator="` + +`escape`:: The character used for escaping CSV separators or other reserved characters. If an escape is specified, the encapsulator is not used unless also explicitly specified since most formats use either encapsulation or escaping, not both. |g | + +Example: `escape=\` + +`keepEmpty`:: +Keep and index zero length (empty) fields. The default is `false`. This parameter can be global or per-field. ++ +Example: `f.price.keepEmpty=true` + +`map`:: Map one value to another. Format is value:replacement (which can be empty). This parameter can be global or per-field. ++ +Example: `map=left:right` or `f.subject.map=history:bunk` + +`split`:: +If `true`, split a field into multiple values by a separate parser. This parameter is used on a per-field basis. + +`overwrite`:: +If `true` (the default), check for and overwrite duplicate documents, based on the uniqueKey field declared in the Solr schema. If you know the documents you are indexing do not contain any duplicates then you may see a considerable speed up setting this to `false`. ++ +This parameter is global. + +`commit`:: +Issues a commit after the data has been ingested. This parameter is global. + +`commitWithin`:: +Add the document within the specified number of milliseconds. This parameter is global. ++ +Example: `commitWithin=10000` + +`rowid`:: +Map the `rowid` (line number) to a field specified by the value of the parameter, for instance if your CSV doesn't have a unique key and you want to use the row id as such. This parameter is global. ++ +Example: `rowid=id` + +`rowidOffset`:: +Add the given offset (as an integer) to the `rowid` before adding it to the document. Default is `0`. This parameter is global. ++ +Example: `rowidOffset=10` [[UploadingDatawithIndexHandlers-IndexingTab-Delimitedfiles]] === Indexing Tab-Delimited files diff --git a/solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adoc b/solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adoc index 670ef2b90ed..8096e8c11d3 100644 --- a/solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adoc +++ b/solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adoc @@ -101,41 +101,73 @@ This command allows you to query the document using an attribute, as in: `\http: The table below describes the parameters accepted by the Extracting Request Handler. -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`capture`:: +Captures XHTML elements with the specified name for a supplementary addition to the Solr document. This parameter can be useful for copying chunks of the XHTML into a separate field. For instance, it could be used to grab paragraphs (`

    `) and index them into a separate field. Note that content is still also captured into the overall "content" field. + +`captureAttr`:: +Indexes attributes of the Tika XHTML elements into separate fields, named after the element. If set to true, for example, when extracting from HTML, Tika can return the href attributes in tags as fields named "a". See the examples below. + +`commitWithin`:: +Add the document within the specified number of milliseconds. + +`date.formats`:: +Defines the date format patterns to identify in the documents. + +`defaultField`:: +If the `uprefix` parameter (see below) is not specified and a field cannot be determined, the default field will be used. + +`extractOnly`:: +Default is `false`. If `true`, returns the extracted content from Tika without indexing the document. This literally includes the extracted XHTML as a string in the response. When viewing manually, it may be useful to use a response format other than XML to aid in viewing the embedded XHTML tags. For an example, see http://wiki.apache.org/solr/TikaExtractOnlyExampleOutput. + +`extractFormat`:: +The default is `xml`, but the other option is `text`. Controls the serialization format of the extract content. The `xml` format is actually XHTML, the same format that results from passing the `-x` command to the Tika command line application, while the text format is like that produced by Tika's `-t` command. This parameter is valid only if `extractOnly` is set to true. + +`fmap._source_field_`:: +Maps (moves) one field name to another. The `source_field` must be a field in incoming documents, and the value is the Solr field to map to. Example: `fmap.content=text` causes the data in the `content` field generated by Tika to be moved to the Solr's `text` field. + +`ignoreTikaException`:: +If `true`, exceptions found during processing will be skipped. Any metadata available, however, will be indexed. + +`literal._fieldname_`:: +Populates a field with the name supplied with the specified value for each document. The data can be multivalued if the field is multivalued. + +`literalsOverride`:: +If `true` (the default), literal field values will override other values with the same field name. If `false`, literal values defined with `literal._fieldname_` will be appended to data already in the fields extracted from Tika. If setting `literalsOverride` to `false`, the field must be multivalued. + +`lowernames`:: +Values are `true` or `false`. If `true`, all field names will be mapped to lowercase with underscores, if needed. For example, "Content-Type" would be mapped to "content_type." + +`multipartUploadLimitInKB`:: +Useful if uploading very large documents, this defines the KB size of documents to allow. + +`passwordsFile`:: +Defines a file path and name for a file of file name to password mappings. + +`resource.name`:: +Specifies the optional name of the file. Tika can use it as a hint for detecting a file's MIME type. + +`resource.password`:: +Defines a password to use for a password-protected PDF or OOXML file + +`tika.config`:: +Defines a file path and name to a customized Tika configuration file. This is only required if you have customized your Tika implementation. + +`uprefix`:: +Prefixes all fields that are not defined in the schema with the given prefix. This is very useful when combined with dynamic field definitions. Example: `uprefix=ignored_` would effectively ignore all unknown fields generated by Tika given the example schema contains `` + +`xpath`:: +When extracting, only return Tika XHTML content that satisfies the given XPath expression. See http://tika.apache.org/1.7/index.html for details on the format of Tika XHTML. See also http://wiki.apache.org/solr/TikaExtractOnlyExampleOutput. -[cols="30,70",options="header"] -|=== -|Parameter |Description -|capture |Captures XHTML elements with the specified name for a supplementary addition to the Solr document. This parameter can be useful for copying chunks of the XHTML into a separate field. For instance, it could be used to grab paragraphs (`

    `) and index them into a separate field. Note that content is still also captured into the overall "content" field. -|captureAttr |Indexes attributes of the Tika XHTML elements into separate fields, named after the element. If set to true, for example, when extracting from HTML, Tika can return the href attributes in tags as fields named "a". See the examples below. -|commitWithin |Add the document within the specified number of milliseconds. -|date.formats |Defines the date format patterns to identify in the documents. -|defaultField |If the uprefix parameter (see below) is not specified and a field cannot be determined, the default field will be used. -|extractOnly |Default is false. If true, returns the extracted content from Tika without indexing the document. This literally includes the extracted XHTML as a string in the response. When viewing manually, it may be useful to use a response format other than XML to aid in viewing the embedded XHTML tags.For an example, see http://wiki.apache.org/solr/TikaExtractOnlyExampleOutput. -|extractFormat |Default is "xml", but the other option is "text". Controls the serialization format of the extract content. The xml format is actually XHTML, the same format that results from passing the `-x` command to the Tika command line application, while the text format is like that produced by Tika's `-t` command. This parameter is valid only if `extractOnly` is set to true. -|fmap.<__source_field__> |Maps (moves) one field name to another. The `source_field` must be a field in incoming documents, and the value is the Solr field to map to. Example: `fmap.content=text` causes the data in the `content` field generated by Tika to be moved to the Solr's `text` field. -|ignoreTikaException |If true, exceptions found during processing will be skipped. Any metadata available, however, will be indexed. -|literal.<__fieldname__> |Populates a field with the name supplied with the specified value for each document. The data can be multivalued if the field is multivalued. -|literalsOverride |If true (the default), literal field values will override other values with the same field name. If false, literal values defined with `literal.<__fieldname__>` will be appended to data already in the fields extracted from Tika. If setting `literalsOverride` to "false", the field must be multivalued. -|lowernames |Values are "true" or "false". If true, all field names will be mapped to lowercase with underscores, if needed. For example, "Content-Type" would be mapped to "content_type." -|multipartUploadLimitInKB |Useful if uploading very large documents, this defines the KB size of documents to allow. -|passwordsFile |Defines a file path and name for a file of file name to password mappings. -|resource.name |Specifies the optional name of the file. Tika can use it as a hint for detecting a file's MIME type. -|resource.password |Defines a password to use for a password-protected PDF or OOXML file -|tika.config |Defines a file path and name to a customized Tika configuration file. This is only required if you have customized your Tika implementation. -|uprefix |Prefixes all fields that are not defined in the schema with the given prefix. This is very useful when combined with dynamic field definitions. Example: `uprefix=ignored_` would effectively ignore all unknown fields generated by Tika given the example schema contains `` -|xpath |When extracting, only return Tika XHTML content that satisfies the given XPath expression. See http://tika.apache.org/1.7/index.html for details on the format of Tika XHTML. See also http://wiki.apache.org/solr/TikaExtractOnlyExampleOutput. -|=== [[UploadingDatawithSolrCellusingApacheTika-OrderofOperations]] == Order of Operations Here is the order in which the Solr Cell framework, using the Extracting Request Handler and Tika, processes its input. -1. Tika generates fields or passes them in as literals specified by `literal.=`. If `literalsOverride=false`, literals will be appended as multi-value to the Tika-generated field. -2. If `lowernames=true`, Tika maps fields to lowercase. -3. Tika applies the mapping rules specified by `fmap.__source__=__target__` parameters. -4. If `uprefix` is specified, any unknown field names are prefixed with that value, else if `defaultField` is specified, any unknown fields are copied to the default field. +. Tika generates fields or passes them in as literals specified by `literal.=`. If `literalsOverride=false`, literals will be appended as multi-value to the Tika-generated field. +. If `lowernames=true`, Tika maps fields to lowercase. +. Tika applies the mapping rules specified by `fmap.__source__=__target__` parameters. +. If `uprefix` is specified, any unknown field names are prefixed with that value, else if `defaultField` is specified, any unknown fields are copied to the default field. [[UploadingDatawithSolrCellusingApacheTika-ConfiguringtheSolrExtractingRequestHandler]] == Configuring the Solr ExtractingRequestHandler @@ -194,7 +226,7 @@ You may also need to adjust the `multipartUploadLimitInKB` attribute as follows ---- [[UploadingDatawithSolrCellusingApacheTika-Parserspecificproperties]] -=== Parser specific properties +=== Parser-Specific Properties Parsers used by Tika may have specific properties to govern how data is extracted. For instance, when using the Tika library from a Java program, the PDFParserConfig class has a method setSortByPosition(boolean) that can extract vertically oriented text. To access that method via configuration with the ExtractingRequestHandler, one can add the parseContext.config property to the solrconfig.xml file (see above) and then set properties in Tika's PDFParserConfig as below. Consult the Tika Java API documentation for configuration parameters that can be set for any particular parsers that require this level of control. @@ -241,16 +273,18 @@ As mentioned before, Tika produces metadata about the document. Metadata describ In addition to Tika's metadata, Solr adds the following metadata (defined in `ExtractingMetadataConstants`): -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`stream_name`:: +The name of the Content Stream as uploaded to Solr. Depending on how the file is uploaded, this may or may not be set. + +`stream_source_info`:: +Any source info about the stream. (See the section on Content Streams later in this section.) + +`stream_size`:: +The size of the stream in bytes. + +`stream_content_type`:: +The content type of the stream, if available. -[cols="30,70",options="header"] -|=== -|Solr Metadata |Description -|stream_name |The name of the Content Stream as uploaded to Solr. Depending on how the file is uploaded, this may or may not be set -|stream_source_info |Any source info about the stream. (See the section on Content Streams later in this section.) -|stream_size |The size of the stream in bytes. -|stream_content_type |The content type of the stream, if available. -|=== [IMPORTANT] ==== diff --git a/solr/solr-ref-guide/src/v2-api.adoc b/solr/solr-ref-guide/src/v2-api.adoc index 51357ab08f9..6906b1c2210 100644 --- a/solr/solr-ref-guide/src/v2-api.adoc +++ b/solr/solr-ref-guide/src/v2-api.adoc @@ -30,9 +30,9 @@ For now the two API styles will coexist, and all the old APIs will continue to w The old API and the v2 API differ in three principle ways: -1. Command format: The old API commands and associated parameters are provided through URL request parameters on HTTP GET requests, while in the v2 API most API commands are provided via a JSON body POST'ed to v2 API endpoints. The v2 API also supports HTTP methods GET and DELETE where appropriate. -2. Endpoint structure: The v2 API endpoint structure has been rationalized and regularized. -3. Documentation: The v2 APIs are self-documenting: append `/_introspect` to any valid v2 API path and the API specification will be returned in JSON format. +. Command format: The old API commands and associated parameters are provided through URL request parameters on HTTP GET requests, while in the v2 API most API commands are provided via a JSON body POST'ed to v2 API endpoints. The v2 API also supports HTTP methods GET and DELETE where appropriate. +. Endpoint structure: The v2 API endpoint structure has been rationalized and regularized. +. Documentation: The v2 APIs are self-documenting: append `/_introspect` to any valid v2 API path and the API specification will be returned in JSON format. [[v2API-v2APIPathPrefixes]] == v2 API Path Prefixes @@ -43,15 +43,15 @@ Following are some v2 API URL paths and path prefixes, along with some of the op |=== |Path prefix |Some Supported Operations |`/v2/collections` or equivalently: `/v2/c` |Create, alias, backup, and restore a collection. -|`/v2/c/__collection-name__/update` |Update requests. -|`/v2/c/__collection-name__/config` |Configuration requests. -|`/v2/c/__collection-name__/schema` |Schema requests. -|`/v2/c/__collection-name__/__handler-name__` |Handler-specific requests. -|`/v2/c/__collection-name__/shards` |Split a shard, create a shard, add a replica. -|`/v2/c/__collection-name__/shards/___shard-name___` |Delete a shard, force leader election -|`/v2/c/__collection-name__/shards/___shard-name____/____replica-name___` |Delete a replica. +|`/v2/c/_collection-name_/update` |Update requests. +|`/v2/c/_collection-name_/config` |Configuration requests. +|`/v2/c/_collection-name_/schema` |Schema requests. +|`/v2/c/_collection-name_/_handler-name_` |Handler-specific requests. +|`/v2/c/_collection-name_/shards` |Split a shard, create a shard, add a replica. +|`/v2/c/_collection-name_/shards/_shard-name_` |Delete a shard, force leader election +|`/v2/c/_collection-name_/shards/_shard-name_/_replica-name_` |Delete a replica. |`/v2/cores` |Create a core. -|`/v2/cores/__core-name__` |Reload, rename, delete, and unload a core. +|`/v2/cores/_core-name_` |Reload, rename, delete, and unload a core. |`/v2/node` |Perform overseer operation, rejoin leader election. |`/v2/cluster` |Add role, remove role, set cluster property. |`/v2/c/.system/blob` |Upload and download blobs and metadata. @@ -68,7 +68,7 @@ To limit the introspect output to include just one particular HTTP method, add r `\http://localhost:8983/v2/c/_introspect?method=POST` -Most endpoints support commands provided in a body sent via POST. To limit the introspect output to only one command, add request param `command=__command-name__` . +Most endpoints support commands provided in a body sent via POST. To limit the introspect output to only one command, add request param `command=_command-name_` . `\http://localhost:8983/v2/c/gettingstarted/_introspect?method=POST&command=modify` diff --git a/solr/solr-ref-guide/src/velocity-response-writer.adoc b/solr/solr-ref-guide/src/velocity-response-writer.adoc index b9bc3bdce5d..3b576f6fb50 100644 --- a/solr/solr-ref-guide/src/velocity-response-writer.adoc +++ b/solr/solr-ref-guide/src/velocity-response-writer.adoc @@ -45,57 +45,62 @@ The above example shows the optional initialization and custom tool parameters u [[VelocityResponseWriter-VelocityResponseWriterinitializationparameters]] === VelocityResponseWriter Initialization Parameters -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`template.base.dir`:: +If specified and exists as a file system directory, a file resource loader will be added for this directory. Templates in this directory will override "solr" resource loader templates. -[cols="20,60,20",options="header"] -|=== -|Parameter |Description |Default value -|template.base.dir |If specified and exists as a file system directory, a file resource loader will be added for this directory. Templates in this directory will override "solr" resource loader templates. | -|init.properties.file |Specifies a properties file name which must exist in the Solr `conf/` directory (**not** under a `velocity/` subdirectory) or root of a JAR file in a . | -|params.resource.loader.enabled a| +`init.properties.file`:: Specifies a properties file name which must exist in the Solr `conf/` directory (*not* under a `velocity/` subdirectory) or root of a JAR file in a . + +`params.resource.loader.enabled`:: The "params" resource loader allows templates to be specified in Solr request parameters. For example: ++ +[source,bash] +http://localhost:8983/solr/gettingstarted/select?q=\*:*&wt=velocity&v.template=custom&v.template.custom=CUSTOM%3A%20%23core_name ++ +where `v.template=custom` says to render a template called "custom" and the value of `v.template.custom` is the custom template. This is `false` by default; it'd be a niche, unusual, use case to need this enabled. -`\http://localhost:8983/solr/gettingstarted/select?q=\*:*&wt=velocity&v.template=custom&v.template.custom=CUSTOM%3A%20%23core_name` +`solr.resource.loader.enabled`:: +The "solr" resource loader is the only template loader registered by default. Templates are served from resources visible to the SolrResourceLoader under a `velocity/` subdirectory. The VelocityResponseWriter itself has some built-in templates (in its JAR file, under `velocity/`) that are available automatically through this loader. These built-in templates can be overridden when the same template name is in conf/velocity/ or by using the `template.base.dir` option. -where `v.template=custom` says to render a template called "custom" and `v.template.custom` 's value is the actual custom template. This is disabled by default; it'd be a niche, unusual, use case to need this enabled. - - |false -|solr.resource.loader.enabled |The "solr" resource loader is the only template loader registered by default. Templates are served from resources visible to the SolrResourceLoader under a `velocity/` subdirectory. The VelocityResponseWriter itself has some built-in templates (in its JAR file, under velocity/) that are available automatically through this loader. These built-in templates can be overridden when the same template name is in conf/velocity/ or by using the `template.base.dir` option. |true -|tools |External "tools" can be specified as list of string name/value (tool name / class name) pairs. Tools, in the Velocity context, are simply Java objects. Tool classes are constructed using a no-arg constructor (or a single-SolrCore-arg constructor if it exists) and added to the Velocity context with the specified name. A custom registered tool can override the built-in context objects with the same name, except for $request, $response, $page, and $debug (these tools are designed to not be overridden). | -|=== +`tools`:: +External "tools" can be specified as list of string name/value (tool name / class name) pairs. Tools, in the Velocity context, are simply Java objects. Tool classes are constructed using a no-arg constructor (or a single-SolrCore-arg constructor if it exists) and added to the Velocity context with the specified name. ++ +A custom registered tool can override the built-in context objects with the same name, except for `$request`, `$response`, `$page`, and `$debug` (these tools are designed to not be overridden). [[VelocityResponseWriter-VelocityResponseWriterrequestparameters]] === VelocityResponseWriter Request Parameters -// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed +`v.template`:: +Specifies the name of the template to render. -[cols="20,60,20",options="header"] -|=== -|Parameter |Description |Default value -|v.template |Specifies the name of the template to render. | -|v.layout a| +`v.layout`:: Specifies a template name to use as the layout around the main, `v.template`, specified template. - ++ The main template is rendered into a string value included into the layout rendering as `$content`. - | -|v.layout.enabled |Determines if the main template should have a layout wrapped around it. True by default, but requires `v.layout` to specified as well. |true -|v.contentType |Specifies the content type used in the HTTP response. If not specified, the default will depend on whether `v.json` is specified or not. a| -without json.wrf: text/html;charset=UTF-8 +`v.layout.enabled`:: +Determines if the main template should have a layout wrapped around it. The default is `true`, but requires `v.layout` to specified as well. -with json.wrf: application/json;charset=UTF-8 +`v.contentType`:: +Specifies the content type used in the HTTP response. If not specified, the default will depend on whether `v.json` is specified or not. ++ +The default without `v.json=wrf`: `text/html;charset=UTF-8`. ++ +The default with `v.json=wrf`: `application/json;charset=UTF-8`. -|v.json a| +`v.json`:: Specifies a function name to wrap around the response rendered as JSON. If specified, the content type used in the response will be "application/json;charset=UTF-8", unless overridden by `v.contentType`. - -Output will be in this format (with v.json=wrf): - ++ +Output will be in this format (with `v.json=wrf`): ++ `wrf("result":"")` - | -|v.locale |Locale to use with the `$resource` tool and other LocaleConfig implementing tools. The default locale is `Locale.ROOT`. Localized resources are loaded from standard Java resource bundles named `resources[_locale-code].properties`. Resource bundles can be added by providing a JAR file visible by the SolrResourceLoader with resource bundles under a velocity sub-directory. Resource bundles are not loadable under conf/, as only the class loader aspect of SolrResourceLoader can be used here. | -|v.template. |When the "params" resource loader is enabled, templates can be specified as part of the Solr request. | -|=== +`v.locale`:: +Locale to use with the `$resource` tool and other LocaleConfig implementing tools. The default locale is `Locale.ROOT`. Localized resources are loaded from standard Java resource bundles named `resources[_locale-code].properties`. ++ +Resource bundles can be added by providing a JAR file visible by the SolrResourceLoader with resource bundles under a velocity sub-directory. Resource bundles are not loadable under `conf/`, as only the class loader aspect of SolrResourceLoader can be used here. + +`v.template._template_name_`:: When the "params" resource loader is enabled, templates can be specified as part of the Solr request. + [[VelocityResponseWriter-VelocityResponseWritercontextobjects]] === VelocityResponseWriter Context Objects @@ -105,19 +110,19 @@ Output will be in this format (with v.json=wrf): [cols="30,70",options="header"] |=== |Context Reference |Description -|request |http://lucene.apache.org/solr/api/org/apache/solr/request/SolrQueryRequest.html[SolrQueryRequest] javadocs -|response |http://lucene.apache.org/solr/api/org/apache/solr/client/solrj/response/QueryResponse.html[QueryResponse] most of the time, but in some cases where https://wiki.apache.org/solr/QueryResponse[QueryResponse] doesn't like the request handlers output (https://wiki.apache.org/solr/AnalysisRequestHandler[AnalysisRequestHandler], for example, causes a ClassCastException parsing "response"), the response will be a https://wiki.apache.org/solr/SolrResponseBase[SolrResponseBase] object. -|esc |A Velocity http://velocity.apache.org/tools/2.0/tools-summary.html#EscapeTool[EscapeTool] instance -|date |A Velocity http://velocity.apache.org/tools/2.0/tools-summary.html#ComparisonDateTool[ComparisonDateTool] instance -|list |A Velocity http://velocity.apache.org/tools/2.0/apidocs/org/apache/velocity/tools/generic/ListTool.html[ListTool] instance -|math |A Velocity http://velocity.apache.org/tools/2.0/tools-summary.html#MathTool[MathTool] instance -|number |A Velocity http://velocity.apache.org/tools/2.0/tools-summary.html#NumberTool[NumberTool] instance -|sort |A Velocity http://velocity.apache.org/tools/2.0/tools-summary.html#SortTool[SortTool] instance -|display |A Velocity http://velocity.apache.org/tools/2.0/tools-summary.html#DisplayTool[DisplayTool] instance -|resource |A Velocity http://velocity.apache.org/tools/2.0/tools-summary.html#ResourceTool[ResourceTool] instance -|engine |The current VelocityEngine instance -|page |An instance of Solr's PageTool (only included if the response is a QueryResponse where paging makes sense) -|debug |A shortcut to the debug part of the response, or null if debug is not on. This is handy for having debug-only sections in a template using `#if($debug)...#end` -|content |The rendered output of the main template, when rendering the layout (v.layout.enabled=true and v.layout=