Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/autoscaling 

# Conflicts:
#	solr/CHANGES.txt
This commit is contained in:
Shalin Shekhar Mangar 2017-09-18 09:23:41 -07:00
parent 6fb2803bca 2e5f9a4369
commit de3107c70f
149 changed files with 5575 additions and 807 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
build.xml
View File

@ -374,7 +374,23 @@
</subant>
</target>
<target name="validate-maven-dependencies" depends="generate-maven-artifacts"
<target name="-install-maven-artifacts" depends="resolve,resolve-groovy,resolve-markdown,install-maven-tasks">
<ant dir="lucene" inheritall="false">
<target name="-unpack-lucene-tgz"/>
<target name="-filter-pom-templates"/>
<propertyset refid="uptodate.and.compiled.properties"/>
</ant>
<ant dir="solr" target="-unpack-solr-tgz" inheritall="false">
<propertyset refid="uptodate.and.compiled.properties"/>
</ant>
<subant target="-install-to-maven-local-repo" inheritall="false" failonerror="true">
<fileset dir="lucene" includes="build.xml" />
<fileset dir="solr" includes="build.xml" />
<propertyset refid="uptodate.and.compiled.properties"/>
</subant>
</target>
<target name="validate-maven-dependencies" depends="-install-maven-artifacts"
description="Validates maven dependencies, licenses, etc.">
<subant target="-validate-maven-dependencies" inheritall="false" failonerror="true">
<fileset dir="lucene" includes="build.xml"/>

9
dev-tools/doap/lucene.rdf
View File

@ -66,11 +66,18 @@
</foaf:Person>
</maintainer>
<release>
<Version>
<name>lucene-6.6.1</name>
<created>2017-09-07</created>
<revision>6.6.1</revision>
</Version>
</release>
<release>
<Version>
<name>lucene-6.6.0</name>
<created>2017-06-06</created>
<revision>6.6.0</revision>
<revision>6.6.0</revision>
</Version>
</release>
<release>

7
dev-tools/doap/solr.rdf
View File

@ -66,6 +66,13 @@
</foaf:Person>
</maintainer>
<release>
<Version>
<name>solr-6.6.1</name>
<created>2017-09-07</created>
<revision>6.6.1</revision>
</Version>
</release>
<release>
<Version>
<name>solr-6.6.0</name>

20
lucene/CHANGES.txt
View File

@ -23,6 +23,8 @@ New Features
* LUCENE-7927: Add LongValueFacetCounts, to compute facet counts for individual
numeric values (Mike McCandless)
* LUCENE-7940: Add BengaliAnalyzer. (Md. Abdulla-Al-Sun via Robert Muir)
Optimizations
* LUCENE-7905: Optimize how OrdinalMap (used by
@ -52,6 +54,17 @@ Bug Fixes
not recommended, lucene-analyzers-icu contains binary data structures
specific to ICU/Unicode versions it is built against. (Chris Koenig, Robert Muir)
* LUCENE-7891: Lucene's taxonomy facets now uses a non-buggy LRU cache
by default. (Jan-Willem van den Broek via Mike McCandless)
* LUCENE-7959: Improve NativeFSLockFactory's exception message if it cannot create
write.lock for an empty index due to bad permissions/read-only filesystem/etc.
(Erick Erickson, Shawn Heisey, Robert Muir)
* LUCENE-7968: AnalyzingSuggester would sometimes order suggestions incorrectly,
it did not properly break ties on the surface forms when both the weights and
the analyzed forms were equal. (Robert Muir)
Build
* SOLR-11181: Switch order of maven artifact publishing procedure: deploy first
@ -192,6 +205,13 @@ Bug Fixes
* LUCENE-7864: IndexMergeTool is not using intermediate hard links (even
if possible). (Dawid Weiss)
* LUCENE-7956: Fixed potential stack overflow error in ICUNormalizer2CharFilter.
(Adrien Grand)
* LUCENE-7963: Remove useless getAttribute() in DefaultIndexingChain that
causes performance drop, introduced by LUCENE-7626. (Daniel Mitterdorfer
via Uwe Schindler)
Improvements
* LUCENE-7489: Better storage of sparse doc-values fields with the default

5
lucene/NOTICE.txt
View File

@ -54,13 +54,14 @@ The KStem stemmer in
was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst)
under the BSD-license.
The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default
The Arabic,Persian,Romanian,Bulgarian, Hindi and Bengali analyzers (common) come with a default
stopword list that is BSD-licensed created by Jacques Savoy. These files reside in:
analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt,
analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt,
analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt,
analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt,
analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt
analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt,
analysis/common/src/resources/org/apache/lucene/analysis/bn/stopwords.txt
See http://members.unine.ch/jacques.savoy/clef/index.html.
The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers

4
lucene/analysis/build.xml
View File

@ -125,6 +125,10 @@
<forall-analyzers target="-dist-maven"/>
</target>
<target name="-install-to-maven-local-repo">
<forall-analyzers target="-install-to-maven-local-repo"/>
</target>
<target name="-validate-maven-dependencies">
<forall-analyzers target="-validate-maven-dependencies"/>
</target>

132
lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliAnalyzer.java Normal file
View File

@ -0,0 +1,132 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.analysis.in.IndicNormalizationFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import java.io.IOException;
import java.io.Reader;
/**
* Analyzer for Bengali.
*/
public final class BengaliAnalyzer extends StopwordAnalyzerBase {
private final CharArraySet stemExclusionSet;
/**
* File containing default Bengali stopwords.
*
* Default stopword list is from http://members.unine.ch/jacques.savoy/clef/bengaliST.txt
* The stopword list is BSD-Licensed.
*/
public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
private static final String STOPWORDS_COMMENT = "#";
/**
* Returns an unmodifiable instance of the default stop-words set.
* @return an unmodifiable instance of the default stop-words set.
*/
public static CharArraySet getDefaultStopSet(){
return DefaultSetHolder.DEFAULT_STOP_SET;
}
/**
* Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class
* accesses the static final set the first time.;
*/
private static class DefaultSetHolder {
static final CharArraySet DEFAULT_STOP_SET;
static {
try {
DEFAULT_STOP_SET = loadStopwordSet(false, BengaliAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
} catch (IOException ex) {
throw new RuntimeException("Unable to load default stopword set");
}
}
}
/**
* Builds an analyzer with the given stop words
*
* @param stopwords a stopword set
* @param stemExclusionSet a stemming exclusion set
*/
public BengaliAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
* Builds an analyzer with the given stop words
*
* @param stopwords a stopword set
*/
public BengaliAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
/**
* Builds an analyzer with the default stop words:
* {@link #DEFAULT_STOPWORD_FILE}.
*/
public BengaliAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Creates
* {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
* used to tokenize all the text in the provided {@link Reader}.
*
* @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
* built from a {@link StandardTokenizer} filtered with
* {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link IndicNormalizationFilter},
* {@link BengaliNormalizationFilter}, {@link SetKeywordMarkerFilter}
* if a stem exclusion set is provided, {@link BengaliStemFilter}, and
* Bengali Stop words
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer();
TokenStream result = new LowerCaseFilter(source);
result = new DecimalDigitFilter(result);
if (!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new IndicNormalizationFilter(result);
result = new BengaliNormalizationFilter(result);
result = new StopFilter(result, stopwords);
result = new BengaliStemFilter(result);
return new TokenStreamComponents(source, result);
}
@Override
protected TokenStream normalize(String fieldName, TokenStream in) {
TokenStream result = new StandardFilter(in);
result = new LowerCaseFilter(result);
result = new DecimalDigitFilter(result);
result = new IndicNormalizationFilter(result);
result = new BengaliNormalizationFilter(result);
return result;
}
}

59
lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilter.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import java.io.IOException;
/**
* A {@link TokenFilter} that applies {@link BengaliNormalizer} to normalize the
* orthography.
* <p>
* In some cases the normalization may cause unrelated terms to conflate, so
* to prevent terms from being normalized use an instance of
* {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets
* the {@link KeywordAttribute} before this {@link TokenStream}.
* </p>
* @see BengaliNormalizer
*/
public final class BengaliNormalizationFilter extends TokenFilter {
private final BengaliNormalizer normalizer = new BengaliNormalizer();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
public BengaliNormalizationFilter(TokenStream input) {
super(input);
}
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (!keywordAtt.isKeyword())
termAtt.setLength(normalizer.normalize(termAtt.buffer(),
termAtt.length()));
return true;
}
return false;
}
}

55
lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import java.util.Map;
/**
* Factory for {@link BengaliNormalizationFilter}.
* <pre class="prettyprint">
* &lt;fieldType name="text_bnnormal" class="solr.TextField" positionIncrementGap="100"&gt;
* &lt;analyzer&gt;
* &lt;tokenizer class="solr.StandardTokenizerFactory"/&gt;
* &lt;filter class="solr.BengaliNormalizationFilterFactory"/&gt;
* &lt;/analyzer&gt;
* &lt;/fieldType&gt;</pre>
*/
public class BengaliNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
public BengaliNormalizationFilterFactory(Map<String,String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
@Override
public TokenStream create(TokenStream input) {
return new BengaliNormalizationFilter(input);
}
@Override
public AbstractAnalysisFactory getMultiTermComponent() {
return this;
}
}

155
lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizer.java Normal file
View File

@ -0,0 +1,155 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import static org.apache.lucene.analysis.util.StemmerUtil.delete;
/**
* Normalizer for Bengali.
* <p>
* Implements the Bengali-language specific algorithm specified in:
* <i>A Double Metaphone encoding for Bangla and its application in spelling checker</i>
* Naushad UzZaman and Mumit Khan.
* http://www.panl10n.net/english/final%20reports/pdf%20files/Bangladesh/BAN16.pdf
* </p>
*/
public class BengaliNormalizer {
/**
* Normalize an input buffer of Bengali text
*
* @param s input buffer
* @param len length of input buffer
* @return length of input buffer after normalization
*/
public int normalize(char s[], int len) {
for (int i = 0; i < len; i++) {
switch (s[i]) {
// delete Chandrabindu
case '\u0981':
len = delete(s, i, len);
i--;
break;
// DirghoI kar -> RosshoI kar
case '\u09C0':
s[i] = '\u09BF';
break;
// DirghoU kar -> RosshoU kar
case '\u09C2':
s[i] = '\u09C1';
break;
// Khio (Ka + Hoshonto + Murdorno Sh)
case '\u0995':
if(i + 2 < len && s[i+1] == '\u09CD' && s[i+2] == '\u09BF') {
if (i == 0) {
s[i] = '\u0996';
len = delete(s, i + 2, len);
len = delete(s, i + 1, len);
} else {
s[i+1] = '\u0996';
len = delete(s, i + 2, len);
}
}
break;
// Nga to Anusvara
case '\u0999':
s[i] = '\u0982';
break;
// Ja Phala
case '\u09AF':
if(i - 2 == 0 && s[i-1] == '\u09CD') {
s[i - 1] = '\u09C7';
if(i + 1 < len && s[i+1] == '\u09BE') {
len = delete(s, i+1, len);
}
len = delete(s, i, len);
i --;
} else if(i - 1 >= 0 && s[i-1] == '\u09CD' ){
len = delete(s, i, len);
len = delete(s, i-1, len);
i -=2;
}
break;
// Ba Phalaa
case '\u09AC':
if((i >= 1 && s[i-1] != '\u09CD') || i == 0)
break;
if(i - 2 == 0) {
len = delete(s, i, len);
len = delete(s, i - 1, len);
i -= 2;
} else if(i - 5 >= 0 && s[i - 3] == '\u09CD') {
len = delete(s, i, len);
len = delete(s, i-1, len);
i -=2;
} else if(i - 2 >= 0){
s[i - 1] = s[i - 2];
len = delete(s, i, len);
i --;
}
break;
// Visarga
case '\u0983':
if(i == len -1) {
if(len <= 3) {
s[i] = '\u09B9';
} else {
len = delete(s, i, len);
}
} else {
s[i] = s[i+1];
}
break;
//All sh
case '\u09B6':
case '\u09B7':
s[i] = '\u09B8';
break;
//check na
case '\u09A3':
s[i] = '\u09A8';
break;
//check ra
case '\u09DC':
case '\u09DD':
s[i] = '\u09B0';
break;
case '\u09CE':
s[i] = '\u09A4';
break;
default:
break;
}
}
return len;
}
}

49
lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilter.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import java.io.IOException;
/**
* A {@link TokenFilter} that applies {@link BengaliStemmer} to stem Bengali words.
*/
public final class BengaliStemFilter extends TokenFilter {
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttribute = addAttribute(KeywordAttribute.class);
private final BengaliStemmer bengaliStemmer = new BengaliStemmer();
public BengaliStemFilter(TokenStream input) {
super(input);
}
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (!keywordAttribute.isKeyword())
termAttribute.setLength(bengaliStemmer.stem(termAttribute.buffer(), termAttribute.length()));
return true;
} else {
return false;
}
}
}

48
lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilterFactory.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import java.util.Map;
/**
* Factory for {@link BengaliStemFilter}.
* <pre class="prettyprint">
* &lt;fieldType name="text_histem" class="solr.TextField" positionIncrementGap="100"&gt;
* &lt;analyzer&gt;
* &lt;tokenizer class="solr.StandardTokenizerFactory"/&gt;
* &lt;filter class="solr.BengaliStemFilterFactory"/&gt;
* &lt;/analyzer&gt;
* &lt;/fieldType&gt;</pre>
*/
public class BengaliStemFilterFactory extends TokenFilterFactory {
public BengaliStemFilterFactory(Map<String,String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
@Override
public TokenStream create(TokenStream input) {
return new BengaliStemFilter(input);
}
}

183
lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemmer.java Normal file
View File

@ -0,0 +1,183 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import static org.apache.lucene.analysis.util.StemmerUtil.endsWith;
/**
* Stemmer for Bengali.
* <p>
* The algorithm is based on the report in:
* <i>Natural Language Processing in an Indian Language (Bengali)-I: Verb Phrase Analysis</i>
* P Sengupta and B B Chaudhuri
* </p>
*
* <p>
* Few Stemmer criteria are taken from:
* <i>http://members.unine.ch/jacques.savoy/clef/BengaliStemmerLight.java.txt</i>
* </p>
*/
public class BengaliStemmer {
public int stem(char buffer[], int len) {
// 8
if (len > 9 && (endsWith(buffer, len, "িয়াছিলাম")
|| endsWith(buffer, len, "িতেছিলাম")
|| endsWith(buffer, len, "িতেছিলেন")
|| endsWith(buffer, len, "ইতেছিলেন")
|| endsWith(buffer, len, "িয়াছিলেন")
|| endsWith(buffer, len, "ইয়াছিলেন")
))
return len - 8;
// 7
if ((len > 8) && (endsWith(buffer, len, "িতেছিলি")
|| endsWith(buffer, len, "িতেছিলে")
|| endsWith(buffer, len, "িয়াছিলা")
|| endsWith(buffer, len, "িয়াছিলে")
|| endsWith(buffer, len, "িতেছিলা")
|| endsWith(buffer, len, "িয়াছিলি")
|| endsWith(buffer, len, "য়েদেরকে")
))
return len - 7;
// 6
if ((len > 7) && (endsWith(buffer, len, "িতেছিস")
|| endsWith(buffer, len, "িতেছেন")
|| endsWith(buffer, len, "িয়াছিস")
|| endsWith(buffer, len, "িয়াছেন")
|| endsWith(buffer, len, "েছিলাম")
|| endsWith(buffer, len, "েছিলেন")
|| endsWith(buffer, len, "েদেরকে")
))
return len - 6;
// 5
if ((len > 6) && (endsWith(buffer, len, "িতেছি")
|| endsWith(buffer, len, "িতেছা")
|| endsWith(buffer, len, "িতেছে")
|| endsWith(buffer, len, "ছিলাম")
|| endsWith(buffer, len, "ছিলেন")
|| endsWith(buffer, len, "িয়াছি")
|| endsWith(buffer, len, "িয়াছা")
|| endsWith(buffer, len, "িয়াছে")
|| endsWith(buffer, len, "েছিলে")
|| endsWith(buffer, len, "েছিলা")
|| endsWith(buffer, len, "য়েদের")
|| endsWith(buffer, len, "দেরকে")
))
return len - 5;
// 4
if ((len > 5) && (endsWith(buffer, len, "িলাম")
|| endsWith(buffer, len, "িলেন")
|| endsWith(buffer, len, "িতাম")
|| endsWith(buffer, len, "িতেন")
|| endsWith(buffer, len, "িবেন")
|| endsWith(buffer, len, "ছিলি")
|| endsWith(buffer, len, "ছিলে")
|| endsWith(buffer, len, "ছিলা")
|| endsWith(buffer, len, "তেছে")
|| endsWith(buffer, len, "িতেছ")
|| endsWith(buffer, len, "খানা")
|| endsWith(buffer, len, "খানি")
|| endsWith(buffer, len, "গুলো")
|| endsWith(buffer, len, "গুলি")
|| endsWith(buffer, len, "য়েরা")
|| endsWith(buffer, len, "েদের")
))
return len - 4;
// 3
if ((len > 4) && (endsWith(buffer, len, "লাম")
|| endsWith(buffer, len, "িলি")
|| endsWith(buffer, len, "ইলি")
|| endsWith(buffer, len, "িলে")
|| endsWith(buffer, len, "ইলে")
|| endsWith(buffer, len, "লেন")
|| endsWith(buffer, len, "িলা")
|| endsWith(buffer, len, "ইলা")
|| endsWith(buffer, len, "তাম")
|| endsWith(buffer, len, "িতি")
|| endsWith(buffer, len, "ইতি")
|| endsWith(buffer, len, "িতে")
|| endsWith(buffer, len, "ইতে")
|| endsWith(buffer, len, "তেন")
|| endsWith(buffer, len, "িতা")
|| endsWith(buffer, len, "িবা")
|| endsWith(buffer, len, "ইবা")
|| endsWith(buffer, len, "িবি")
|| endsWith(buffer, len, "ইবি")
|| endsWith(buffer, len, "বেন")
|| endsWith(buffer, len, "িবে")
|| endsWith(buffer, len, "ইবে")
|| endsWith(buffer, len, "ছেন")
|| endsWith(buffer, len, "য়োন")
|| endsWith(buffer, len, "য়ের")
|| endsWith(buffer, len, "েরা")
|| endsWith(buffer, len, "দের")
))
return len - 3;
// 2
if ((len > 3) && (endsWith(buffer, len, "িস")
|| endsWith(buffer, len, "েন")
|| endsWith(buffer, len, "লি")
|| endsWith(buffer, len, "লে")
|| endsWith(buffer, len, "লা")
|| endsWith(buffer, len, "তি")
|| endsWith(buffer, len, "তে")
|| endsWith(buffer, len, "তা")
|| endsWith(buffer, len, "বি")
|| endsWith(buffer, len, "বে")
|| endsWith(buffer, len, "বা")
|| endsWith(buffer, len, "ছি")
|| endsWith(buffer, len, "ছা")
|| endsWith(buffer, len, "ছে")
|| endsWith(buffer, len, "ুন")
|| endsWith(buffer, len, "ুক")
|| endsWith(buffer, len, "টা")
|| endsWith(buffer, len, "টি")
|| endsWith(buffer, len, "নি")
|| endsWith(buffer, len, "ের")
|| endsWith(buffer, len, "তে")
|| endsWith(buffer, len, "রা")
|| endsWith(buffer, len, "কে")
))
return len - 2;
// 1
if ((len > 2) && (endsWith(buffer, len, "ি")
|| endsWith(buffer, len, "ী")
|| endsWith(buffer, len, "া")
|| endsWith(buffer, len, "ো")
|| endsWith(buffer, len, "ে")
|| endsWith(buffer, len, "ব")
|| endsWith(buffer, len, "ত")
))
return len - 1;
return len;
}
}

20
solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java → lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/package-info.java
View File

@ -14,24 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
/**
* Parse Solr's variant of Lucene QueryParser syntax, including the
* deprecated sort specification after the query.
* <br>Example: <code>{!lucenePlusSort}myfield:foo +bar -baz;price asc</code>
*
* @deprecated This class should have been removed a long time ago, it will be removed in Solr 8.0
* Analyzer for Bengali Language.
*/
@Deprecated
public class OldLuceneQParserPlugin extends QParserPlugin {
public static final String NAME = "lucenePlusSort";
@Override
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new OldLuceneQParser(qstr, localParams, params, req);
}
}
package org.apache.lucene.analysis.bn;

2
lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
View File

@ -17,6 +17,8 @@ org.apache.lucene.analysis.tr.ApostropheFilterFactory
org.apache.lucene.analysis.ar.ArabicNormalizationFilterFactory
org.apache.lucene.analysis.ar.ArabicStemFilterFactory
org.apache.lucene.analysis.bg.BulgarianStemFilterFactory
org.apache.lucene.analysis.bn.BengaliNormalizationFilterFactory
org.apache.lucene.analysis.bn.BengaliStemFilterFactory
org.apache.lucene.analysis.br.BrazilianStemFilterFactory
org.apache.lucene.analysis.cjk.CJKBigramFilterFactory
org.apache.lucene.analysis.cjk.CJKWidthFilterFactory

121
lucene/analysis/common/src/resources/org/apache/lucene/analysis/bn/stopwords.txt Normal file
View File

@ -0,0 +1,121 @@
# See http://members.unine.ch/jacques.savoy/clef/index.html.
# This file was created by Jacques Savoy and is distributed under the BSD license
এই
ও
থেকে
করে
এ
না
ওই
এক্
নিয়ে
করা
বলেন
সঙ্গে
যে
এব
তা
আর
কোনো
বলে
সেই
দিন
হয়
কি
দু
পরে
সব
দেওয়া
মধ্যে
এর
সি
শুরু
কাজ
কিছু
কাছে
সে
তবে
বা
বন
আগে
জ্নজন
পি
পর
তো
ছিল
এখন
আমরা
প্রায়
দুই
আমাদের
তাই
অন্য
গিয়ে
প্রযন্ত
মনে
নতুন
মতো
কেখা
প্রথম
আজ
টি
ধামার
অনেক
বিভিন্ন
র
হাজার
জানা
নয়
অবশ্য
বেশি
এস
করে
কে
হতে
বি
কয়েক
সহ
বেশ
এমন
এমনি
কেন
কেউ
নেওয়া
চেষ্টা
লক্ষ
বলা
কারণ
আছে
শুধু
তখন
যা
এসে
চার
ছিল
যদি
আবার
কোটি
উত্তর
সামনে
উপর
বক্তব্য
এত
প্রাথমিক
উপরে
আছে
প্রতি
কাজে
যখন
খুব
বহু
গেল
পেয়্র্
চালু
ই
নাগাদ
থাকা
পাচ
যাওয়া
রকম
সাধারণ
কমনে

53
lucene/analysis/common/src/test/org/apache/lucene/analysis/bn/TestBengaliAnalyzer.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
/**
* Tests the BengaliAnalyzer
*/
public class TestBengaliAnalyzer extends BaseTokenStreamTestCase {
public void testResourcesAvailable() {
new BengaliAnalyzer().close();
}
public void testBasics() throws Exception {
Analyzer a = new BengaliAnalyzer();
checkOneTerm(a, "বাড়ী", "বার");
checkOneTerm(a, "বারী", "বার");
a.close();
}
/**
* test Digits
*/
public void testDigits() throws Exception {
BengaliAnalyzer a = new BengaliAnalyzer();
checkOneTerm(a, "১২৩৪৫৬৭৮৯০", "1234567890");
a.close();
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Analyzer analyzer = new BengaliAnalyzer();
checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
analyzer.close();
}
}

80
lucene/analysis/common/src/test/org/apache/lucene/analysis/bn/TestBengaliFilters.java Normal file
View File

@ -0,0 +1,80 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Test Bengali Filter Factory
*/
public class TestBengaliFilters extends BaseTokenStreamFactoryTestCase {
/**
* Test IndicNormalizationFilterFactory
*/
public void testIndicNormalizer() throws Exception {
Reader reader = new StringReader("ত্‍ আমি");
TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "ৎ", "আমি" });
}
/**
* Test BengaliNormalizationFilterFactory
*/
public void testBengaliNormalizer() throws Exception {
Reader reader = new StringReader("বাড়ী");
TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
stream = tokenFilterFactory("BengaliNormalization").create(stream);
assertTokenStreamContents(stream, new String[] {"বারি"});
}
/**
* Test BengaliStemFilterFactory
*/
public void testStemmer() throws Exception {
Reader reader = new StringReader("বাড়ী");
TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
stream = tokenFilterFactory("BengaliNormalization").create(stream);
stream = tokenFilterFactory("BengaliStem").create(stream);
assertTokenStreamContents(stream, new String[] {"বার"});
}
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
tokenFilterFactory("IndicNormalization", "bogusArg", "bogusValue");
});
assertTrue(expected.getMessage().contains("Unknown parameters"));
expected = expectThrows(IllegalArgumentException.class, () -> {
tokenFilterFactory("BengaliNormalization", "bogusArg", "bogusValue");
});
assertTrue(expected.getMessage().contains("Unknown parameters"));
expected = expectThrows(IllegalArgumentException.class, () -> {
tokenFilterFactory("BengaliStem", "bogusArg", "bogusValue");
});
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
}

110
lucene/analysis/common/src/test/org/apache/lucene/analysis/bn/TestBengaliNormalizer.java Normal file
View File

@ -0,0 +1,110 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.util.TestUtil;
import java.io.IOException;
/**
* Test BengaliNormalizer
*/
public class TestBengaliNormalizer extends BaseTokenStreamTestCase {
/**
* Test some basic normalization, with an example from the paper.
*/
public void testChndrobindu() throws IOException {
check("চাঁদ", "চাদ");
}
public void testRosshoIKar() throws IOException {
check("বাড়ী", "বারি");
check("তীর", "তির");
}
public void testRosshoUKar() throws IOException {
check("ভূল", "ভুল");
check("অনূপ", "অনুপ");
}
public void testNga() throws IOException {
check("বাঙলা", "বাংলা");
}
public void testJaPhaala() throws IOException {
check("ব্যাক্তি", "বেক্তি");
check( "সন্ধ্যা", "সন্ধা");
}
public void testBaPhalaa() throws IOException {
check("স্বদেশ", "সদেস");
check("তত্ত্ব", "তত্ত");
check("বিশ্ব", "বিসস");
}
public void testVisarga() throws IOException {
check("দুঃখ", "দুখখ");
check("উঃ", "উহ");
check("পুনঃ", "পুন");
}
public void testBasics() throws IOException {
check("কণা", "কনা");
check("শরীর", "সরির");
check("বাড়ি", "বারি");
}
/** creates random strings in the bengali block and ensures the normalizer doesn't trip up on them */
public void testRandom() throws IOException {
BengaliNormalizer normalizer = new BengaliNormalizer();
for (int i = 0; i < 100000; i++) {
String randomBengali = TestUtil.randomSimpleStringRange(random(), '\u0980', '\u09FF', 7);
try {
int newLen = normalizer.normalize(randomBengali.toCharArray(), randomBengali.length());
assertTrue(newLen >= 0); // should not return negative length
assertTrue(newLen <= randomBengali.length()); // should not increase length of string
} catch (Exception e) {
System.err.println("normalizer failed on input: '" + randomBengali + "' (" + escape(randomBengali) + ")");
throw e;
}
}
}
private void check(String input, String output) throws IOException {
Tokenizer tokenizer = whitespaceMockTokenizer(input);
TokenFilter tf = new BengaliNormalizationFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new BengaliNormalizationFilter(tokenizer));
}
};
checkOneTerm(a, "", "");
a.close();
}
}

79
lucene/analysis/common/src/test/org/apache/lucene/analysis/bn/TestBengaliStemmer.java Normal file
View File

@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.bn;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import java.io.IOException;
/**
* Test Codes for BengaliStemmer
*/
public class TestBengaliStemmer extends BaseTokenStreamTestCase {
/**
* Testing few verbal words
*/
public void testVerbsInShadhuForm() throws IOException {
check("করেছিলাম", "কর");
check("করিতেছিলে", "কর");
check("খাইতাম", "খাই");
check("যাইবে", "যা");
}
public void testVerbsInCholitoForm() throws IOException {
check("করছিলাম", "কর");
check("করছিলে", "কর");
check("করতাম", "কর");
check("যাব", "যা");
check("যাবে", "যা");
check("করি", "কর");
check("করো", "কর");
}
public void testNouns() throws IOException {
check("মেয়েরা", "মে");
check("মেয়েদেরকে", "মে");
check("মেয়েদের", "মে");
check("একটি", "এক");
check("মানুষগুলি", "মানুষ");
}
private void check(String input, String output) throws IOException {
Tokenizer tokenizer = whitespaceMockTokenizer(input);
TokenFilter tf = new BengaliStemFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new BengaliStemFilter(tokenizer));
}
};
checkOneTerm(a, "", "");
a.close();
}
}

35
lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilter.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.Reader;
import java.util.Objects;
import org.apache.lucene.analysis.CharacterUtils;
import org.apache.lucene.analysis.charfilter.BaseCharFilter;
import com.ibm.icu.text.Normalizer2;
@ -61,7 +62,7 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
ICUNormalizer2CharFilter(Reader in, Normalizer2 normalizer, int bufferSize) {
super(in);
this.normalizer = Objects.requireNonNull(normalizer);
this.tmpBuffer = new char[bufferSize];
this.tmpBuffer = CharacterUtils.newCharacterBuffer(bufferSize);
}
@Override
@ -94,23 +95,31 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
return -1;
}
private final char[] tmpBuffer;
private final CharacterUtils.CharacterBuffer tmpBuffer;
private int readInputToBuffer() throws IOException {
final int len = input.read(tmpBuffer);
if (len == -1) {
inputFinished = true;
return 0;
private void readInputToBuffer() throws IOException {
while (true) {
// CharacterUtils.fill is supplementary char aware
final boolean hasRemainingChars = CharacterUtils.fill(tmpBuffer, input);
assert tmpBuffer.getOffset() == 0;
inputBuffer.append(tmpBuffer.getBuffer(), 0, tmpBuffer.getLength());
if (hasRemainingChars == false) {
inputFinished = true;
break;
}
final int lastCodePoint = Character.codePointBefore(tmpBuffer.getBuffer(), tmpBuffer.getLength(), 0);
if (normalizer.isInert(lastCodePoint)) {
// we require an inert char so that we can normalize content before and
// after this character independently
break;
}
}
inputBuffer.append(tmpBuffer, 0, len);
// if checkedInputBoundary was at the end of a buffer, we need to check that char again
checkedInputBoundary = Math.max(checkedInputBoundary - 1, 0);
// this loop depends on 'isInert' (changes under normalization) but looks only at characters.
// so we treat all surrogates as non-inert for simplicity
if (normalizer.isInert(tmpBuffer[len - 1]) && !Character.isSurrogate(tmpBuffer[len-1])) {
return len;
} else return len + readInputToBuffer();
}
private int readAndNormalizeFromInput() {

21
lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
View File

@ -20,12 +20,14 @@ package org.apache.lucene.analysis.icu;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.ngram.NGramTokenizer;
import org.apache.lucene.util.TestUtil;
@ -418,4 +420,23 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase {
}
a.close();
}
// https://issues.apache.org/jira/browse/LUCENE-7956
public void testVeryLargeInputOfNonInertChars() throws Exception {
char[] text = new char[1000000];
Arrays.fill(text, 'a');
try (Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
return new TokenStreamComponents(new KeywordTokenizer());
}
@Override
protected Reader initReader(String fieldName, Reader reader) {
return new ICUNormalizer2CharFilter(reader, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
}
}) {
checkAnalysisConsistency(random(), a, false, new String(text));
}
}
}

4
lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
View File

@ -432,7 +432,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
"6.5.1-cfs",
"6.5.1-nocfs",
"6.6.0-cfs",
"6.6.0-nocfs"
"6.6.0-nocfs",
"6.6.1-cfs",
"6.6.1-nocfs"
};
// TODO: on 6.0.0 release, gen the single segment indices and add here:

BIN
lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.1-cfs.zip Normal file
View File

Binary file not shown.

BIN
lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.1-nocfs.zip Normal file
View File

Binary file not shown.

13
lucene/build.xml
View File

@ -428,6 +428,19 @@
</sequential>
</target>
<target name="-install-to-maven-local-repo" depends="install-maven-tasks">
<sequential>
<m2-install pom.xml="${filtered.pom.templates.dir}/pom.xml"/> <!-- Lucene/Solr grandparent POM -->
<m2-install pom.xml="${filtered.pom.templates.dir}/lucene/pom.xml"/> <!-- Lucene parent POM -->
<subant target="-install-to-maven-local-repo" failonerror="true" inheritall="false">
<propertyset refid="uptodate.and.compiled.properties"/>
<fileset dir="${common.dir}/core" includes="build.xml"/>
<fileset dir="${common.dir}/test-framework" includes="build.xml"/>
</subant>
<modules-crawl target="-install-to-maven-local-repo"/>
</sequential>
</target>
<target name="generate-maven-artifacts" depends="-unpack-lucene-tgz">
<ant dir=".." target="resolve" inheritall="false"/>
<antcall target="-filter-pom-templates" inheritall="false"/>

2
lucene/codecs/build.xml
View File

@ -23,4 +23,6 @@
<import file="../module-build.xml"/>
<target name="-dist-maven" depends="-dist-maven-src-java"/>
<target name="-install-to-maven-local-repo" depends="-install-src-java-to-maven-local-repo"/>
</project>

53
lucene/common-build.xml
View File

@ -595,7 +595,20 @@
</artifact:install>
</sequential>
</macrodef>
<macrodef name="m2-install" description="Installs a Maven artifact into the local repository">
<element name="parent-poms" optional="yes"/>
<attribute name="pom.xml"/>
<attribute name="jar.file" default="${dist.jar.dir.prefix}-${version}/${dist.jar.dir.suffix}/${final.name}.jar"/>
<sequential>
<parent-poms/>
<artifact:pom id="maven.project" file="@{pom.xml}"/>
<artifact:install file="@{jar.file}">
<pom refid="maven.project"/>
</artifact:install>
</sequential>
</macrodef>
<!-- validate maven dependencies -->
<macrodef name="m2-validate-dependencies">
<attribute name="pom.xml"/>
@ -1713,6 +1726,44 @@ ${tests-output}/junit4-*.suites - per-JVM executed suites
</sequential>
</target>
<target name="-install-to-maven-local-repo" depends="install-maven-tasks">
<sequential>
<property name="top.level.dir" location="${common.dir}/.."/>
<pathconvert property="pom.xml">
<mapper>
<chainedmapper>
<globmapper from="${top.level.dir}*" to="${filtered.pom.templates.dir}*"/>
<globmapper from="*build.xml" to="*pom.xml"/>
</chainedmapper>
</mapper>
<path location="${ant.file}"/>
</pathconvert>
<artifact:pom id="maven.project" file="${pom.xml}"/>
<artifact:install file="${dist.jar.dir.prefix}-${version}/${dist.jar.dir.suffix}/${final.name}.jar">
<pom refid="maven.project"/>
</artifact:install>
</sequential>
</target>
<target name="-install-src-java-to-maven-local-repo" depends="install-maven-tasks">
<sequential>
<property name="top.level.dir" location="${common.dir}/.."/>
<pathconvert property="pom.xml">
<mapper>
<chainedmapper>
<globmapper from="${top.level.dir}*" to="${filtered.pom.templates.dir}*"/>
<globmapper from="*build.xml" to="*/src/java/pom.xml"/>
</chainedmapper>
</mapper>
<path location="${ant.file}"/>
</pathconvert>
<artifact:pom id="maven.project" file="${pom.xml}"/>
<artifact:install file="${dist.jar.dir.prefix}-${version}/${dist.jar.dir.suffix}/${final.name}.jar">
<pom refid="maven.project"/>
</artifact:install>
</sequential>
</target>
<target name="-dist-maven-src-java" depends="install-maven-tasks, jar-src, javadocs">
<sequential>
<property name="top.level.dir" location="${common.dir}/.."/>

2
lucene/core/build.xml
View File

@ -64,6 +64,8 @@
<target name="-dist-maven" depends="-dist-maven-src-java"/>
<target name="-install-to-maven-local-repo" depends="-install-src-java-to-maven-local-repo"/>
<macrodef name="createLevAutomaton">
<attribute name="n"/>
<sequential>

2
lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
View File

@ -27,7 +27,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsConsumer;
@ -733,7 +732,6 @@ final class DefaultIndexingChain extends DocConsumer {
stream.reset();
invertState.setAttributeSource(stream);
termsHashPerField.start(field, first);
CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
while (stream.incrementToken()) {

14
lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java
View File

@ -93,15 +93,27 @@ public final class NativeFSLockFactory extends FSLockFactory {
Path lockFile = lockDir.resolve(lockName);
IOException creationException = null;
try {
Files.createFile(lockFile);
} catch (IOException ignore) {
// we must create the file to have a truly canonical path.
// if it's already created, we don't care. if it cant be created, it will fail below.
creationException = ignore;
}
// fails if the lock file does not exist
final Path realPath = lockFile.toRealPath();
final Path realPath;
try {
realPath = lockFile.toRealPath();
} catch (IOException e) {
// if we couldn't resolve the lock file, it might be because we couldn't create it.
// so append any exception from createFile as a suppressed exception, in case its useful
if (creationException != null) {
e.addSuppressed(creationException);
}
throw e;
}
// used as a best-effort check, to see if the underlying file has changed
final FileTime creationTime = Files.readAttributes(realPath, BasicFileAttributes.class).creationTime();

42
lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
View File

@ -18,9 +18,17 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.AccessDeniedException;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttribute;
import java.util.Set;
import org.apache.lucene.mockfile.FilterFileSystemProvider;
import org.apache.lucene.mockfile.FilterPath;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
@ -89,4 +97,38 @@ public class TestNativeFSLockFactory extends BaseLockFactoryTestCase {
IOUtils.closeWhileHandlingException(lock);
}
}
/** MockFileSystem that throws AccessDeniedException on creating test.lock */
static class MockBadPermissionsFileSystem extends FilterFileSystemProvider {
public MockBadPermissionsFileSystem(FileSystem delegateInstance) {
super("mockbadpermissions://", delegateInstance);
}
@Override
public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
if (path.getFileName().toString().equals("test.lock")) {
throw new AccessDeniedException(path.toString(), null, "fake access denied");
}
return super.newByteChannel(path, options, attrs);
}
}
public void testBadPermissions() throws IOException {
// create a mock filesystem that will throw exc on creating test.lock
Path tmpDir = createTempDir();
tmpDir = FilterPath.unwrap(tmpDir).toRealPath();
FileSystem mock = new MockBadPermissionsFileSystem(tmpDir.getFileSystem()).getFileSystem(null);
Path mockPath = mock.getPath(tmpDir.toString());
// we should get an IOException (typically NoSuchFileException but no guarantee) with
// our fake AccessDenied added as suppressed.
Directory dir = getDirectory(mockPath.resolve("indexDir"));
IOException expected = expectThrows(IOException.class, () -> {
dir.obtainLock("test.lock");
});
AccessDeniedException suppressed = (AccessDeniedException) expected.getSuppressed()[0];
assertTrue(suppressed.getMessage().contains("fake access denied"));
dir.close();
}
}

20
lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/LruTaxonomyWriterCache.java
View File

@ -32,8 +32,12 @@ public class LruTaxonomyWriterCache implements TaxonomyWriterCache {
* function, LRU_STRING should be used.
*/
public enum LRUType {
/** Use the label's hash as the key; this can lead to
* silent conflicts! */
/** Use only the label's 64 bit longHashCode as the hash key. Do not
* check equals, unlike most hash maps.
* Note that while these hashes are very likely to be unique, the chance
* of a collision is still greater than zero. If such an unlikely event
* occurs, your document will get an incorrect facet.
*/
LRU_HASHED,
/** Use the label as the hash key; this is always
@ -43,15 +47,15 @@ public class LruTaxonomyWriterCache implements TaxonomyWriterCache {
private NameIntCacheLRU cache;
/** Creates this with {@link LRUType#LRU_HASHED} method. */
/** Creates this with {@link LRUType#LRU_STRING} method. */
public LruTaxonomyWriterCache(int cacheSize) {
// TODO (Facet): choose between NameHashIntCacheLRU and NameIntCacheLRU.
// For guaranteed correctness - not relying on no-collisions in the hash
// function, NameIntCacheLRU should be used:
// On the other hand, NameHashIntCacheLRU takes less RAM but if there
// are collisions (which we never found) two different paths would be
// mapped to the same ordinal...
this(cacheSize, LRUType.LRU_HASHED);
// are collisions two different paths would be mapped to the same
// ordinal...
this(cacheSize, LRUType.LRU_STRING);
}
/** Creates this with the specified method. */
@ -60,8 +64,8 @@ public class LruTaxonomyWriterCache implements TaxonomyWriterCache {
// For guaranteed correctness - not relying on no-collisions in the hash
// function, NameIntCacheLRU should be used:
// On the other hand, NameHashIntCacheLRU takes less RAM but if there
// are collisions (which we never found) two different paths would be
// mapped to the same ordinal...
// are collisions two different paths would be mapped to the same
// ordinal...
if (lruType == LRUType.LRU_HASHED) {
this.cache = new NameHashIntCacheLRU(cacheSize);
} else {

50
lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestLruTaxonomyWriterCache.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.facet.taxonomy.writercache;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.taxonomy.FacetLabel;
import org.junit.Test;
public class TestLruTaxonomyWriterCache extends FacetTestCase {
@Test
public void testDefaultLRUTypeIsCollisionSafe() {
// These labels are clearly different, but have identical longHashCodes.
// Note that these labels are clearly contrived. We did encounter
// collisions in actual production data, but we aren't allowed to publish
// those.
final FacetLabel a = new FacetLabel("\0", "\u0003\uFFE2");
final FacetLabel b = new FacetLabel("\1", "\0");
// If this fails, then the longHashCode implementation has changed. This
// cannot prevent collisions. (All hashes must allow for collisions.) It
// will however stop the rest of this test from making sense. To fix, find
// new colliding labels, or make a subclass of FacetLabel that produces
// collisions.
assertEquals(a.longHashCode(), b.longHashCode());
// Make a cache with capacity > 2 so both our labels will fit. Don't
// specify an LRUType, since we want to check if the default is
// collision-safe.
final LruTaxonomyWriterCache cache = new LruTaxonomyWriterCache(10);
cache.put(a, 0);
cache.put(b, 1);
assertEquals(cache.get(a), 0);
assertEquals(cache.get(b), 1);
}
}

2
lucene/ivy-versions.properties
View File

@ -94,7 +94,7 @@ org.apache.calcite.version = 1.13.0
/org.apache.commons/commons-compress = 1.11
/org.apache.commons/commons-exec = 1.3
/org.apache.commons/commons-lang3 = 3.6
/org.apache.commons/commons-math3 = 3.4.1
/org.apache.commons/commons-math3 = 3.6.1
org.apache.curator.version = 2.8.0
/org.apache.curator/curator-client = ${org.apache.curator.version}

9
lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java
View File

@ -82,6 +82,11 @@ public class GeoBBoxFactory {
//System.err.println(" not vertical line");
if (extent >= Math.PI) {
if (Math.abs(topLat - bottomLat) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
if (Math.abs(topLat - Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
return new GeoDegeneratePoint(planetModel, topLat, 0.0);
} else if (Math.abs(bottomLat + Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
return new GeoDegeneratePoint(planetModel, bottomLat, 0.0);
}
//System.err.println(" wide degenerate line");
return new GeoWideDegenerateHorizontalLine(planetModel, topLat, leftLon, rightLon);
}
@ -94,8 +99,10 @@ public class GeoBBoxFactory {
return new GeoWideRectangle(planetModel, topLat, bottomLat, leftLon, rightLon);
}
if (Math.abs(topLat - bottomLat) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
if (Math.abs(topLat - Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION || Math.abs(topLat + Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
if (Math.abs(topLat - Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
return new GeoDegeneratePoint(planetModel, topLat, 0.0);
} else if (Math.abs(bottomLat + Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
return new GeoDegeneratePoint(planetModel, bottomLat, 0.0);
}
//System.err.println(" horizontal line");
return new GeoDegenerateHorizontalLine(planetModel, topLat, leftLon, rightLon);

788
lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java Normal file
View File

@ -0,0 +1,788 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.spatial3d.geom;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* GeoShape representing a path across the surface of the globe,
* with a specified half-width. Path is described by a series of points.
* Distances are measured from the starting point along the path, and then at right
* angles to the path.
*
* @lucene.internal
*/
class GeoDegeneratePath extends GeoBasePath {
/** The original list of path points */
protected final List<GeoPoint> points = new ArrayList<GeoPoint>();
/** A list of SegmentEndpoints */
protected List<SegmentEndpoint> endPoints;
/** A list of PathSegments */
protected List<PathSegment> segments;
/** A point on the edge */
protected GeoPoint[] edgePoints;
/** Set to true if path has been completely constructed */
protected boolean isDone = false;
/** Constructor.
*@param planetModel is the planet model.
*@param pathPoints are the points in the path.
*/
public GeoDegeneratePath(final PlanetModel planetModel, final GeoPoint[] pathPoints) {
this(planetModel);
Collections.addAll(points, pathPoints);
done();
}
/** Piece-wise constructor. Use in conjunction with addPoint() and done().
*@param planetModel is the planet model.
*/
public GeoDegeneratePath(final PlanetModel planetModel) {
super(planetModel);
}
/** Add a point to the path.
*@param lat is the latitude of the point.
*@param lon is the longitude of the point.
*/
public void addPoint(final double lat, final double lon) {
if (isDone)
throw new IllegalStateException("Can't call addPoint() if done() already called");
points.add(new GeoPoint(planetModel, lat, lon));
}
/** Complete the path.
*/
public void done() {
if (isDone)
throw new IllegalStateException("Can't call done() twice");
if (points.size() == 0)
throw new IllegalArgumentException("Path must have at least one point");
isDone = true;
endPoints = new ArrayList<>(points.size());
segments = new ArrayList<>(points.size());
// First, build all segments. We'll then go back and build corresponding segment endpoints.
GeoPoint lastPoint = null;
for (final GeoPoint end : points) {
if (lastPoint != null) {
final Plane normalizedConnectingPlane = new Plane(lastPoint, end);
if (normalizedConnectingPlane == null) {
continue;
}
segments.add(new PathSegment(planetModel, lastPoint, end, normalizedConnectingPlane));
}
lastPoint = end;
}
if (segments.size() == 0) {
// Simple circle
final GeoPoint point = points.get(0);
final SegmentEndpoint onlyEndpoint = new SegmentEndpoint(point);
endPoints.add(onlyEndpoint);
this.edgePoints = new GeoPoint[]{point};
return;
}
// Create segment endpoints. Use an appropriate constructor for the start and end of the path.
for (int i = 0; i < segments.size(); i++) {
final PathSegment currentSegment = segments.get(i);
if (i == 0) {
// Starting endpoint
final SegmentEndpoint startEndpoint = new SegmentEndpoint(currentSegment.start,
currentSegment.startCutoffPlane);
endPoints.add(startEndpoint);
this.edgePoints = new GeoPoint[]{currentSegment.start};
continue;
}
endPoints.add(new SegmentEndpoint(currentSegment.start,
segments.get(i-1).endCutoffPlane,
currentSegment.startCutoffPlane));
}
// Do final endpoint
final PathSegment lastSegment = segments.get(segments.size()-1);
endPoints.add(new SegmentEndpoint(lastSegment.end,
lastSegment.endCutoffPlane));
}
/**
* Constructor for deserialization.
* @param planetModel is the planet model.
* @param inputStream is the input stream.
*/
public GeoDegeneratePath(final PlanetModel planetModel, final InputStream inputStream) throws IOException {
this(planetModel,
SerializableObject.readPointArray(planetModel, inputStream));
}
@Override
public void write(final OutputStream outputStream) throws IOException {
SerializableObject.writePointArray(outputStream, points);
}
@Override
public double computePathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
// Walk along path and keep track of the closest distance we find
double closestDistance = Double.POSITIVE_INFINITY;
// Segments first
for (PathSegment segment : segments) {
final double segmentDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
if (segmentDistance < closestDistance) {
closestDistance = segmentDistance;
}
}
// Now, endpoints
for (SegmentEndpoint endpoint : endPoints) {
final double endpointDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
if (endpointDistance < closestDistance) {
closestDistance = endpointDistance;
}
}
return closestDistance;
}
@Override
public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
double currentDistance = 0.0;
double minPathCenterDistance = Double.POSITIVE_INFINITY;
double bestDistance = Double.POSITIVE_INFINITY;
int segmentIndex = 0;
for (SegmentEndpoint endpoint : endPoints) {
final double endpointPathCenterDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
if (endpointPathCenterDistance < minPathCenterDistance) {
// Use this endpoint
minPathCenterDistance = endpointPathCenterDistance;
bestDistance = currentDistance;
}
// Look at the following segment, if any
if (segmentIndex < segments.size()) {
final PathSegment segment = segments.get(segmentIndex++);
final double segmentPathCenterDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
if (segmentPathCenterDistance < minPathCenterDistance) {
minPathCenterDistance = segmentPathCenterDistance;
bestDistance = distanceStyle.aggregateDistances(currentDistance, segment.nearestPathDistance(planetModel, distanceStyle, x, y, z));
}
currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
}
}
return bestDistance;
}
@Override
protected double distance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
// Algorithm:
// (1) If the point is within any of the segments along the path, return that value.
// (2) If the point is within any of the segment end circles along the path, return that value.
double currentDistance = 0.0;
for (PathSegment segment : segments) {
double distance = segment.pathDistance(planetModel, distanceStyle, x,y,z);
if (distance != Double.POSITIVE_INFINITY)
return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
}
int segmentIndex = 0;
currentDistance = 0.0;
for (SegmentEndpoint endpoint : endPoints) {
double distance = endpoint.pathDistance(distanceStyle, x, y, z);
if (distance != Double.POSITIVE_INFINITY)
return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
if (segmentIndex < segments.size())
currentDistance = distanceStyle.aggregateDistances(currentDistance, segments.get(segmentIndex++).fullPathDistance(distanceStyle));
}
return Double.POSITIVE_INFINITY;
}
@Override
protected void distanceBounds(final Bounds bounds, final DistanceStyle distanceStyle, final double distanceValue) {
// TBD: Compute actual bounds based on distance
getBounds(bounds);
}
@Override
protected double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
double minDistance = Double.POSITIVE_INFINITY;
for (final SegmentEndpoint endpoint : endPoints) {
final double newDistance = endpoint.outsideDistance(distanceStyle, x,y,z);
if (newDistance < minDistance)
minDistance = newDistance;
}
for (final PathSegment segment : segments) {
final double newDistance = segment.outsideDistance(planetModel, distanceStyle, x, y, z);
if (newDistance < minDistance)
minDistance = newDistance;
}
return minDistance;
}
@Override
public boolean isWithin(final double x, final double y, final double z) {
for (SegmentEndpoint pathPoint : endPoints) {
if (pathPoint.isWithin(x, y, z)) {
return true;
}
}
for (PathSegment pathSegment : segments) {
if (pathSegment.isWithin(x, y, z)) {
return true;
}
}
return false;
}
@Override
public GeoPoint[] getEdgePoints() {
return edgePoints;
}
@Override
public boolean intersects(final Plane plane, final GeoPoint[] notablePoints, final Membership... bounds) {
// We look for an intersection with any of the exterior edges of the path.
// We also have to look for intersections with the cones described by the endpoints.
// Return "true" if any such intersections are found.
// For plane intersections, the basic idea is to come up with an equation of the line that is
// the intersection (if any). Then, find the intersections with the unit sphere (if any). If
// any of the intersection points are within the bounds, then we've detected an intersection.
// Well, sort of. We can detect intersections also due to overlap of segments with each other.
// But that's an edge case and we won't be optimizing for it.
//System.err.println(" Looking for intersection of plane "+plane+" with path "+this);
// Since the endpoints are included in the path segments, we only need to do this if there are
// no path segments
if (endPoints.size() == 1) {
return endPoints.get(0).intersects(planetModel, plane, notablePoints, bounds);
}
for (final PathSegment pathSegment : segments) {
if (pathSegment.intersects(planetModel, plane, notablePoints, bounds)) {
return true;
}
}
return false;
}
@Override
public boolean intersects(GeoShape geoShape) {
// Since the endpoints are included in the path segments, we only need to do this if there are
// no path segments
if (endPoints.size() == 1) {
return endPoints.get(0).intersects(geoShape);
}
for (final PathSegment pathSegment : segments) {
if (pathSegment.intersects(geoShape)) {
return true;
}
}
return false;
}
@Override
public void getBounds(Bounds bounds) {
super.getBounds(bounds);
// For building bounds, order matters. We want to traverse
// never more than 180 degrees longitude at a pop or we risk having the
// bounds object get itself inverted. So do the edges first.
for (PathSegment pathSegment : segments) {
pathSegment.getBounds(planetModel, bounds);
}
if (endPoints.size() == 1) {
endPoints.get(0).getBounds(planetModel, bounds);
}
}
@Override
public boolean equals(Object o) {
if (!(o instanceof GeoDegeneratePath))
return false;
GeoDegeneratePath p = (GeoDegeneratePath) o;
if (!super.equals(p))
return false;
return points.equals(p.points);
}
@Override
public int hashCode() {
int result = super.hashCode();
result = 31 * result + points.hashCode();
return result;
}
@Override
public String toString() {
return "GeoDegeneratePath: {planetmodel=" + planetModel+", points={" + points + "}}";
}
/**
* This is precalculated data for segment endpoint. Since the path is degenerate, there are several different cases:
* (1) The path consists of a single endpoint. In this case, the degenerate path consists of this one point.
* (2) This is the end of a path. There is a bounding plane passed in which describes the part of the world that is considered
* to belong to this endpoint.
* (3) Intersection. There are two cutoff planes, one for each end of the intersection.
*/
private static class SegmentEndpoint {
/** The center point of the endpoint */
public final GeoPoint point;
/** Pertinent cutoff planes from adjoining segments */
public final Membership[] cutoffPlanes;
/** Notable points for this segment endpoint */
public final GeoPoint[] notablePoints;
/** No notable points from the circle itself */
public final static GeoPoint[] circlePoints = new GeoPoint[0];
/** Null membership */
public final static Membership[] NO_MEMBERSHIP = new Membership[0];
/** Constructor for case (1).
*@param point is the center point.
*/
public SegmentEndpoint(final GeoPoint point) {
this.point = point;
this.cutoffPlanes = NO_MEMBERSHIP;
this.notablePoints = circlePoints;
}
/** Constructor for case (2).
* Generate an endpoint, given a single cutoff plane plus upper and lower edge points.
*@param point is the center point.
*@param cutoffPlane is the plane from the adjoining path segment marking the boundary between this endpoint and that segment.
*/
public SegmentEndpoint(final GeoPoint point, final SidedPlane cutoffPlane) {
this.point = point;
this.cutoffPlanes = new Membership[]{new SidedPlane(cutoffPlane)};
this.notablePoints = new GeoPoint[]{point};
}
/** Constructor for case (3).
* Generate an endpoint, given two cutoff planes.
*@param point is the center.
*@param cutoffPlane1 is one adjoining path segment cutoff plane.
*@param cutoffPlane2 is another adjoining path segment cutoff plane.
*/
public SegmentEndpoint(final GeoPoint point,
final SidedPlane cutoffPlane1, final SidedPlane cutoffPlane2) {
this.point = point;
this.cutoffPlanes = new Membership[]{new SidedPlane(cutoffPlane1), new SidedPlane(cutoffPlane2)};
this.notablePoints = new GeoPoint[]{point};
}
/** Check if point is within this endpoint.
*@param point is the point.
*@return true of within.
*/
public boolean isWithin(final Vector point) {
return this.point.isIdentical(point.x, point.y, point.z);
}
/** Check if point is within this endpoint.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return true of within.
*/
public boolean isWithin(final double x, final double y, final double z) {
return this.point.isIdentical(x, y, z);
}
/** Compute interior path distance.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric, in aggregation form.
*/
public double pathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
if (!isWithin(x,y,z))
return Double.POSITIVE_INFINITY;
return distanceStyle.toAggregationForm(distanceStyle.computeDistance(this.point, x, y, z));
}
/** Compute nearest path distance.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric (always value zero), in aggregation form, or POSITIVE_INFINITY
* if the point is not within the bounds of the endpoint.
*/
public double nearestPathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
for (final Membership m : cutoffPlanes) {
if (!m.isWithin(x,y,z)) {
return Double.POSITIVE_INFINITY;
}
}
return distanceStyle.toAggregationForm(0.0);
}
/** Compute path center distance.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric, or POSITIVE_INFINITY
* if the point is not within the bounds of the endpoint.
*/
public double pathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
for (final Membership m : cutoffPlanes) {
if (!m.isWithin(x,y,z)) {
return Double.POSITIVE_INFINITY;
}
}
return distanceStyle.computeDistance(this.point, x, y, z);
}
/** Compute external distance.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric.
*/
public double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
return distanceStyle.computeDistance(this.point, x, y, z);
}
/** Determine if this endpoint intersects a specified plane.
*@param planetModel is the planet model.
*@param p is the plane.
*@param notablePoints are the points associated with the plane.
*@param bounds are any bounds which the intersection must lie within.
*@return true if there is a matching intersection.
*/
public boolean intersects(final PlanetModel planetModel, final Plane p, final GeoPoint[] notablePoints, final Membership[] bounds) {
// If not on the plane, no intersection
if (!p.evaluateIsZero(point))
return false;
for (Membership m : bounds) {
if (!m.isWithin(point))
return false;
}
return true;
}
/** Determine if this endpoint intersects a GeoShape.
*@param geoShape is the GeoShape.
*@return true if there is shape intersect this endpoint.
*/
public boolean intersects(final GeoShape geoShape) {
return geoShape.isWithin(point);
}
/** Get the bounds for a segment endpoint.
*@param planetModel is the planet model.
*@param bounds are the bounds to be modified.
*/
public void getBounds(final PlanetModel planetModel, Bounds bounds) {
bounds.addPoint(point);
}
@Override
public boolean equals(Object o) {
if (!(o instanceof SegmentEndpoint))
return false;
SegmentEndpoint other = (SegmentEndpoint) o;
return point.equals(other.point);
}
@Override
public int hashCode() {
return point.hashCode();
}
@Override
public String toString() {
return point.toString();
}
}
/**
* This is the pre-calculated data for a path segment.
*/
private static class PathSegment {
/** Starting point of the segment */
public final GeoPoint start;
/** End point of the segment */
public final GeoPoint end;
/** Place to keep any complete segment distances we've calculated so far */
public final Map<DistanceStyle,Double> fullDistanceCache = new HashMap<DistanceStyle,Double>();
/** Normalized plane connecting the two points and going through world center */
public final Plane normalizedConnectingPlane;
/** Plane going through the center and start point, marking the start edge of the segment */
public final SidedPlane startCutoffPlane;
/** Plane going through the center and end point, marking the end edge of the segment */
public final SidedPlane endCutoffPlane;
/** Notable points for the connecting plane */
public final GeoPoint[] connectingPlanePoints;
/** Construct a path segment.
*@param planetModel is the planet model.
*@param start is the starting point.
*@param end is the ending point.
*@param normalizedConnectingPlane is the connecting plane.
*/
public PathSegment(final PlanetModel planetModel, final GeoPoint start, final GeoPoint end,
final Plane normalizedConnectingPlane) {
this.start = start;
this.end = end;
this.normalizedConnectingPlane = normalizedConnectingPlane;
// Cutoff planes use opposite endpoints as correct side examples
startCutoffPlane = new SidedPlane(end, normalizedConnectingPlane, start);
endCutoffPlane = new SidedPlane(start, normalizedConnectingPlane, end);
connectingPlanePoints = new GeoPoint[]{start, end};
}
/** Compute the full distance along this path segment.
*@param distanceStyle is the distance style.
*@return the distance metric, in aggregation form.
*/
public double fullPathDistance(final DistanceStyle distanceStyle) {
synchronized (fullDistanceCache) {
Double dist = fullDistanceCache.get(distanceStyle);
if (dist == null) {
dist = new Double(distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, end.x, end.y, end.z)));
fullDistanceCache.put(distanceStyle, dist);
}
return dist.doubleValue();
}
}
/** Check if point is within this segment.
*@param point is the point.
*@return true of within.
*/
public boolean isWithin(final Vector point) {
return startCutoffPlane.isWithin(point) &&
endCutoffPlane.isWithin(point) &&
normalizedConnectingPlane.evaluateIsZero(point);
}
/** Check if point is within this segment.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return true of within.
*/
public boolean isWithin(final double x, final double y, final double z) {
return startCutoffPlane.isWithin(x, y, z) &&
endCutoffPlane.isWithin(x, y, z) &&
normalizedConnectingPlane.evaluateIsZero(x, y, z);
}
/** Compute path center distance.
*@param planetModel is the planet model.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric, or Double.POSITIVE_INFINITY if outside this segment
*/
public double pathCenterDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
// First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
return Double.POSITIVE_INFINITY;
}
// (1) Compute normalizedPerpPlane. If degenerate, then there is no such plane, which means that the point given
// is insufficient to distinguish between a family of such planes. This can happen only if the point is one of the
// "poles", imagining the normalized plane to be the "equator". In that case, the distance returned should be zero.
// Want no allocations or expensive operations! so we do this the hard way
final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
return distanceStyle.computeDistance(start, x, y, z);
final double normFactor = 1.0/magnitude;
final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
GeoPoint thePoint;
if (intersectionPoints.length == 0)
throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
else if (intersectionPoints.length == 1)
thePoint = intersectionPoints[0];
else {
if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
thePoint = intersectionPoints[0];
else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
thePoint = intersectionPoints[1];
else
throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
}
return distanceStyle.computeDistance(thePoint, x, y, z);
}
/** Compute nearest path distance.
*@param planetModel is the planet model.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric, in aggregation form, or Double.POSITIVE_INFINITY if outside this segment
*/
public double nearestPathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
// First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
return Double.POSITIVE_INFINITY;
}
// (1) Compute normalizedPerpPlane. If degenerate, then there is no such plane, which means that the point given
// is insufficient to distinguish between a family of such planes. This can happen only if the point is one of the
// "poles", imagining the normalized plane to be the "equator". In that case, the distance returned should be zero.
// Want no allocations or expensive operations! so we do this the hard way
final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
return distanceStyle.toAggregationForm(0.0);
final double normFactor = 1.0/magnitude;
final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
GeoPoint thePoint;
if (intersectionPoints.length == 0)
throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
else if (intersectionPoints.length == 1)
thePoint = intersectionPoints[0];
else {
if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
thePoint = intersectionPoints[0];
else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
thePoint = intersectionPoints[1];
else
throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
}
return distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z));
}
/** Compute interior path distance.
*@param planetModel is the planet model.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric, in aggregation form.
*/
public double pathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
if (!isWithin(x,y,z))
return Double.POSITIVE_INFINITY;
// (1) Compute normalizedPerpPlane. If degenerate, then return point distance from start to point.
// Want no allocations or expensive operations! so we do this the hard way
final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
return distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, x,y,z));
final double normFactor = 1.0/magnitude;
final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
// Old computation: too expensive, because it calculates the intersection point twice.
//return distanceStyle.computeDistance(planetModel, normalizedConnectingPlane, x, y, z, startCutoffPlane, endCutoffPlane) +
// distanceStyle.computeDistance(planetModel, normalizedPerpPlane, start.x, start.y, start.z, upperConnectingPlane, lowerConnectingPlane);
final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
GeoPoint thePoint;
if (intersectionPoints.length == 0)
throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
else if (intersectionPoints.length == 1)
thePoint = intersectionPoints[0];
else {
if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
thePoint = intersectionPoints[0];
else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
thePoint = intersectionPoints[1];
else
throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
}
return distanceStyle.aggregateDistances(distanceStyle.toAggregationForm(distanceStyle.computeDistance(thePoint, x, y, z)),
distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z)));
}
/** Compute external distance.
*@param planetModel is the planet model.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric.
*/
public double outsideDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
final double distance = distanceStyle.computeDistance(planetModel, normalizedConnectingPlane, x,y,z, startCutoffPlane, endCutoffPlane);
final double startDistance = distanceStyle.computeDistance(start, x,y,z);
final double endDistance = distanceStyle.computeDistance(end, x,y,z);
return Math.min(
Math.min(startDistance, endDistance),
distance);
}
/** Determine if this endpoint intersects a specified plane.
*@param planetModel is the planet model.
*@param p is the plane.
*@param notablePoints are the points associated with the plane.
*@param bounds are any bounds which the intersection must lie within.
*@return true if there is a matching intersection.
*/
public boolean intersects(final PlanetModel planetModel, final Plane p, final GeoPoint[] notablePoints, final Membership[] bounds) {
return normalizedConnectingPlane.intersects(planetModel, p, connectingPlanePoints, notablePoints, bounds, startCutoffPlane, endCutoffPlane);
}
/** Determine if this endpoint intersects a specified GeoShape.
*@param geoShape is the GeoShape.
*@return true if there GeoShape intersects this endpoint.
*/
public boolean intersects(final GeoShape geoShape) {
return geoShape.intersects(normalizedConnectingPlane, connectingPlanePoints, startCutoffPlane, endCutoffPlane);
}
/** Get the bounds for a segment endpoint.
*@param planetModel is the planet model.
*@param bounds are the bounds to be modified.
*/
public void getBounds(final PlanetModel planetModel, Bounds bounds) {
// We need to do all bounding planes as well as corner points
bounds.addPoint(start).addPoint(end)
.addPlane(planetModel, normalizedConnectingPlane, startCutoffPlane, endCutoffPlane);
}
}
}

2
lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java
View File

@ -125,7 +125,7 @@ class GeoDegeneratePoint extends GeoPoint implements GeoPointShape {
@Override
public boolean isWithin(final double x, final double y, final double z) {
return x == this.x && y == this.y && z == this.z;
return this.isIdentical(x, y, z);
}
@Override

58
lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java
View File

@ -22,4 +22,62 @@ package org.apache.lucene.spatial3d.geom;
* @lucene.experimental
*/
public interface GeoPath extends GeoDistanceShape {
// The following methods compute distances along the path from the shape to a point
// that doesn't need to be inside the shape. The perpendicular distance from the path
// itself to the point is not included in the calculation.
/**
* Compute the nearest path distance to the GeoPoint.
* The path distance will not include the distance from the path itself to the
* point, but just the distance along the path to the nearest point on the path.
*
* @param distanceStyle is the distance style.
* @param point is the point to compute the distance to.
* @return the distance to the nearest path point.
*/
public default double computeNearestDistance(final DistanceStyle distanceStyle, final GeoPoint point) {
return computeNearestDistance(distanceStyle, point.x, point.y, point.z);
}
/**
* Compute the nearest path distance to the GeoPoint.
* The path distance will not include the distance from the path itself to the
* point, but just the distance along the path to the nearest point on the path.
*
* @param x is the point's unit x coordinate (using U.S. convention).
* @param y is the point's unit y coordinate (using U.S. convention).
* @param z is the point's unit z coordinate (using U.S. convention).
* @return the distance to the nearest path point.
*/
public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z);
// The following methods compute the best distance from the path center to the point.
/**
* Compute the shortest distance from the path center to the GeoPoint.
* The distance is meant to allow comparisons between different
* paths to find the one that goes closest to a point.
*
* @param distanceStyle is the distance style.
* @param point is the point to compute the distance to.
* @return the shortest distance from the path center to the point.
*/
public default double computePathCenterDistance(final DistanceStyle distanceStyle, final GeoPoint point) {
return computePathCenterDistance(distanceStyle, point.x, point.y, point.z);
}
/**
* Compute the shortest distance from the path center to the GeoPoint.
* The distance is meant to allow comparisons between different
* paths to find the one that goes closest to a point.
*
* @param distanceStyle is the distance style.
* @param x is the point's unit x coordinate (using U.S. convention).
* @param y is the point's unit y coordinate (using U.S. convention).
* @param z is the point's unit z coordinate (using U.S. convention).
* @return the shortest distance from the path center to the point.
*/
public double computePathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z);
}

3
lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java
View File

@ -33,6 +33,9 @@ public class GeoPathFactory {
* @return a GeoPath corresponding to what was specified.
*/
public static GeoPath makeGeoPath(final PlanetModel planetModel, final double maxCutoffAngle, final GeoPoint[] pathPoints) {
if (maxCutoffAngle < Vector.MINIMUM_ANGULAR_RESOLUTION) {
return new GeoDegeneratePath(planetModel, pathPoints);
}
return new GeoStandardPath(planetModel, maxCutoffAngle, pathPoints);
}

170
lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java
View File

@ -214,6 +214,55 @@ class GeoStandardPath extends GeoBasePath {
SerializableObject.writePointArray(outputStream, points);
}
@Override
public double computePathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
// Walk along path and keep track of the closest distance we find
double closestDistance = Double.POSITIVE_INFINITY;
// Segments first
for (PathSegment segment : segments) {
final double segmentDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
if (segmentDistance < closestDistance) {
closestDistance = segmentDistance;
}
}
// Now, endpoints
for (SegmentEndpoint endpoint : endPoints) {
final double endpointDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
if (endpointDistance < closestDistance) {
closestDistance = endpointDistance;
}
}
return closestDistance;
}
@Override
public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
double currentDistance = 0.0;
double minPathCenterDistance = Double.POSITIVE_INFINITY;
double bestDistance = Double.POSITIVE_INFINITY;
int segmentIndex = 0;
for (SegmentEndpoint endpoint : endPoints) {
final double endpointPathCenterDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
if (endpointPathCenterDistance < minPathCenterDistance) {
// Use this endpoint
minPathCenterDistance = endpointPathCenterDistance;
bestDistance = currentDistance;
}
// Look at the following segment, if any
if (segmentIndex < segments.size()) {
final PathSegment segment = segments.get(segmentIndex++);
final double segmentPathCenterDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
if (segmentPathCenterDistance < minPathCenterDistance) {
minPathCenterDistance = segmentPathCenterDistance;
bestDistance = distanceStyle.aggregateDistances(currentDistance, segment.nearestPathDistance(planetModel, distanceStyle, x, y, z));
}
currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
}
}
return bestDistance;
}
@Override
protected double distance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
// Algorithm:
@ -562,6 +611,40 @@ class GeoStandardPath extends GeoBasePath {
return distanceStyle.toAggregationForm(distanceStyle.computeDistance(this.point, x, y, z));
}
/** Compute nearest path distance.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric (always value zero), in aggregation form, or POSITIVE_INFINITY
* if the point is not within the bounds of the endpoint.
*/
public double nearestPathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
for (final Membership m : cutoffPlanes) {
if (!m.isWithin(x,y,z)) {
return Double.POSITIVE_INFINITY;
}
}
return distanceStyle.toAggregationForm(0.0);
}
/** Compute path center distance.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric, or POSITIVE_INFINITY
* if the point is not within the bounds of the endpoint.
*/
public double pathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
for (final Membership m : cutoffPlanes) {
if (!m.isWithin(x,y,z)) {
return Double.POSITIVE_INFINITY;
}
}
return distanceStyle.computeDistance(this.point, x, y, z);
}
/** Compute external distance.
*@param distanceStyle is the distance style.
*@param x is the point x.
@ -766,6 +849,93 @@ class GeoStandardPath extends GeoBasePath {
lowerConnectingPlane.isWithin(x, y, z);
}
/** Compute path center distance.
*@param planetModel is the planet model.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric, or Double.POSITIVE_INFINITY if outside this segment
*/
public double pathCenterDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
// First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
return Double.POSITIVE_INFINITY;
}
// (1) Compute normalizedPerpPlane. If degenerate, then there is no such plane, which means that the point given
// is insufficient to distinguish between a family of such planes. This can happen only if the point is one of the
// "poles", imagining the normalized plane to be the "equator". In that case, the distance returned should be zero.
// Want no allocations or expensive operations! so we do this the hard way
final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
return distanceStyle.computeDistance(start, x, y, z);
final double normFactor = 1.0/magnitude;
final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
GeoPoint thePoint;
if (intersectionPoints.length == 0)
throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
else if (intersectionPoints.length == 1)
thePoint = intersectionPoints[0];
else {
if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
thePoint = intersectionPoints[0];
else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
thePoint = intersectionPoints[1];
else
throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
}
return distanceStyle.computeDistance(thePoint, x, y, z);
}
/** Compute nearest path distance.
*@param planetModel is the planet model.
*@param distanceStyle is the distance style.
*@param x is the point x.
*@param y is the point y.
*@param z is the point z.
*@return the distance metric, in aggregation form, or Double.POSITIVE_INFINITY if outside this segment
*/
public double nearestPathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
// First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
return Double.POSITIVE_INFINITY;
}
// (1) Compute normalizedPerpPlane. If degenerate, then there is no such plane, which means that the point given
// is insufficient to distinguish between a family of such planes. This can happen only if the point is one of the
// "poles", imagining the normalized plane to be the "equator". In that case, the distance returned should be zero.
// Want no allocations or expensive operations! so we do this the hard way
final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
return distanceStyle.toAggregationForm(0.0);
final double normFactor = 1.0/magnitude;
final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
GeoPoint thePoint;
if (intersectionPoints.length == 0)
throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
else if (intersectionPoints.length == 1)
thePoint = intersectionPoints[0];
else {
if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
thePoint = intersectionPoints[0];
else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
thePoint = intersectionPoints[1];
else
throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
}
return distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z));
}
/** Compute interior path distance.
*@param planetModel is the planet model.
*@param distanceStyle is the distance style.

1
lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardObjects.java
View File

@ -73,6 +73,7 @@ class StandardObjects {
classRegsitry.put(XYdZSolid.class, 33);
classRegsitry.put(StandardXYZSolid.class, 34);
classRegsitry.put(PlanetModel.class, 35);
classRegsitry.put(GeoDegeneratePath.class, 36);
for (Class<?> clazz : classRegsitry.keySet()){
codeRegsitry.put(classRegsitry.get(clazz), clazz);

75
lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java
View File

@ -56,8 +56,10 @@ public class GeoPathTest {
p.done();
gp = new GeoPoint(PlanetModel.SPHERE, 0.05, 0.15);
assertEquals(0.15 + 0.05, p.computeDistance(DistanceStyle.ARC,gp), 0.000001);
assertEquals(0.15, p.computeNearestDistance(DistanceStyle.ARC,gp), 0.000001);
gp = new GeoPoint(PlanetModel.SPHERE, 0.0, 0.12);
assertEquals(0.12, p.computeDistance(DistanceStyle.ARC,gp), 0.000001);
assertEquals(0.12, p.computeNearestDistance(DistanceStyle.ARC,gp), 0.000001);
// Now try a vertical path, and make sure distances are as expected
p = new GeoStandardPath(PlanetModel.SPHERE, 0.1);
@ -302,4 +304,77 @@ public class GeoPathTest {
assertTrue(solid.isWithin(point));
}
@Test
public void testInterpolation() {
final double lat = 52.51607;
final double lon = 13.37698;
final double[] pathLats = new double[] {52.5355,52.54,52.5626,52.5665,52.6007,52.6135,52.6303,52.6651,52.7074};
final double[] pathLons = new double[] {13.3634,13.3704,13.3307,13.3076,13.2806,13.2484,13.2406,13.241,13.1926};
// Set up a point in the right way
final GeoPoint carPoint = new GeoPoint(PlanetModel.SPHERE, toRadians(lat), toRadians(lon));
// Create the path, but use a tiny width (e.g. zero)
final GeoPoint[] pathPoints = new GeoPoint[pathLats.length];
for (int i = 0; i < pathPoints.length; i++) {
pathPoints[i] = new GeoPoint(PlanetModel.SPHERE, toRadians(pathLats[i]), toRadians(pathLons[i]));
}
// Construct a path with no width
final GeoPath thisPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 0.0, pathPoints);
// Construct a path with a width
final GeoPath legacyPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 1e-6, pathPoints);
// Compute the inside distance to the atPoint using zero-width path
final double distance = thisPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
// Compute the inside distance using legacy path
final double legacyDistance = legacyPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
// Compute the inside distance using the legacy formula
final double oldFormulaDistance = thisPath.computeDistance(DistanceStyle.ARC, carPoint);
// Compute the inside distance using the legacy formula with the legacy shape
final double oldFormulaLegacyDistance = legacyPath.computeDistance(DistanceStyle.ARC, carPoint);
// These should be about the same
assertEquals(legacyDistance, distance, 1e-12);
assertEquals(oldFormulaLegacyDistance, oldFormulaDistance, 1e-12);
// This isn't true because example search center is off of the path.
//assertEquals(oldFormulaDistance, distance, 1e-12);
}
@Test
public void testInterpolation2() {
final double lat = 52.5665;
final double lon = 13.3076;
final double[] pathLats = new double[] {52.5355,52.54,52.5626,52.5665,52.6007,52.6135,52.6303,52.6651,52.7074};
final double[] pathLons = new double[] {13.3634,13.3704,13.3307,13.3076,13.2806,13.2484,13.2406,13.241,13.1926};
final GeoPoint carPoint = new GeoPoint(PlanetModel.SPHERE, toRadians(lat), toRadians(lon));
final GeoPoint[] pathPoints = new GeoPoint[pathLats.length];
for (int i = 0; i < pathPoints.length; i++) {
pathPoints[i] = new GeoPoint(PlanetModel.SPHERE, toRadians(pathLats[i]), toRadians(pathLons[i]));
}
// Construct a path with no width
final GeoPath thisPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 0.0, pathPoints);
// Construct a path with a width
final GeoPath legacyPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 1e-6, pathPoints);
// Compute the inside distance to the atPoint using zero-width path
final double distance = thisPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
// Compute the inside distance using legacy path
final double legacyDistance = legacyPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
// Compute the inside distance using the legacy formula
final double oldFormulaDistance = thisPath.computeDistance(DistanceStyle.ARC, carPoint);
// Compute the inside distance using the legacy formula with the legacy shape
final double oldFormulaLegacyDistance = legacyPath.computeDistance(DistanceStyle.ARC, carPoint);
// These should be about the same
assertEquals(legacyDistance, distance, 1e-12);
assertEquals(oldFormulaLegacyDistance, oldFormulaDistance, 1e-12);
// Since the point we picked is actually on the path, this should also be true
assertEquals(oldFormulaDistance, distance, 1e-12);
}
}

10
lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomBinaryCodecTest.java
View File

@ -27,18 +27,18 @@ import org.junit.Test;
/**
* Test to check Serialization
*/
public class RandomBinaryCodecTest extends RandomGeoShapeGenerator{
public class RandomBinaryCodecTest extends RandomGeo3dShapeGenerator {
@Test
@Repeat(iterations = 10)
public void testRandomPointCodec() throws IOException{
PlanetModel planetModel = randomPlanetModel();
GeoPoint shape = randomGeoPoint(planetModel, getEmptyConstraint());
GeoPoint shape = randomGeoPoint(planetModel);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
SerializableObject.writeObject(outputStream, shape);
ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
SerializableObject shapeCopy = SerializableObject.readObject(planetModel, inputStream);
assertEquals(shape, shapeCopy);
assertEquals(shape.toString(), shape, shapeCopy);
}
@Test
@ -51,7 +51,7 @@ public class RandomBinaryCodecTest extends RandomGeoShapeGenerator{
SerializableObject.writePlanetObject(outputStream, shape);
ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
SerializableObject shapeCopy = SerializableObject.readPlanetObject(inputStream);
assertEquals(shape, shapeCopy);
assertEquals(shape.toString(), shape, shapeCopy);
}
@Test
@ -64,6 +64,6 @@ public class RandomBinaryCodecTest extends RandomGeoShapeGenerator{
SerializableObject.writeObject(outputStream, shape);
ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
SerializableObject shapeCopy = SerializableObject.readObject(planetModel, inputStream);
assertEquals(shape, shapeCopy);
assertEquals(shape.toString(), shape, shapeCopy);
}
}

169
lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeGenerator.java → lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java
View File

@ -36,7 +36,7 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomDouble;
* created using GeoConvexPolygons and GeoConcavePolygons.
*
*/
public class RandomGeoShapeGenerator extends LuceneTestCase {
public class RandomGeo3dShapeGenerator extends LuceneTestCase {
/* Max num of iterations to find right shape under given constrains */
final private static int MAX_SHAPE_ITERATIONS = 50;
@ -53,19 +53,31 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
final protected static int RECTANGLE = 6;
final protected static int PATH = 7;
final protected static int COLLECTION = 8;
final protected static int POINT = 9;
final protected static int LINE = 10;
/* Helper shapes for generating constraints whch are just three sided polygons */
final protected static int CONVEX_SIMPLE_POLYGON = 500;
final protected static int CONCAVE_SIMPLE_POLYGON = 501;
/**
* Method that returns empty Constraints object..
* Method that returns a random generated Planet model from the supported
* Planet models. currently SPHERE and WGS84
*
* @return an empty Constraints object
* @return a random generated Planet model
*/
public Constraints getEmptyConstraint(){
return new Constraints();
public PlanetModel randomPlanetModel() {
final int shapeType = random().nextInt(2);
switch (shapeType) {
case 0: {
return PlanetModel.SPHERE;
}
case 1: {
return PlanetModel.WGS84;
}
default:
throw new IllegalStateException("Unexpected planet model");
}
}
/**
@ -75,7 +87,20 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
* @return a random generated shape code
*/
public int randomShapeType(){
return random().nextInt(9);
return random().nextInt(11);
}
/**
* Method that returns a random generated GeoAreaShape code from all
* supported GeoAreaShapes.
*
* We are removing Collections because it is difficult to create shapes
* with properties in some cases.
*
* @return a random generated polygon code
*/
public int randomGeoAreaShapeType(){
return random().nextInt(11);
}
/**
@ -106,19 +131,6 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
return shapeType;
}
/**
* Method that returns a random generated GeoAreaShape code from all
* supported GeoAreaShapes.
*
* We are removing Collections because it is difficult to create shapes
* with properties in some cases.
*
* @return a random generated polygon code
*/
public int randomGeoAreaShapeType(){
return random().nextInt(8);
}
/**
* Check if a shape code represents a concave shape
*
@ -129,23 +141,26 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
}
/**
* Method that returns a random generated Planet model from the supported
* Planet models. currently SPHERE and WGS84
* Method that returns empty Constraints object..
*
* @return a random generated Planet model
* @return an empty Constraints object
*/
public PlanetModel randomPlanetModel() {
final int shapeType = random().nextInt(2);
switch (shapeType) {
case 0: {
return PlanetModel.SPHERE;
}
case 1: {
return PlanetModel.WGS84;
}
default:
throw new IllegalStateException("Unexpected planet model");
public Constraints getEmptyConstraint(){
return new Constraints();
}
/**
* Method that returns a random generated GeoPoint.
*
* @param planetModel The planet model.
* @return The random generated GeoPoint.
*/
public GeoPoint randomGeoPoint(PlanetModel planetModel) {
GeoPoint point = null;
while (point == null) {
point = randomGeoPoint(planetModel, getEmptyConstraint());
}
return point;
}
/**
@ -159,13 +174,13 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
public GeoPoint randomGeoPoint(PlanetModel planetModel, Constraints constraints) {
int iterations = 0;
while (iterations < MAX_POINT_ITERATIONS) {
double lat = randomDouble();
if (Math.PI/2 - Math.abs(lat) <0){
continue;
double lat = randomDouble() * Math.PI/2;
if (random().nextBoolean()) {
lat = (-1)*lat;
}
double lon = randomDouble();
if (Math.PI - Math.abs(lat) <0){
continue;
double lon = randomDouble() * Math.PI;
if (random().nextBoolean()) {
lon = (-1)*lon;
}
iterations++;
GeoPoint point = new GeoPoint(planetModel, lat, lon);
@ -257,6 +272,12 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
case COLLECTION: {
return collection(planetModel, constraints);
}
case POINT: {
return point(planetModel, constraints);
}
case LINE: {
return line(planetModel, constraints);
}
case CONVEX_SIMPLE_POLYGON: {
return simpleConvexPolygon(planetModel, constraints);
}
@ -268,6 +289,36 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
}
}
/**
* Method that returns a random generated a GeoPointShape under given constraints. Returns
* NULL if it cannot build the GeoCircle under the given constraints.
*
* @param planetModel The planet model.
* @param constraints The given constraints.
* @return The random generated GeoPointShape.
*/
private GeoPointShape point(PlanetModel planetModel , Constraints constraints) {
int iterations=0;
while (iterations < MAX_SHAPE_ITERATIONS) {
iterations++;
final GeoPoint point = randomGeoPoint(planetModel, constraints);
if (point == null){
continue;
}
try {
GeoPointShape pointShape = GeoPointShapeFactory.makeGeoPointShape(planetModel, point.getLatitude(), point.getLongitude());
if (!constraints.valid(pointShape)) {
continue;
}
return pointShape;
} catch (IllegalArgumentException e) {
continue;
}
}
return null;
}
/**
* Method that returns a random generated a GeoCircle under given constraints. Returns
* NULL if it cannot build the GeoCircle under the given constraints.
@ -339,6 +390,33 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
return null;
}
/**
* Method that returns a random generated degenerate GeoPath under given constraints. Returns
* NULL if it cannot build the degenerate GeoPath under the given constraints.
*
* @param planetModel The planet model.
* @param constraints The given constraints.
* @return The random generated degenerated GeoPath.
*/
private GeoPath line(PlanetModel planetModel, Constraints constraints) {
int iterations = 0;
while (iterations < MAX_SHAPE_ITERATIONS) {
iterations++;
int vertexCount = random().nextInt(2) + 2;
List<GeoPoint> geoPoints = points(vertexCount, planetModel, constraints);
try {
GeoPath path = GeoPathFactory.makeGeoPath(planetModel, 0, geoPoints.toArray(new GeoPoint[geoPoints.size()]));
if (!constraints.valid(path)) {
continue;
}
return path;
} catch (IllegalArgumentException e) {
continue;
}
}
return null;
}
/**
* Method that returns a random generated a GeoPath under given constraints. Returns
* NULL if it cannot build the GeoPath under the given constraints.
@ -732,13 +810,7 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
* @return the cutoff angle.
*/
private double randomCutoffAngle() {
while(true) {
double radius = randomDouble();
if (radius <0 || radius > Math.PI){
continue;
}
return radius;
}
return randomDouble() * Math.PI;
}
/**
@ -939,6 +1011,3 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
}
}
}

65
lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeRelationshipTest.java
View File

@ -23,8 +23,7 @@ import org.junit.Test;
/**
* Random test to check relationship between GeoAreaShapes and GeoShapes.
*/
public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
public class RandomGeoShapeRelationshipTest extends RandomGeo3dShapeGenerator {
/**
* Test for WITHIN points. We build a WITHIN shape with respect the geoAreaShape
@ -38,6 +37,9 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
int referenceShapeType = CONVEX_POLYGON;
PlanetModel planetModel = randomPlanetModel();
int shapeType = randomShapeType();
while (shapeType == POINT || shapeType == LINE) {
shapeType = randomShapeType();
}
GeoAreaShape shape = null;
GeoPoint point = null;
while (point == null) {
@ -51,7 +53,10 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
point = randomGeoPoint(planetModel, constraints);
}
}
assertTrue(shape.isWithin(point));
StringBuilder b = new StringBuilder();
b.append("shape: " + shape + "\n");
b.append("point: " + point);
assertTrue(b.toString(), shape.isWithin(point));
}
/**
@ -78,7 +83,10 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
point = randomGeoPoint(planetModel, constraints);
}
}
assertFalse(shape.isWithin(point));
StringBuilder b = new StringBuilder();
b.append("shape: " + shape + "\n");
b.append("point: " + point);
assertFalse(b.toString(), shape.isWithin(point));
}
/**
@ -94,7 +102,7 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
int referenceShapeType = CONVEX_SIMPLE_POLYGON;
PlanetModel planetModel = randomPlanetModel();
int geoAreaShapeType = randomGeoAreaShapeType();
int shapeType =randomConvexShapeType();
int shapeType = randomConvexShapeType();
GeoShape shape = null;
GeoAreaShape geoAreaShape = null;
@ -109,11 +117,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
shape = randomGeoShape(shapeType, planetModel, constraints);
}
}
StringBuilder b = new StringBuilder();
b.append("geoAreaShape: " + geoAreaShape + "\n");
b.append("shape: " + shape);
int rel = geoAreaShape.getRelationship(shape);
assertEquals(GeoArea.DISJOINT, rel);
assertEquals(b.toString(), GeoArea.DISJOINT, rel);
if (shape instanceof GeoArea) {
rel = ((GeoArea)shape).getRelationship(geoAreaShape);
assertEquals(GeoArea.DISJOINT, rel);
assertEquals(b.toString(), GeoArea.DISJOINT, rel);
}
}
@ -129,7 +140,11 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
public void testRandomWithIn() {
PlanetModel planetModel = randomPlanetModel();
int geoAreaShapeType = randomGeoAreaShapeType();
int shapeType =randomShapeType();
//shapes cannot be point or line -- no area!
while(geoAreaShapeType == POINT || geoAreaShapeType == LINE) {
geoAreaShapeType = randomGeoAreaShapeType();
}
int shapeType = LINE;//randomShapeType();
int referenceShapeType = CONVEX_SIMPLE_POLYGON;
if (!isConcave(geoAreaShapeType)){
shapeType =randomConvexShapeType();
@ -150,11 +165,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
shape = randomGeoShape(shapeType, planetModel, constraints);
}
}
StringBuilder b = new StringBuilder();
b.append("geoAreaShape: " + geoAreaShape + "\n");
b.append("shape: " + shape);
int rel = geoAreaShape.getRelationship(shape);
assertEquals(GeoArea.WITHIN, rel);
assertEquals(b.toString(), GeoArea.WITHIN, rel);
if (shape instanceof GeoArea) {
rel = ((GeoArea)shape).getRelationship(geoAreaShape);
assertEquals(GeoArea.CONTAINS, rel);
assertEquals(b.toString(), GeoArea.CONTAINS, rel);
}
}
@ -174,10 +192,13 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
int referenceShapeType = CONVEX_SIMPLE_POLYGON;
PlanetModel planetModel = randomPlanetModel();
int geoAreaShapeType = randomGeoAreaShapeType();
while (geoAreaShapeType == COLLECTION){
while (geoAreaShapeType == COLLECTION ){
geoAreaShapeType = randomGeoAreaShapeType();
}
int shapeType = randomShapeType();
while (shapeType == POINT || shapeType == LINE) {
shapeType = randomShapeType();
}
if (isConcave(geoAreaShapeType)){
shapeType = randomConcaveShapeType();
}
@ -197,11 +218,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
shape = randomGeoShape(shapeType, planetModel, constraints);
}
}
StringBuilder b = new StringBuilder();
b.append("geoAreaShape: " + geoAreaShape + "\n");
b.append("shape: " + shape);
int rel = geoAreaShape.getRelationship(shape);
assertEquals(GeoArea.CONTAINS, rel);
assertEquals(b.toString(), GeoArea.CONTAINS, rel);
if (shape instanceof GeoArea) {
rel = ((GeoArea)shape).getRelationship(geoAreaShape);
assertEquals(GeoArea.WITHIN, rel);
assertEquals(b.toString(), GeoArea.WITHIN, rel);
}
}
@ -216,8 +240,13 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
public void testRandomOverlaps() {
PlanetModel planetModel = randomPlanetModel();
int geoAreaShapeType = randomGeoAreaShapeType();
while (geoAreaShapeType == POINT || geoAreaShapeType == LINE) {
geoAreaShapeType = randomGeoAreaShapeType();
}
int shapeType = randomShapeType();
while (shapeType == POINT || shapeType == LINE) {
shapeType = randomShapeType();
}
GeoShape shape = null;
GeoAreaShape geoAreaShape = null;
while (shape == null) {
@ -246,12 +275,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
shape = randomGeoShape(shapeType, planetModel, constraints);
}
}
StringBuilder b = new StringBuilder();
b.append("geoAreaShape: " + geoAreaShape + "\n");
b.append("shape: " + shape);
int rel = geoAreaShape.getRelationship(shape);
assertEquals(GeoArea.OVERLAPS, rel);
assertEquals(b.toString(), GeoArea.OVERLAPS, rel);
if (shape instanceof GeoArea) {
rel = ((GeoArea)shape).getRelationship(geoAreaShape);
assertEquals(GeoArea.OVERLAPS, rel);
assertEquals(b.toString(), GeoArea.OVERLAPS, rel);
}
}
}

24
lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java
View File

@ -24,6 +24,7 @@ import java.util.List;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Check relationship between polygon and GeoShapes of basic polygons. Normally we construct
@ -736,6 +737,29 @@ public class SimpleGeoPolygonRelationshipsTest {
assertEquals(GeoArea.CONTAINS, rel);
}
@Test
public void testDegeneratedPointInPole(){
GeoBBox bBox1 = GeoBBoxFactory.makeGeoBBox(PlanetModel.SPHERE, Math.PI*0.5, Math.PI*0.5, 0, 0);
GeoPoint point = new GeoPoint(PlanetModel.SPHERE, Math.PI*0.5, Math.PI);
System.out.println("bbox1 = "+bBox1+"; point = "+point);
assertTrue(bBox1.isWithin(point));
}
@Test
public void testDegeneratePathShape(){
GeoPoint point1 = new GeoPoint(PlanetModel.SPHERE, 0, 0);
GeoPoint point2 = new GeoPoint(PlanetModel.SPHERE, 0, 1);
GeoPoint[] pointPath1 = new GeoPoint[] {point1, point2};
GeoPath path1 = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 0, pointPath1);
GeoPath path2 = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 1, pointPath1);
int rel = path1.getRelationship(path2);
//if an end point is inside the shape it will always return intersects
assertEquals(GeoArea.CONTAINS, rel); //should be contains?
rel = path2.getRelationship(path1);
assertEquals(GeoArea.WITHIN, rel);
}
private GeoPolygon buildConvexGeoPolygon(double lon1, double lat1,
double lon2, double lat2,
double lon3, double lat3,

6
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
View File

@ -390,9 +390,11 @@ public class AnalyzingSuggester extends Lookup implements Accountable {
} else {
scratchA.offset = readerA.getPosition();
scratchB.offset = readerB.getPosition();
scratchA.length = a.length - scratchA.offset;
scratchB.length = b.length - scratchB.offset;
scratchA.length = readerA.length() - readerA.getPosition();
scratchB.length = readerB.length() - readerB.getPosition();
}
assert scratchA.isValid();
assert scratchB.isValid();
return scratchA.compareTo(scratchB);
}

60
lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
View File

@ -1111,6 +1111,66 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
IOUtils.close(a, tempDir);
}
/**
* Adds 50 random keys, that all analyze to the same thing (dog), with the same cost,
* and checks that they come back in surface-form order.
*/
public void testTieBreakOnSurfaceForm() throws Exception {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@Override
public TokenStream getTokenStream() {
return new CannedTokenStream(new Token[] {
token("dog", 1, 1)
});
}
@Override
protected void setReader(final Reader reader) {
}
};
}
};
Directory tempDir = getDirectory();
AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
// make 50 inputs all with the same cost of 1, random strings
Input[] inputs = new Input[100];
for (int i = 0; i < inputs.length; i++) {
inputs[i] = new Input(TestUtil.randomSimpleString(random()), 1);
}
suggester.build(new InputArrayIterator(inputs));
// Try to save/load:
Path tmpDir = createTempDir("AnalyzingSuggesterTest");
Path path = tmpDir.resolve("suggester");
OutputStream os = Files.newOutputStream(path);
suggester.store(os);
os.close();
InputStream is = Files.newInputStream(path);
suggester.load(is);
is.close();
// now suggest everything, and check that stuff comes back in order
List<LookupResult> results = suggester.lookup("", false, 50);
assertEquals(50, results.size());
for (int i = 1; i < 50; i++) {
String previous = results.get(i-1).toString();
String current = results.get(i).toString();
assertTrue("surface forms out of order: previous=" + previous + ",current=" + current,
current.compareTo(previous) >= 0);
}
IOUtils.close(a, tempDir);
}
public void test0ByteKeys() throws Exception {
final Analyzer a = new Analyzer() {
@Override

32
solr/CHANGES.txt
View File

@ -69,7 +69,7 @@ Upgrade Notes
* SOLR-11254: the abstract DocTransformer class now has an abstract score-less transform method variant.
* SOLR-11283: all Stream Evaluators in solrj.io.eval have been refactored to have a simplier and more
robust structure. This simplifies and condenses the code required to implement a new Evaluator and
robust structure. This simplifies and condenses the code required to implement a new Evaluator and
makes it much easier for evaluators to handle differing data types (primitives, objects, arrays,
lists, and so forth). (Dennis Gove)
@ -100,6 +100,13 @@ New Features
* SOLR-11072: Implement trigger for searchRate event type. (ab)
* SOLR-11317: JSON Facet API: min/max aggregations on numeric fields are now typed better so int/long
fields return an appropriate integral type rather than a double. (yonik)
* SOLR-11316: JSON Facet API: min/max aggregations are now supported on single-valued date fields.
(yonik)
Bug Fixes
----------------------
@ -142,6 +149,12 @@ Bug Fixes
* SOLR-11293: Potential data loss in TLOG replicas after replication failures (noble)
* SOLR-10101: TestLazyCores hangs (Erick Erickson)
* SOLR-11332: Fix sorting on 'enum' fieldTypes that use sortMissingFirst or sortMissingLast (hossman)
* SOLR-11348: Fix the DIH database example (James Dyer)
Optimizations
----------------------
@ -154,6 +167,10 @@ Optimizations
* SOLR-10769: Allow nodeAdded / nodeLost events to report multiple nodes in one event. (ab)
* SOLR-11242: QueryParser: re-use the LookaheadSuccess exception. (Michael Braun via David Smiley)
* SOLR-11314: FastCharStream: re-use the READ_PAST_EOF exception. (Michael Braun via David Smiley)
Other Changes
----------------------
@ -216,6 +233,15 @@ Other Changes
* SOLR-11209: Upgrade HttpClient to 4.5.3. (Hrishikesh Gadre via Mark Miller)
* SOLR-11322: JSON Facet API: instead of returning NaN, min & max aggregations omit
the value for any bucket with no values in the numeric field. (yonik)
* SOLR-10990: Breakup QueryComponent.process method for readability. (Christine Poerschke)
* SOLR-11132: Refactor common getSortField logic in various FieldTypes (Jason Gerlowski, hossman)
* SOLR-11351: Make LTRScoringModel model more extensible. (Christine Poerschke)
================== 7.0.0 ==================
Versions of Major Components
@ -848,7 +874,7 @@ Other Changes
* SOLR-10494: Make default response format JSON (wt=json), and also indent text responses formats
(indent=on) by default (Trey Grainger & Cassandra Targett via hossman)
* SOLR-10760: Remove trie field types and fields from example schemas. (Steve Rowe)
* SOLR-10760,SOLR-11315,SOLR-11313: Remove trie field types and fields from example schemas. (Steve Rowe)
* SOLR-11056: Add random range query test that compares results across Trie*, *Point and DocValue-only fields
(Tomás Fernández Löbbe)
@ -942,6 +968,8 @@ Other Changes
* SOLR-11261, SOLR-10966: Upgrade to Hadoop 2.7.4 to fix incompatibility with Java 9.
(Uwe Schindler)
* SOLR-11324: Clean up mention of trie fields in documentation and source comments. (Steve Rowe)
================== 6.6.1 ==================
Bug Fixes

16
solr/build.xml
View File

@ -709,7 +709,21 @@
<contrib-crawl target="-dist-maven"/>
</sequential>
</target>
<target name="-install-to-maven-local-repo" depends="install-maven-tasks">
<sequential>
<m2-install pom.xml="${filtered.pom.templates.dir}/solr/pom.xml"/> <!-- Solr parent POM -->
<subant target="-install-to-maven-local-repo" inheritall="false">
<propertyset refid="uptodate.and.compiled.properties"/>
<fileset dir="core" includes="build.xml"/>
<fileset dir="solrj" includes="build.xml"/>
<fileset dir="test-framework" includes="build.xml"/>
<fileset dir="webapp" includes="build.xml"/>
</subant>
<contrib-crawl target="-install-to-maven-local-repo"/>
</sequential>
</target>
<target name="generate-maven-artifacts" depends="-unpack-solr-tgz">
<ant dir=".." target="resolve" inheritall="false"/>
<antcall target="-filter-pom-templates" inheritall="false"/>

4
solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java
View File

@ -80,7 +80,7 @@ public abstract class LTRScoringModel {
protected final List<Feature> features;
private final List<Feature> allFeatures;
private final Map<String,Object> params;
private final List<Normalizer> norms;
protected final List<Normalizer> norms;
public static LTRScoringModel getInstance(SolrResourceLoader solrResourceLoader,
String className, String name, List<Feature> features,
@ -123,6 +123,8 @@ public abstract class LTRScoringModel {
* {@link ModelException} if they do not make sense.
*/
protected void validate() throws ModelException {
final List<Feature> features = getFeatures();
final List<Normalizer> norms = getNorms();
if (features.isEmpty()) {
throw new ModelException("no features declared for model "+name);
}

12
solr/contrib/ltr/src/test/org/apache/solr/ltr/TestRerankBase.java
View File

@ -239,12 +239,14 @@ public class TestRerankBase extends RestTestBase {
.append(",\n");
sb.append("\"class\":").append('"').append(type).append('"').append(",\n");
sb.append("\"features\":").append('[');
for (final String feature : features) {
sb.append("\n\t{ ");
sb.append("\"name\":").append('"').append(feature).append('"')
.append("},");
if (features.length > 0) {
for (final String feature : features) {
sb.append("\n\t{ ");
sb.append("\"name\":").append('"').append(feature).append('"')
.append("},");
}
sb.deleteCharAt(sb.length() - 1);
}
sb.deleteCharAt(sb.length() - 1);
sb.append("\n]\n");
if (params != null) {
sb.append(",\n");

7
solr/core/build.xml
View File

@ -60,6 +60,8 @@
<target name="-dist-maven" depends="-dist-maven-src-java"/>
<target name="-install-to-maven-local-repo" depends="-install-src-java-to-maven-local-repo"/>
<target name="resolve" depends="ivy-availability-check,ivy-fail,ivy-configure">
<sequential>
<ivy:retrieve conf="compile,compile.hadoop" type="jar,bundle" sync="${ivy.sync}" log="download-only" symlink="${ivy.symlink}"/>
@ -83,6 +85,11 @@
byline="true"
match="public QueryParser\(QueryParserTokenManager "
replace="protected QueryParser(QueryParserTokenManager "/>
<!-- change an exception used for signaling to be static -->
<replaceregexp file="src/java/org/apache/solr/parser/QueryParser.java"
byline="true"
match="final private LookaheadSuccess jj_ls ="
replace="static final private LookaheadSuccess jj_ls =" />
<replace token="StringBuffer" value="StringBuilder" encoding="UTF-8">
<fileset dir="src/java/org/apache/solr/parser" includes="ParseException.java TokenMgrError.java"/>
</replace>

107
solr/core/src/java/org/apache/solr/handler/StreamHandler.java
View File

@ -33,84 +33,7 @@ import org.apache.solr.client.solrj.io.ModelCache;
import org.apache.solr.client.solrj.io.SolrClientCache;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.comp.StreamComparator;
import org.apache.solr.client.solrj.io.eval.AbsoluteValueEvaluator;
import org.apache.solr.client.solrj.io.eval.AddEvaluator;
import org.apache.solr.client.solrj.io.eval.AndEvaluator;
import org.apache.solr.client.solrj.io.eval.AnovaEvaluator;
import org.apache.solr.client.solrj.io.eval.AppendEvaluator;
import org.apache.solr.client.solrj.io.eval.ArcCosineEvaluator;
import org.apache.solr.client.solrj.io.eval.ArcSineEvaluator;
import org.apache.solr.client.solrj.io.eval.ArcTangentEvaluator;
import org.apache.solr.client.solrj.io.eval.ArrayEvaluator;
import org.apache.solr.client.solrj.io.eval.AscEvaluator;
import org.apache.solr.client.solrj.io.eval.CeilingEvaluator;
import org.apache.solr.client.solrj.io.eval.CoalesceEvaluator;
import org.apache.solr.client.solrj.io.eval.ColumnEvaluator;
import org.apache.solr.client.solrj.io.eval.ConversionEvaluator;
import org.apache.solr.client.solrj.io.eval.ConvolutionEvaluator;
import org.apache.solr.client.solrj.io.eval.CopyOfEvaluator;
import org.apache.solr.client.solrj.io.eval.CopyOfRangeEvaluator;
import org.apache.solr.client.solrj.io.eval.CorrelationEvaluator;
import org.apache.solr.client.solrj.io.eval.CosineEvaluator;
import org.apache.solr.client.solrj.io.eval.CovarianceEvaluator;
import org.apache.solr.client.solrj.io.eval.CubedRootEvaluator;
import org.apache.solr.client.solrj.io.eval.CumulativeProbabilityEvaluator;
import org.apache.solr.client.solrj.io.eval.DescribeEvaluator;
import org.apache.solr.client.solrj.io.eval.DivideEvaluator;
import org.apache.solr.client.solrj.io.eval.EmpiricalDistributionEvaluator;
import org.apache.solr.client.solrj.io.eval.EqualToEvaluator;
import org.apache.solr.client.solrj.io.eval.EuclideanDistanceEvaluator;
import org.apache.solr.client.solrj.io.eval.ExclusiveOrEvaluator;
import org.apache.solr.client.solrj.io.eval.FindDelayEvaluator;
import org.apache.solr.client.solrj.io.eval.FloorEvaluator;
import org.apache.solr.client.solrj.io.eval.GreaterThanEqualToEvaluator;
import org.apache.solr.client.solrj.io.eval.GreaterThanEvaluator;
import org.apache.solr.client.solrj.io.eval.HistogramEvaluator;
import org.apache.solr.client.solrj.io.eval.HyperbolicCosineEvaluator;
import org.apache.solr.client.solrj.io.eval.HyperbolicSineEvaluator;
import org.apache.solr.client.solrj.io.eval.HyperbolicTangentEvaluator;
import org.apache.solr.client.solrj.io.eval.IfThenElseEvaluator;
import org.apache.solr.client.solrj.io.eval.KolmogorovSmirnovEvaluator;
import org.apache.solr.client.solrj.io.eval.LengthEvaluator;
import org.apache.solr.client.solrj.io.eval.LessThanEqualToEvaluator;
import org.apache.solr.client.solrj.io.eval.LessThanEvaluator;
import org.apache.solr.client.solrj.io.eval.ModuloEvaluator;
import org.apache.solr.client.solrj.io.eval.MovingAverageEvaluator;
import org.apache.solr.client.solrj.io.eval.MultiplyEvaluator;
import org.apache.solr.client.solrj.io.eval.NaturalLogEvaluator;
import org.apache.solr.client.solrj.io.eval.NormalDistributionEvaluator;
import org.apache.solr.client.solrj.io.eval.NormalizeEvaluator;
import org.apache.solr.client.solrj.io.eval.NotEvaluator;
import org.apache.solr.client.solrj.io.eval.OrEvaluator;
import org.apache.solr.client.solrj.io.eval.PercentileEvaluator;
import org.apache.solr.client.solrj.io.eval.PowerEvaluator;
import org.apache.solr.client.solrj.io.eval.PredictEvaluator;
import org.apache.solr.client.solrj.io.eval.RankEvaluator;
import org.apache.solr.client.solrj.io.eval.RawValueEvaluator;
import org.apache.solr.client.solrj.io.eval.RegressionEvaluator;
import org.apache.solr.client.solrj.io.eval.ResidualsEvaluator;
import org.apache.solr.client.solrj.io.eval.ReverseEvaluator;
import org.apache.solr.client.solrj.io.eval.RoundEvaluator;
import org.apache.solr.client.solrj.io.eval.SampleEvaluator;
import org.apache.solr.client.solrj.io.eval.ScaleEvaluator;
import org.apache.solr.client.solrj.io.eval.SequenceEvaluator;
import org.apache.solr.client.solrj.io.eval.SineEvaluator;
import org.apache.solr.client.solrj.io.eval.SquareRootEvaluator;
import org.apache.solr.client.solrj.io.eval.SubtractEvaluator;
import org.apache.solr.client.solrj.io.eval.TangentEvaluator;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDay;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDayOfQuarter;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDayOfYear;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorEpoch;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorHour;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorMinute;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorMonth;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorQuarter;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorSecond;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorWeek;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorYear;
import org.apache.solr.client.solrj.io.eval.UniformDistributionEvaluator;
import org.apache.solr.client.solrj.io.eval.UuidEvaluator;
import org.apache.solr.client.solrj.io.eval.*;
import org.apache.solr.client.solrj.io.graph.GatherNodesStream;
import org.apache.solr.client.solrj.io.graph.ShortestPathStream;
import org.apache.solr.client.solrj.io.ops.ConcatOperation;
@ -326,8 +249,15 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("copyOfRange", CopyOfRangeEvaluator.class)
.withFunctionName("copyOf", CopyOfEvaluator.class)
.withFunctionName("cov", CovarianceEvaluator.class)
.withFunctionName("corr", CorrelationEvaluator.class)
.withFunctionName("kendallsCorr", KendallsCorrelationEvaluator.class)
.withFunctionName("spearmansCorr", SpearmansCorrelationEvaluator.class)
.withFunctionName("describe", DescribeEvaluator.class)
.withFunctionName("distance", EuclideanDistanceEvaluator.class)
.withFunctionName("manhattanDistance", ManhattanDistanceEvaluator.class)
.withFunctionName("earthMoversDistance", EarthMoversDistanceEvaluator.class)
.withFunctionName("canberraDistance", CanberraDistanceEvaluator.class)
.withFunctionName("chebyshevDistance", ChebyshevDistanceEvaluator.class)
.withFunctionName("empiricalDistribution", EmpiricalDistributionEvaluator.class)
.withFunctionName("finddelay", FindDelayEvaluator.class)
.withFunctionName("hist", HistogramEvaluator.class)
@ -352,9 +282,27 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("ks", KolmogorovSmirnovEvaluator.class)
.withFunctionName("asc", AscEvaluator.class)
.withFunctionName("cumulativeProbability", CumulativeProbabilityEvaluator.class)
.withFunctionName("ebeAdd", EBEAddEvaluator.class)
.withFunctionName("ebeSubtract", EBESubtractEvaluator.class)
.withFunctionName("ebeMultiply", EBEMultiplyEvaluator.class)
.withFunctionName("ebeDivide", EBEDivideEvaluator.class)
.withFunctionName("dotProduct", DotProductEvaluator.class)
.withFunctionName("cosineSimilarity", CosineSimilarityEvaluator.class)
.withFunctionName("freqTable", FrequencyTableEvaluator.class)
.withFunctionName("uniformIntegerDistribution", UniformIntegerDistributionEvaluator.class)
.withFunctionName("binomialDistribution", BinomialDistributionEvaluator.class)
.withFunctionName("poissonDistribution", PoissonDistributionEvaluator.class)
.withFunctionName("enumeratedDistribution", EnumeratedDistributionEvaluator.class)
.withFunctionName("probability", ProbabilityEvaluator.class)
.withFunctionName("sumDifference", SumDifferenceEvaluator.class)
.withFunctionName("meanDifference", MeanDifferenceEvaluator.class)
.withFunctionName("primes", PrimesEvaluator.class)
// Boolean Stream Evaluators
.withFunctionName("and", AndEvaluator.class)
.withFunctionName("and", AndEvaluator.class)
.withFunctionName("eor", ExclusiveOrEvaluator.class)
.withFunctionName("eq", EqualToEvaluator.class)
.withFunctionName("gt", GreaterThanEvaluator.class)
@ -402,7 +350,6 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("cbrt", CubedRootEvaluator.class)
.withFunctionName("coalesce", CoalesceEvaluator.class)
.withFunctionName("uuid", UuidEvaluator.class)
.withFunctionName("corr", CorrelationEvaluator.class)
// Conditional Stream Evaluators
.withFunctionName("if", IfThenElseEvaluator.class)

460
solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
View File

@ -306,12 +306,12 @@ public class QueryComponent extends SearchComponent
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
SolrIndexSearcher searcher = req.getSearcher();
StatsCache statsCache = req.getCore().getStatsCache();
int purpose = params.getInt(ShardParams.SHARDS_PURPOSE, ShardRequest.PURPOSE_GET_TOP_IDS);
if ((purpose & ShardRequest.PURPOSE_GET_TERM_STATS) != 0) {
SolrIndexSearcher searcher = req.getSearcher();
statsCache.returnLocalStats(rb, searcher);
return;
}
@ -321,50 +321,11 @@ public class QueryComponent extends SearchComponent
statsCache.receiveGlobalStats(req);
}
SolrQueryResponse rsp = rb.rsp;
IndexSchema schema = searcher.getSchema();
// Optional: This could also be implemented by the top-level searcher sending
// a filter that lists the ids... that would be transparent to
// the request handler, but would be more expensive (and would preserve score
// too if desired).
String ids = params.get(ShardParams.IDS);
if (ids != null) {
SchemaField idField = schema.getUniqueKeyField();
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
int[] luceneIds = new int[idArr.size()];
int docs = 0;
if (idField.getType().isPointField()) {
for (int i=0; i<idArr.size(); i++) {
int id = searcher.search(
idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
if (id >= 0) {
luceneIds[docs++] = id;
}
}
} else {
for (int i=0; i<idArr.size(); i++) {
int id = searcher.getFirstMatch(
new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
if (id >= 0)
luceneIds[docs++] = id;
}
}
DocListAndSet res = new DocListAndSet();
res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
if (rb.isNeedDocSet()) {
// TODO: create a cache for this!
List<Query> queries = new ArrayList<>();
queries.add(rb.getQuery());
List<Query> filters = rb.getFilters();
if (filters != null) queries.addAll(filters);
res.docSet = searcher.getDocSet(queries);
}
rb.setResults(res);
ResultContext ctx = new BasicResultContext(rb);
rsp.addResponse(ctx);
if (doProcessSearchByIds(rb)) {
return;
}
@ -395,145 +356,15 @@ public class QueryComponent extends SearchComponent
if (groupingSpec != null) {
cmd.setSegmentTerminateEarly(false); // not supported, silently ignore any segmentTerminateEarly flag
try {
boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
.setQueryCommand(cmd)
.setNeedDocSet(false) // Order matters here
.setIncludeHitCount(true)
.setSearcher(searcher);
for (String field : groupingSpec.getFields()) {
topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
.setField(schema.getField(field))
.setGroupSort(groupingSpec.getGroupSort())
.setTopNGroups(cmd.getOffset() + cmd.getLen())
.setIncludeGroupCount(groupingSpec.isIncludeGroupCount())
.build()
);
}
CommandHandler commandHandler = topsGroupsActionBuilder.build();
commandHandler.execute();
SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
rsp.add("firstPhase", commandHandler.processResult(result, serializer));
rsp.add("totalHitCount", commandHandler.getTotalHitCount());
rb.setResult(result);
doProcessGroupedDistributedSearchFirstPhase(rb, cmd, result);
return;
} else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) {
CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
.setQueryCommand(cmd)
.setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
.setSearcher(searcher);
int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
docsToCollect = Math.max(docsToCollect, 1);
for (String field : groupingSpec.getFields()) {
SchemaField schemaField = schema.getField(field);
String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
if (topGroupsParam == null) {
topGroupsParam = new String[0];
}
List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
for (String topGroup : topGroupsParam) {
SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
BytesRefBuilder builder = new BytesRefBuilder();
schemaField.getType().readableToIndexed(topGroup, builder);
searchGroup.groupValue = builder.get();
}
topGroups.add(searchGroup);
}
secondPhaseBuilder.addCommandField(
new TopGroupsFieldCommand.Builder()
.setField(schemaField)
.setGroupSort(groupingSpec.getGroupSort())
.setSortWithinGroup(groupingSpec.getSortWithinGroup())
.setFirstPhaseGroups(topGroups)
.setMaxDocPerGroup(docsToCollect)
.setNeedScores(needScores)
.setNeedMaxScore(needScores)
.build()
);
}
for (String query : groupingSpec.getQueries()) {
secondPhaseBuilder.addCommandField(new Builder()
.setDocsToCollect(docsToCollect)
.setSort(groupingSpec.getGroupSort())
.setQuery(query, rb.req)
.setDocSet(searcher)
.build()
);
}
CommandHandler commandHandler = secondPhaseBuilder.build();
commandHandler.execute();
TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
rsp.add("secondPhase", commandHandler.processResult(result, serializer));
rb.setResult(result);
doProcessGroupedDistributedSearchSecondPhase(rb, cmd, result);
return;
}
int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
int limitDefault = cmd.getLen(); // this is normally from "rows"
Grouping grouping =
new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
grouping.setGroupSort(groupingSpec.getGroupSort())
.setWithinGroupSort(groupingSpec.getSortWithinGroup())
.setDefaultFormat(groupingSpec.getResponseFormat())
.setLimitDefault(limitDefault)
.setDefaultTotalCount(defaultTotalCount)
.setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
.setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
.setGetGroupedDocSet(groupingSpec.isTruncateGroups());
if (groupingSpec.getFields() != null) {
for (String field : groupingSpec.getFields()) {
grouping.addFieldCommand(field, rb.req);
}
}
if (groupingSpec.getFunctions() != null) {
for (String groupByStr : groupingSpec.getFunctions()) {
grouping.addFunctionCommand(groupByStr, rb.req);
}
}
if (groupingSpec.getQueries() != null) {
for (String groupByStr : groupingSpec.getQueries()) {
grouping.addQueryCommand(groupByStr, rb.req);
}
}
if( rb.isNeedDocList() || rb.isDebug() ){
// we need a single list of the returned docs
cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
}
grouping.execute();
if (grouping.isSignalCacheWarning()) {
rsp.add(
"cacheWarning",
String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
);
}
rb.setResult(result);
if (grouping.mainResult != null) {
ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
rsp.addResponse(ctx);
rsp.getToLog().add("hits", grouping.mainResult.matches());
} else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
rsp.add("grouped", result.groupedResults);
rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
}
doProcessGroupedSearch(rb, cmd, result);
return;
} catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
@ -541,27 +372,7 @@ public class QueryComponent extends SearchComponent
}
// normal search result
searcher.search(result, cmd);
rb.setResult(result);
ResultContext ctx = new BasicResultContext(rb);
rsp.addResponse(ctx);
rsp.getToLog().add("hits", rb.getResults().docList.matches());
if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
if (null != rb.getNextCursorMark()) {
rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
rb.getNextCursorMark().getSerializedTotem());
}
}
if(rb.mergeFieldHandler != null) {
rb.mergeFieldHandler.handleMergeFields(rb, searcher);
} else {
doFieldSortValues(rb, searcher);
}
doPrefetch(rb);
doProcessUngroupedSearch(rb, cmd, result);
}
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException
@ -1385,6 +1196,265 @@ public class QueryComponent extends SearchComponent
return Category.QUERY;
}
private boolean doProcessSearchByIds(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
String ids = params.get(ShardParams.IDS);
if (ids == null) {
return false;
}
SolrIndexSearcher searcher = req.getSearcher();
IndexSchema schema = searcher.getSchema();
SchemaField idField = schema.getUniqueKeyField();
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
int[] luceneIds = new int[idArr.size()];
int docs = 0;
if (idField.getType().isPointField()) {
for (int i=0; i<idArr.size(); i++) {
int id = searcher.search(
idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
if (id >= 0) {
luceneIds[docs++] = id;
}
}
} else {
for (int i=0; i<idArr.size(); i++) {
int id = searcher.getFirstMatch(
new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
if (id >= 0)
luceneIds[docs++] = id;
}
}
DocListAndSet res = new DocListAndSet();
res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
if (rb.isNeedDocSet()) {
// TODO: create a cache for this!
List<Query> queries = new ArrayList<>();
queries.add(rb.getQuery());
List<Query> filters = rb.getFilters();
if (filters != null) queries.addAll(filters);
res.docSet = searcher.getDocSet(queries);
}
rb.setResults(res);
ResultContext ctx = new BasicResultContext(rb);
rsp.addResponse(ctx);
return true;
}
private void doProcessGroupedDistributedSearchFirstPhase(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException {
GroupingSpecification groupingSpec = rb.getGroupingSpec();
assert null != groupingSpec : "GroupingSpecification is null";
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrIndexSearcher searcher = req.getSearcher();
IndexSchema schema = searcher.getSchema();
CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
.setQueryCommand(cmd)
.setNeedDocSet(false) // Order matters here
.setIncludeHitCount(true)
.setSearcher(searcher);
for (String field : groupingSpec.getFields()) {
topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
.setField(schema.getField(field))
.setGroupSort(groupingSpec.getGroupSort())
.setTopNGroups(cmd.getOffset() + cmd.getLen())
.setIncludeGroupCount(groupingSpec.isIncludeGroupCount())
.build()
);
}
CommandHandler commandHandler = topsGroupsActionBuilder.build();
commandHandler.execute();
SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
rsp.add("firstPhase", commandHandler.processResult(result, serializer));
rsp.add("totalHitCount", commandHandler.getTotalHitCount());
rb.setResult(result);
}
private void doProcessGroupedDistributedSearchSecondPhase(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException, SyntaxError {
GroupingSpecification groupingSpec = rb.getGroupingSpec();
assert null != groupingSpec : "GroupingSpecification is null";
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
SolrIndexSearcher searcher = req.getSearcher();
IndexSchema schema = searcher.getSchema();
boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
.setQueryCommand(cmd)
.setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
.setSearcher(searcher);
int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
docsToCollect = Math.max(docsToCollect, 1);
for (String field : groupingSpec.getFields()) {
SchemaField schemaField = schema.getField(field);
String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
if (topGroupsParam == null) {
topGroupsParam = new String[0];
}
List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
for (String topGroup : topGroupsParam) {
SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
BytesRefBuilder builder = new BytesRefBuilder();
schemaField.getType().readableToIndexed(topGroup, builder);
searchGroup.groupValue = builder.get();
}
topGroups.add(searchGroup);
}
secondPhaseBuilder.addCommandField(
new TopGroupsFieldCommand.Builder()
.setField(schemaField)
.setGroupSort(groupingSpec.getGroupSort())
.setSortWithinGroup(groupingSpec.getSortWithinGroup())
.setFirstPhaseGroups(topGroups)
.setMaxDocPerGroup(docsToCollect)
.setNeedScores(needScores)
.setNeedMaxScore(needScores)
.build()
);
}
for (String query : groupingSpec.getQueries()) {
secondPhaseBuilder.addCommandField(new Builder()
.setDocsToCollect(docsToCollect)
.setSort(groupingSpec.getGroupSort())
.setQuery(query, rb.req)
.setDocSet(searcher)
.build()
);
}
CommandHandler commandHandler = secondPhaseBuilder.build();
commandHandler.execute();
TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
rsp.add("secondPhase", commandHandler.processResult(result, serializer));
rb.setResult(result);
}
private void doProcessGroupedSearch(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException, SyntaxError {
GroupingSpecification groupingSpec = rb.getGroupingSpec();
assert null != groupingSpec : "GroupingSpecification is null";
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
SolrIndexSearcher searcher = req.getSearcher();
int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
int limitDefault = cmd.getLen(); // this is normally from "rows"
Grouping grouping =
new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
grouping.setGroupSort(groupingSpec.getGroupSort())
.setWithinGroupSort(groupingSpec.getSortWithinGroup())
.setDefaultFormat(groupingSpec.getResponseFormat())
.setLimitDefault(limitDefault)
.setDefaultTotalCount(defaultTotalCount)
.setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
.setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
.setGetGroupedDocSet(groupingSpec.isTruncateGroups());
if (groupingSpec.getFields() != null) {
for (String field : groupingSpec.getFields()) {
grouping.addFieldCommand(field, rb.req);
}
}
if (groupingSpec.getFunctions() != null) {
for (String groupByStr : groupingSpec.getFunctions()) {
grouping.addFunctionCommand(groupByStr, rb.req);
}
}
if (groupingSpec.getQueries() != null) {
for (String groupByStr : groupingSpec.getQueries()) {
grouping.addQueryCommand(groupByStr, rb.req);
}
}
if( rb.isNeedDocList() || rb.isDebug() ){
// we need a single list of the returned docs
cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
}
grouping.execute();
if (grouping.isSignalCacheWarning()) {
rsp.add(
"cacheWarning",
String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
);
}
rb.setResult(result);
if (grouping.mainResult != null) {
ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
rsp.addResponse(ctx);
rsp.getToLog().add("hits", grouping.mainResult.matches());
} else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
rsp.add("grouped", result.groupedResults);
rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
}
}
private void doProcessUngroupedSearch(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrIndexSearcher searcher = req.getSearcher();
searcher.search(result, cmd);
rb.setResult(result);
ResultContext ctx = new BasicResultContext(rb);
rsp.addResponse(ctx);
rsp.getToLog().add("hits", rb.getResults().docList.matches());
if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
if (null != rb.getNextCursorMark()) {
rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
rb.getNextCursorMark().getSerializedTotem());
}
}
if(rb.mergeFieldHandler != null) {
rb.mergeFieldHandler.handleMergeFields(rb, searcher);
} else {
doFieldSortValues(rb, searcher);
}
doPrefetch(rb);
}
/**
* Fake scorer for a single document
*

7
solr/core/src/java/org/apache/solr/parser/FastCharStream.java
View File

@ -69,7 +69,7 @@ public final class FastCharStream implements CharStream {
int charsRead = // fill space in buffer
input.read(buffer, newPosition, buffer.length-newPosition);
if (charsRead == -1)
throw new IOException("read past eof");
throw READ_PAST_EOF;
else
bufferLength += charsRead;
}
@ -80,6 +80,11 @@ public final class FastCharStream implements CharStream {
return readChar();
}
/**
* This Exception is used as a signal rather than an exceptional state.
*/
private static final IOException READ_PAST_EOF = new IOException("read past eof");
@Override
public final void backup(int amount) {
bufferPosition -= amount;

2
solr/core/src/java/org/apache/solr/parser/QueryParser.java
View File

@ -767,7 +767,7 @@ public class QueryParser extends SolrQueryParserBase implements QueryParserConst
}
static private final class LookaheadSuccess extends java.lang.Error { }
final private LookaheadSuccess jj_ls = new LookaheadSuccess();
static final private LookaheadSuccess jj_ls = new LookaheadSuccess();
private boolean jj_scan_token(int kind) {
if (jj_scanpos == jj_lastpos) {
jj_la--;

11
solr/core/src/java/org/apache/solr/schema/AbstractEnumField.java
View File

@ -250,11 +250,12 @@ public abstract class AbstractEnumField extends PrimitiveFieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
final Object missingValue = Integer.MIN_VALUE;
SortField sf = new SortField(field.getName(), SortField.Type.INT, top);
sf.setMissingValue(missingValue);
return sf;
SortField result = getSortField(field, SortField.Type.INT, top, Integer.MIN_VALUE, Integer.MAX_VALUE);
if (null == result.getMissingValue()) {
// special case default behavior: assume missing values are "below" all enum values
result.setMissingValue(Integer.MIN_VALUE);
}
return result;
}
@Override

15
solr/core/src/java/org/apache/solr/schema/DatePointField.java
View File

@ -190,20 +190,7 @@ public class DatePointField extends PointField implements DateValueFieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
Object missingValue = null;
boolean sortMissingLast = field.sortMissingLast();
boolean sortMissingFirst = field.sortMissingFirst();
if (sortMissingLast) {
missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
} else if (sortMissingFirst) {
missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
}
SortField sf = new SortField(field.getName(), SortField.Type.LONG, top);
sf.setMissingValue(missingValue);
return sf;
return getSortField(field, SortField.Type.LONG, top, Long.MIN_VALUE, Long.MAX_VALUE);
}
@Override

4
solr/core/src/java/org/apache/solr/schema/DateRangeField.java
View File

@ -39,7 +39,7 @@ import org.apache.solr.util.DateMathParser;
import org.locationtech.spatial4j.shape.Shape;
/**
* A field for indexed dates and date ranges. It's mostly compatible with TrieDateField. It has the potential to allow
* A field for indexed dates and date ranges. It's mostly compatible with DatePointField. It has the potential to allow
* efficient faceting, similar to facet.enum.
*
* @see NumberRangePrefixTreeStrategy
@ -75,7 +75,7 @@ public class DateRangeField extends AbstractSpatialPrefixTreeFieldType<NumberRan
if (shape instanceof UnitNRShape) {
UnitNRShape unitShape = (UnitNRShape) shape;
if (unitShape.getLevel() == tree.getMaxLevels()) {
//fully precise date. We can be fully compatible with TrieDateField (incl. 'Z')
//fully precise date. We can be fully compatible with DatePointField (incl. 'Z')
return shape.toString() + 'Z';
}
}

15
solr/core/src/java/org/apache/solr/schema/DoublePointField.java
View File

@ -134,20 +134,7 @@ public class DoublePointField extends PointField implements DoubleValueFieldType
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
Object missingValue = null;
boolean sortMissingLast = field.sortMissingLast();
boolean sortMissingFirst = field.sortMissingFirst();
if (sortMissingLast) {
missingValue = top ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
} else if (sortMissingFirst) {
missingValue = top ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
}
SortField sf = new SortField(field.getName(), SortField.Type.DOUBLE, top);
sf.setMissingValue(missingValue);
return sf;
return getSortField(field, SortField.Type.DOUBLE, top, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY);
}
@Override

65
solr/core/src/java/org/apache/solr/schema/FieldType.java
View File

@ -48,6 +48,7 @@ import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedSetSortField;
import org.apache.lucene.search.SortedNumericSelector;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.search.TermInSetQuery;
@ -69,7 +70,6 @@ import org.apache.solr.query.SolrRangeQuery;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QueryUtils;
import org.apache.solr.search.Sorting;
import org.apache.solr.uninverting.UninvertingReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -662,17 +662,76 @@ public abstract class FieldType extends FieldProperties {
* Returns the SortField instance that should be used to sort fields
* of this type.
* @see SchemaField#checkSortability
* @see #getSortField(SchemaField,SortField.Type,boolean,Object,Object)
*/
public abstract SortField getSortField(SchemaField field, boolean top);
/**
* <p>A Helper utility method for use by subclasses.</p>
* <p>This method deals with:</p>
* <ul>
* <li>{@link SchemaField#checkSortability}</li>
* <li>Creating a {@link SortField} on <code>field</code> with the specified
* <code>reverse</code> &amp; <code>sortType</code></li>
* <li>Setting the {@link SortField#setMissingValue} to <code>missingLow</code> or <code>missingHigh</code>
* as appropriate based on the value of <code>reverse</code> and the
* <code>sortMissingFirst</code> &amp; <code>sortMissingLast</code> properties of the
* <code>field</code></li>
* </ul>
*
* @param field The SchemaField to sort on. May use <code>sortMissingFirst</code> or <code>sortMissingLast</code> or neither.
* @param sortType The sort Type of the underlying values in the <code>field</code>
* @param reverse True if natural order of the <code>sortType</code> should be reversed
* @param missingLow The <code>missingValue</code> to be used if the other params indicate that docs w/o values should sort as "low" as possible.
* @param missingHigh The <code>missingValue</code> to be used if the other params indicate that docs w/o values should sort as "high" as possible.
* @see #getSortedSetSortField
*/
protected static SortField getSortField(SchemaField field, SortField.Type sortType, boolean reverse,
Object missingLow, Object missingHigh) {
field.checkSortability();
SortField sf = new SortField(field.getName(), sortType, reverse);
applySetMissingValue(field, sf, missingLow, missingHigh);
return sf;
}
/**
* Same as {@link #getSortField} but using {@link SortedSetSortField}
*/
protected static SortField getSortedSetSortField(SchemaField field, SortedSetSelector.Type selector,
boolean reverse, Object missingLow, Object missingHigh) {
field.checkSortability();
SortField sf = new SortedSetSortField(field.getName(), reverse, selector);
applySetMissingValue(field, sf, missingLow, missingHigh);
return sf;
}
/**
* @see #getSortField
* @see #getSortedSetSortField
*/
private static void applySetMissingValue(SchemaField field, SortField sortField,
Object missingLow, Object missingHigh) {
final boolean reverse = sortField.getReverse();
if (field.sortMissingLast()) {
sortField.setMissingValue(reverse ? missingLow : missingHigh);
} else if (field.sortMissingFirst()) {
sortField.setMissingValue(reverse ? missingHigh : missingLow);
}
}
/**
* Utility usable by subclasses when they want to get basic String sorting
* using common checks.
* @see SchemaField#checkSortability
*/
protected SortField getStringSort(SchemaField field, boolean reverse) {
field.checkSortability();
return Sorting.getStringSortField(field.name, reverse, field.sortMissingLast(),field.sortMissingFirst());
return getSortField(field, SortField.Type.STRING, reverse, SortField.STRING_FIRST, SortField.STRING_LAST);
}
/** called to get the default value source (normally, from the

15
solr/core/src/java/org/apache/solr/schema/FloatPointField.java
View File

@ -134,20 +134,7 @@ public class FloatPointField extends PointField implements FloatValueFieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
Object missingValue = null;
boolean sortMissingLast = field.sortMissingLast();
boolean sortMissingFirst = field.sortMissingFirst();
if (sortMissingLast) {
missingValue = top ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY;
} else if (sortMissingFirst) {
missingValue = top ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY;
}
SortField sf = new SortField(field.getName(), SortField.Type.FLOAT, top);
sf.setMissingValue(missingValue);
return sf;
return getSortField(field, SortField.Type.FLOAT, top, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY);
}
@Override

15
solr/core/src/java/org/apache/solr/schema/IntPointField.java
View File

@ -132,20 +132,7 @@ public class IntPointField extends PointField implements IntValueFieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
Object missingValue = null;
boolean sortMissingLast = field.sortMissingLast();
boolean sortMissingFirst = field.sortMissingFirst();
if (sortMissingLast) {
missingValue = top ? Integer.MIN_VALUE : Integer.MAX_VALUE;
} else if (sortMissingFirst) {
missingValue = top ? Integer.MAX_VALUE : Integer.MIN_VALUE;
}
SortField sf = new SortField(field.getName(), SortField.Type.INT, top);
sf.setMissingValue(missingValue);
return sf;
return getSortField(field, SortField.Type.INT, top, Integer.MIN_VALUE, Integer.MAX_VALUE);
}
@Override

15
solr/core/src/java/org/apache/solr/schema/LongPointField.java
View File

@ -131,20 +131,7 @@ public class LongPointField extends PointField implements LongValueFieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
Object missingValue = null;
boolean sortMissingLast = field.sortMissingLast();
boolean sortMissingFirst = field.sortMissingFirst();
if (sortMissingLast) {
missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
} else if (sortMissingFirst) {
missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
}
SortField sf = new SortField(field.getName(), SortField.Type.LONG, top);
sf.setMissingValue(missingValue);
return sf;
return getSortField(field, SortField.Type.LONG, top, Long.MIN_VALUE, Long.MAX_VALUE);
}
@Override

6
solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
View File

@ -34,13 +34,13 @@ import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource.State;
import org.apache.lucene.util.AttributeSource;
import org.apache.solr.analysis.SolrAnalyzer;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.Sorting;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -132,8 +132,8 @@ public class PreAnalyzedField extends TextField implements HasImplicitIndexAnaly
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
return Sorting.getTextSortField(field.getName(), top, field.sortMissingLast(), field.sortMissingFirst());
return getSortedSetSortField(field, SortedSetSelector.Type.MIN, top,
SortField.STRING_FIRST, SortField.STRING_LAST);
}
@Override

5
solr/core/src/java/org/apache/solr/schema/TextField.java
View File

@ -32,7 +32,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.query.SolrRangeQuery;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.Sorting;
import org.apache.solr.uninverting.UninvertingReader.Type;
/** <code>TextField</code> is the basic type for configurable text analysis.
@ -108,8 +107,8 @@ public class TextField extends FieldType {
@Override
public SortField getSortField(SchemaField field, boolean reverse) {
/* :TODO: maybe warn if isTokenized(), but doesn't use LimitTokenCountFilter in its chain? */
field.checkSortability();
return Sorting.getTextSortField(field.getName(), reverse, field.sortMissingLast(), field.sortMissingFirst());
return getSortedSetSortField(field, SortedSetSelector.Type.MIN, reverse,
SortField.STRING_FIRST, SortField.STRING_LAST);
}
@Override

44
solr/core/src/java/org/apache/solr/schema/TrieField.java
View File

@ -171,50 +171,14 @@ public class TrieField extends NumericFieldType {
switch (type) {
case INTEGER:
if( sortMissingLast ) {
missingValue = top ? Integer.MIN_VALUE : Integer.MAX_VALUE;
}
else if( sortMissingFirst ) {
missingValue = top ? Integer.MAX_VALUE : Integer.MIN_VALUE;
}
sf = new SortField( field.getName(), SortField.Type.INT, top);
sf.setMissingValue(missingValue);
return sf;
return getSortField(field, SortField.Type.INT, top, Integer.MIN_VALUE, Integer.MAX_VALUE);
case FLOAT:
if( sortMissingLast ) {
missingValue = top ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY;
}
else if( sortMissingFirst ) {
missingValue = top ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY;
}
sf = new SortField( field.getName(), SortField.Type.FLOAT, top);
sf.setMissingValue(missingValue);
return sf;
return getSortField(field, SortField.Type.FLOAT, top, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY);
case DATE: // fallthrough
case LONG:
if( sortMissingLast ) {
missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
}
else if( sortMissingFirst ) {
missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
}
sf = new SortField( field.getName(), SortField.Type.LONG, top);
sf.setMissingValue(missingValue);
return sf;
return getSortField(field, SortField.Type.LONG, top, Long.MIN_VALUE, Long.MAX_VALUE);
case DOUBLE:
if( sortMissingLast ) {
missingValue = top ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
}
else if( sortMissingFirst ) {
missingValue = top ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
}
sf = new SortField( field.getName(), SortField.Type.DOUBLE, top);
sf.setMissingValue(missingValue);
return sf;
return getSortField(field, SortField.Type.DOUBLE, top, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY);
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + field.name);
}

11
solr/core/src/java/org/apache/solr/search/FunctionQParser.java
View File

@ -25,6 +25,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.facet.AggValueSource;
import org.apache.solr.search.function.FieldNameValueSource;
import java.util.ArrayList;
import java.util.List;
@ -33,6 +34,7 @@ public class FunctionQParser extends QParser {
public static final int FLAG_CONSUME_DELIMITER = 0x01; // consume delimiter after parsing arg
public static final int FLAG_IS_AGG = 0x02;
public static final int FLAG_USE_FIELDNAME_SOURCE = 0x04; // When a field name is encountered, use the placeholder FieldNameValueSource instead of resolving to a real ValueSource
public static final int FLAG_DEFAULT = FLAG_CONSUME_DELIMITER;
/** @lucene.internal */
@ -374,8 +376,13 @@ public class FunctionQParser extends QParser {
} else if ("false".equals(id)) {
valueSource = new BoolConstValueSource(false);
} else {
SchemaField f = req.getSchema().getField(id);
valueSource = f.getType().getValueSource(f, this);
if ((flags & FLAG_USE_FIELDNAME_SOURCE) != 0) {
// Don't try to create a ValueSource for the field, just use a placeholder.
valueSource = new FieldNameValueSource(id);
} else {
SchemaField f = req.getSchema().getField(id);
valueSource = f.getType().getValueSource(f, this);
}
}
}

54
solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java
View File

@ -16,13 +16,9 @@
*/
package org.apache.solr.search;
import org.apache.lucene.search.Query;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.SolrQueryRequest;
import java.util.List;
/**
* Parse Solr's variant on the Lucene QueryParser syntax.
* <br>Other parameters:<ul>
@ -42,53 +38,3 @@ public class LuceneQParserPlugin extends QParserPlugin {
}
}
@Deprecated
class OldLuceneQParser extends LuceneQParser {
String sortStr;
public OldLuceneQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req);
}
@Override
public Query parse() throws SyntaxError {
// handle legacy "query;sort" syntax
if (getLocalParams() == null) {
String qstr = getString();
if (qstr == null || qstr.length() == 0)
return null;
sortStr = getParams().get(CommonParams.SORT);
if (sortStr == null) {
// sort may be legacy form, included in the query string
List<String> commands = StrUtils.splitSmart(qstr,';');
if (commands.size() == 2) {
qstr = commands.get(0);
sortStr = commands.get(1);
} else if (commands.size() == 1) {
// This is need to support the case where someone sends: "q=query;"
qstr = commands.get(0);
}
else if (commands.size() > 2) {
throw new SyntaxError("If you want to use multiple ';' in the query, use the 'sort' param.");
}
}
setString(qstr);
}
return super.parse();
}
@Override
public SortSpec getSortSpec(boolean useGlobal) throws SyntaxError {
SortSpec sort = super.getSortSpec(useGlobal);
if (sortStr != null && sortStr.length()>0 && sort.getSort()==null) {
SortSpec oldSort = SortSpecParsing.parseSortSpec(sortStr, getReq());
if( oldSort.getSort() != null ) {
sort.setSortAndFields(oldSort.getSort(), oldSort.getSchemaFields());
}
}
return sort;
}
}

12
solr/core/src/java/org/apache/solr/search/Sorting.java
View File

@ -16,15 +16,16 @@
*/
package org.apache.solr.search;
import org.apache.solr.schema.FieldType;
import org.apache.lucene.search.*;
/**
* Extra lucene sorting utilities &amp; convenience methods
*
*
*
* @deprecated custom {@link FieldType}s should use the helper methods in the base class. Other usage should leverage th underling lucene {@link SortField} classes directly.
*/
@Deprecated
public class Sorting {
@ -37,14 +38,19 @@ public class Sorting {
* @param nullLast true if null should come last, regardless of sort order
* @param nullFirst true if null should come first, regardless of sort order
* @return SortField
* @deprecated custom {@link FieldType}s should use {@link FieldType#getSortField}. Other usage should leverage th underling lucene {@link SortField} classes directly.
*/
@Deprecated
public static SortField getStringSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
SortField sortField = new SortField(fieldName, SortField.Type.STRING, reverse);
applyMissingFirstLast(sortField, reverse, nullLast, nullFirst);
return sortField;
}
/** Like {@link #getStringSortField}) except safe for tokenized fields */
/** Like {@link #getStringSortField}) except safe for tokenized fields
* @deprecated custom {@link FieldType}s should use {@link FieldType#getSortedSetSortField}. Other usage should leverage th underling lucene {@link SortedSetSortField} classes directly.
*/
@Deprecated
public static SortField getTextSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
SortField sortField = new SortedSetSortField(fieldName, reverse);
applyMissingFirstLast(sortField, reverse, nullLast, nullFirst);

4
solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
View File

@ -1017,14 +1017,14 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
addParser("agg_min", new ValueSourceParser() {
@Override
public ValueSource parse(FunctionQParser fp) throws SyntaxError {
return new MinMaxAgg("min", fp.parseValueSource());
return new MinMaxAgg("min", fp.parseValueSource(FunctionQParser.FLAG_DEFAULT | FunctionQParser.FLAG_USE_FIELDNAME_SOURCE));
}
});
addParser("agg_max", new ValueSourceParser() {
@Override
public ValueSource parse(FunctionQParser fp) throws SyntaxError {
return new MinMaxAgg("max", fp.parseValueSource());
return new MinMaxAgg("max", fp.parseValueSource(FunctionQParser.FLAG_DEFAULT | FunctionQParser.FLAG_USE_FIELDNAME_SOURCE));
}
});

145
solr/core/src/java/org/apache/solr/search/facet/MinMaxAgg.java
View File

@ -18,6 +18,7 @@ package org.apache.solr.search.facet;
import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
@ -25,9 +26,12 @@ import org.apache.lucene.index.OrdinalMap;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LongValues;
import org.apache.solr.common.SolrException;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.StrFieldSource;
import org.apache.solr.search.function.FieldNameValueSource;
public class MinMaxAgg extends SimpleAggValueSource {
final int minmax; // a multiplier to reverse the normal order of compare if this is max instead of min (i.e. max will be -1)
@ -41,28 +45,46 @@ public class MinMaxAgg extends SimpleAggValueSource {
public SlotAcc createSlotAcc(FacetContext fcontext, int numDocs, int numSlots) throws IOException {
ValueSource vs = getArg();
if (vs instanceof StrFieldSource) {
String field = ((StrFieldSource) vs).getField();
SchemaField sf = fcontext.qcontext.searcher().getSchema().getField(field);
SchemaField sf = null;
if (vs instanceof FieldNameValueSource) {
String field = ((FieldNameValueSource)vs).getFieldName();
sf = fcontext.qcontext.searcher().getSchema().getField(field);
if (sf.multiValued() || sf.getType().multiValuedFieldCache()) {
if (sf.hasDocValues()) {
// dv
} else {
// uif
}
vs = null;
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "min/max aggregations can't be used on multi-valued field " + field);
} else {
return new SingleValuedOrdAcc(fcontext, sf, numSlots);
vs = sf.getType().getValueSource(sf, null);
}
}
if (vs instanceof StrFieldSource) {
return new SingleValuedOrdAcc(fcontext, sf, numSlots);
}
// Since functions don't currently have types, we rely on the type of the field
if (sf != null && sf.getType().getNumberType() != null) {
switch (sf.getType().getNumberType()) {
case FLOAT:
case DOUBLE:
return new DFuncAcc(vs, fcontext, numSlots);
case INTEGER:
case LONG:
return new LFuncAcc(vs, fcontext, numSlots);
case DATE:
return new DateFuncAcc(vs, fcontext, numSlots);
}
}
// numeric functions
return new ValSlotAcc(vs, fcontext, numSlots);
return new DFuncAcc(vs, fcontext, numSlots);
}
@Override
public FacetMerger createFacetMerger(Object prototype) {
if (prototype instanceof Number)
return new NumericMerger();
if (prototype instanceof Double)
return new NumericMerger(); // still use NumericMerger to handle NaN?
else if (prototype instanceof Comparable) {
return new ComparableMerger();
} else {
@ -114,8 +136,8 @@ public class MinMaxAgg extends SimpleAggValueSource {
}
}
class ValSlotAcc extends DoubleFuncSlotAcc {
public ValSlotAcc(ValueSource values, FacetContext fcontext, int numSlots) {
class DFuncAcc extends DoubleFuncSlotAcc {
public DFuncAcc(ValueSource values, FacetContext fcontext, int numSlots) {
super(values, fcontext, numSlots, Double.NaN);
}
@ -129,6 +151,101 @@ public class MinMaxAgg extends SimpleAggValueSource {
result[slotNum] = val;
}
}
@Override
public Object getValue(int slot) {
double val = result[slot];
if (Double.isNaN(val)) {
return null;
} else {
return val;
}
}
}
class LFuncAcc extends LongFuncSlotAcc {
FixedBitSet exists;
public LFuncAcc(ValueSource values, FacetContext fcontext, int numSlots) {
super(values, fcontext, numSlots, 0);
exists = new FixedBitSet(numSlots);
}
@Override
public void collect(int doc, int slotNum) throws IOException {
long val = values.longVal(doc);
if (val == 0 && !values.exists(doc)) return; // depend on fact that non existing values return 0 for func query
long currVal = result[slotNum];
if (currVal == 0 && !exists.get(slotNum)) {
exists.set(slotNum);
result[slotNum] = val;
} else if (Long.compare(val, currVal) * minmax < 0) {
result[slotNum] = val;
}
}
@Override
public Object getValue(int slot) {
long val = result[slot];
if (val == 0 && !exists.get(slot)) {
return null;
} else {
return val;
}
}
@Override
public void resize(Resizer resizer) {
super.resize(resizer);
exists = resizer.resize(exists);
}
@Override
public int compare(int slotA, int slotB) {
long a = result[slotA];
long b = result[slotB];
boolean ea = a != 0 || exists.get(slotA);
boolean eb = b != 0 || exists.get(slotB);
if (ea != eb) {
if (ea) return 1; // a exists and b doesn't TODO: we need context to be able to sort missing last! SOLR-10618
if (eb) return -1; // b exists and a is missing
}
return Long.compare(a, b);
}
@Override
public void reset() {
super.reset();
exists.clear(0, exists.length());
}
}
class DateFuncAcc extends LongFuncSlotAcc {
private static final long MISSING = Long.MIN_VALUE;
public DateFuncAcc(ValueSource values, FacetContext fcontext, int numSlots) {
super(values, fcontext, numSlots, MISSING);
}
@Override
public void collect(int doc, int slotNum) throws IOException {
long val = values.longVal(doc);
if (val == 0 && !values.exists(doc)) return; // depend on fact that non existing values return 0 for func query
long currVal = result[slotNum];
if (Long.compare(val, currVal) * minmax < 0 || currVal == MISSING) {
result[slotNum] = val;
}
}
// let compare be the default for now (since we can't yet correctly handle sortMissingLast
@Override
public Object getValue(int slot) {
return result[slot] == MISSING ? null : new Date(result[slot]);
}
}

84
solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
View File

@ -16,14 +16,6 @@
*/
package org.apache.solr.search.facet;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrIndexSearcher;
import java.io.Closeable;
import java.io.IOException;
import java.lang.reflect.Array;
@ -32,6 +24,16 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrIndexSearcher;
/**
* Accumulates statistics separated by a slot number.
* There is a separate statistic per slot. The slot is usually an ordinal into a set of values, e.g. tracking a count
@ -140,6 +142,38 @@ public abstract class SlotAcc implements Closeable {
return values;
}
public long[] resize(long[] old, long defaultValue) {
long[] values = new long[getNewSize()];
if (defaultValue != 0) {
Arrays.fill(values, 0, values.length, defaultValue);
}
for (int i = 0; i < old.length; i++) {
long val = old[i];
if (val != defaultValue) {
int newSlot = getNewSlot(i);
if (newSlot >= 0) {
values[newSlot] = val;
}
}
}
return values;
}
public FixedBitSet resize(FixedBitSet old) {
FixedBitSet values = new FixedBitSet(getNewSize());
int oldSize = old.length();
for(int oldSlot = 0;;) {
oldSlot = values.nextSetBit(oldSlot);
if (oldSlot == DocIdSetIterator.NO_MORE_DOCS) break;
int newSlot = getNewSlot(oldSlot);
values.set(newSlot);
if (++oldSlot >= oldSize) break;
}
return values;
}
public <T> T[] resize(T[] old, T defaultValue) {
T[] values = (T[]) Array.newInstance(old.getClass().getComponentType(), getNewSize());
if (defaultValue != null) {
@ -222,6 +256,40 @@ abstract class DoubleFuncSlotAcc extends FuncSlotAcc {
}
}
abstract class LongFuncSlotAcc extends FuncSlotAcc {
long[] result;
long initialValue;
public LongFuncSlotAcc(ValueSource values, FacetContext fcontext, int numSlots, long initialValue) {
super(values, fcontext, numSlots);
this.initialValue = initialValue;
result = new long[numSlots];
if (initialValue != 0) {
reset();
}
}
@Override
public int compare(int slotA, int slotB) {
return Long.compare(result[slotA], result[slotB]);
}
@Override
public Object getValue(int slot) {
return result[slot];
}
@Override
public void reset() {
Arrays.fill(result, initialValue);
}
@Override
public void resize(Resizer resizer) {
result = resizer.resize(result, initialValue);
}
}
abstract class IntSlotAcc extends SlotAcc {
int[] result; // use LongArray32
int initialValue;

60
solr/core/src/java/org/apache/solr/search/function/FieldNameValueSource.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search.function;
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
/** Placeholder value source.
* @lucene.internal */
public class FieldNameValueSource extends ValueSource {
private String fieldName;
public FieldNameValueSource(String fieldName) {
this.fieldName = fieldName;
}
public String getFieldName() {
return fieldName;
}
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
throw new UnsupportedOperationException("FieldNameValueSource should not be directly used: " + this);
}
@Override
public boolean equals(Object o) {
return o instanceof FieldNameValueSource && fieldName.equals(((FieldNameValueSource)o).getFieldName());
}
@Override
public int hashCode() {
return fieldName.hashCode();
}
@Override
public String description() {
return "FIELDNAME(" + fieldName + ")";
}
}

6
solr/core/src/java/org/apache/solr/update/processor/FieldLengthUpdateProcessorFactory.java
View File

@ -35,7 +35,7 @@ import static org.apache.solr.update.processor.FieldValueMutatingUpdateProcessor
* <p>For example, with the configuration listed below any documents
* containing String values (such as "<code>abcdef</code>" or
* "<code>xyz</code>") in a field declared in the schema using
* <code>TrieIntField</code> or <code>TrieLongField</code>
* <code>IntPointField</code> or <code>LongPointField</code>
* would have those Strings replaced with the length of those fields as an
* Integer
* (ie: <code>6</code> and <code>3</code> respectively)
@ -43,8 +43,8 @@ import static org.apache.solr.update.processor.FieldValueMutatingUpdateProcessor
* <pre class="prettyprint">
* &lt;processor class="solr.FieldLengthUpdateProcessorFactory"&gt;
* &lt;arr name="typeClass"&gt;
* &lt;str&gt;solr.TrieIntField&lt;/str&gt;
* &lt;str&gt;solr.TrieLongField&lt;/str&gt;
* &lt;str&gt;solr.IntPointField&lt;/str&gt;
* &lt;str&gt;solr.LongPointField&lt;/str&gt;
* &lt;/arr&gt;
* &lt;/processor&gt;</pre>
*/

4
solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java
View File

@ -79,7 +79,7 @@ import static org.apache.solr.update.processor.FieldMutatingUpdateProcessor.SELE
* In the ExampleFieldMutatingUpdateProcessorFactory configured below,
* fields will be mutated if the name starts with "foo" <i>or</i> "bar";
* <b>unless</b> the field name contains the substring "SKIP" <i>or</i>
* the fieldType is (or subclasses) TrieDateField. Meaning a field named
* the fieldType is (or subclasses) DatePointField. Meaning a field named
* "foo_SKIP" is guaranteed not to be selected, but a field named "bar_smith"
* that uses StrField will be selected.
* </p>
@ -92,7 +92,7 @@ import static org.apache.solr.update.processor.FieldMutatingUpdateProcessor.SELE
* &lt;str name="fieldRegex"&gt;.*SKIP.*&lt;/str&gt;
* &lt;/lst&gt;
* &lt;lst name="exclude"&gt;
* &lt;str name="typeClass"&gt;solr.TrieDateField&lt;/str&gt;
* &lt;str name="typeClass"&gt;solr.DatePointField&lt;/str&gt;
* &lt;/lst&gt;
* &lt;/processor&gt;</pre>
*

4
solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
View File

@ -47,8 +47,8 @@ import org.slf4j.LoggerFactory;
* </p>
* <p>
* The default selection behavior is to mutate both those fields that don't match
* a schema field, as well as those fields that match a schema field with a field
* type that uses class solr.TrieDateField.
* a schema field, as well as those fields that match a schema field with a date
* field type.
* </p>
* <p>
* If all values are parseable as dates (or are already Date), then the field will

4
solr/core/src/java/org/apache/solr/update/processor/ParseDoubleFieldUpdateProcessorFactory.java
View File

@ -38,8 +38,8 @@ import java.util.Locale;
* </p>
* <p>
* The default selection behavior is to mutate both those fields that don't match
* a schema field, as well as those fields that match a schema field with a field
* type that uses class solr.TrieDoubleField.
* a schema field, as well as those fields that match a schema field with a double
* field type.
* </p>
* <p>
* If all values are parseable as double (or are already Double), then the field

4
solr/core/src/java/org/apache/solr/update/processor/ParseFloatFieldUpdateProcessorFactory.java
View File

@ -38,8 +38,8 @@ import java.util.Locale;
* </p>
* <p>
* The default selection behavior is to mutate both those fields that don't match
* a schema field, as well as those fields that match a schema field with a field
* type that uses class solr.TrieFloatField.
* a schema field, as well as those fields that match a schema field with a float
* field type.
* </p>
* <p>
* If all values are parseable as float (or are already Float), then the field

4
solr/core/src/java/org/apache/solr/update/processor/ParseIntFieldUpdateProcessorFactory.java
View File

@ -35,8 +35,8 @@ import java.util.Locale;
* </p>
* <p>
* The default selection behavior is to mutate both those fields that don't match
* a schema field, as well as those fields that match a schema field with a field
* type that uses class solr.TrieIntField.
* a schema field, as well as those fields that match a schema field with an int
* field type.
* </p>
* <p>
* If all values are parseable as int (or are already Integer), then the field

4
solr/core/src/java/org/apache/solr/update/processor/ParseLongFieldUpdateProcessorFactory.java
View File

@ -35,8 +35,8 @@ import java.util.Locale;
* </p>
* <p>
* The default selection behavior is to mutate both those fields that don't match
* a schema field, as well as those fields that match a schema field with a field
* type that uses class solr.TrieLongField.
* a schema field, as well as those fields that match a schema field with a long
* field type.
* </p>
* <p>
* If all values are parseable as long (or are already Long), then the field

23
solr/core/src/java/org/apache/solr/util/configuration/providers/package-info.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* TODO
*/
package org.apache.solr.util.configuration.providers;

10
solr/core/src/test-files/solr/collection1/conf/schema-enums.xml
View File

@ -18,12 +18,22 @@
<schema name="tiny" version="1.1">
<field name="id" type="string" indexed="true" stored="true" required="true"/>
<field name="_version_" type="long" indexed="true" stored="true" multiValued="false"/>
<!-- Test EnumField and EnumFieldType -->
<field name="severity" type="severityType" indexed="${solr.tests.EnumFieldTest.indexed}" stored="true" multiValued="false" docValues="${solr.tests.numeric.dv}"/>
<!-- NOTE: because these test sortMissingLast/sortMissingFirst, we force indexed="true" so we don't get
random errors on schema init about inconsistent properties -->
<field name="severity_missingLast" type="severityType" indexed="true" stored="true" multiValued="false" docValues="${solr.tests.numeric.dv}" sortMissingLast="true"/>
<field name="severity_missingFirst" type="severityType" indexed="true" stored="true" multiValued="false" docValues="${solr.tests.numeric.dv}" sortMissingFirst="true"/>
<field name="severity_mv" type="severityType" indexed="${solr.tests.EnumFieldTest.indexed}" stored="true" multiValued="true" docValues="${solr.tests.numeric.dv}"/>
<field name="text" type="text" indexed="true" stored="true" multiValued="true"/>
<uniqueKey>id</uniqueKey>
<copyField source="severity" dest="severity_missingLast" />
<copyField source="severity" dest="severity_missingFirst" />
<fieldType name="text" class="solr.TextField">
<analyzer>
<tokenizer class="solr.WhitespaceTokenizerFactory"/>

3
solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
View File

@ -489,9 +489,6 @@
<queryParser name="foo" class="FooQParserPlugin"/>
<!-- deprecated parser, delete once class is deleted in Solr 8.0 -->
<queryParser name="lucenePlusSort" class="solr.OldLuceneQParserPlugin"/>
<updateRequestProcessorChain name="dedupe">
<processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
<bool name="enabled">false</bool>

2
solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
View File

@ -869,7 +869,7 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
// testing everything from query level is hard because
// time marches on ... and there is no easy way to reach into the
// bowels of TrieDateField and muck with the definition of "now"
// bowels of DatePointField and muck with the definition of "now"
// ...
// BUT: we can test that crazy combinations of "NOW" all work correctly,
// assuming the test doesn't take too long to run...

70
solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java
View File

@ -43,7 +43,6 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
SolrQueryRequest req = null;
Map<String,String> args = new HashMap<>();
lrf.args.put(CommonParams.VERSION,"2.2");
lrf.args.put("defType","lucenePlusSort");
// compact the index, keep things from getting out of hand
@ -215,16 +214,16 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
,"*[count(//doc)=0]"
);
args = new HashMap<>();
args.put("defType","lucenePlusSort");
req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z];val_s1 asc",
args.put("sort","val_s1 asc");
req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z]",
"/select", 0, 0 , args);
assertQ(req
,"//*[@numFound='3'] "
,"*[count(//doc)=0]"
);
args = new HashMap<>();
args.put("defType","lucenePlusSort");
req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z];val_s1 desc",
args.put("sort","val_s1 desc");
req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z]",
"/select", 0, 0 , args);
assertQ(req
,"//*[@numFound='3'] "
@ -518,11 +517,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
,"//@numFound[.='1'] "
,"//int[.='-2147483648']"
);
assertQ(req("id:44;num_i1 asc;")
assertQ(req("q", "id:44", "sort","num_i1 asc")
,"//doc[1]/int[.='-2147483648'] "
,"//doc[last()]/int[.='2147483647']"
);
assertQ(req("id:44;num_i1 desc;")
assertQ(req("q","id:44","sort","num_i1 desc")
,"//doc[1]/int[.='2147483647'] "
,"//doc[last()]/int[.='-2147483648']"
);
@ -561,11 +560,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
,"//@numFound[.='1'] "
,"//long[.='-9223372036854775808']"
);
assertQ(req("id:44;num_l1 asc;")
assertQ(req("q","id:44","sort","num_l1 asc")
,"//doc[1]/long[.='-9223372036854775808'] "
,"//doc[last()]/long[.='9223372036854775807']"
);
assertQ(req("id:44;num_l1 desc;")
assertQ(req("q","id:44", "sort", "num_l1 desc")
,"//doc[1]/long[.='9223372036854775807'] "
,"//doc[last()]/long[.='-9223372036854775808']"
);
@ -611,11 +610,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
assertQ(req("num_f1:\"-1e20\"")
,"//@numFound[.='1']"
);
assertQ(req("id:44;num_f1 asc;")
assertQ(req("q", "id:44", "sort", "num_f1 asc")
,"//doc[1]/float[.='-Infinity'] "
,"//doc[last()]/float[.='NaN']"
);
assertQ(req("id:44;num_f1 desc;")
assertQ(req("q", "id:44", "sort","num_f1 desc")
,"//doc[1]/float[.='NaN'] "
,"//doc[last()]/float[.='-Infinity']"
);
@ -663,11 +662,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
assertQ(req("num_d1:\"1e-100\"")
,"//@numFound[.='1']"
);
assertQ(req("id:44;num_d1 asc;")
assertQ(req("q", "id:44", "sort", "num_d1 asc")
,"//doc[1]/double[.='-Infinity'] "
,"//doc[last()]/double[.='NaN']"
);
assertQ(req("id:44;num_d1 desc;")
assertQ(req("q","id:44","sort","num_d1 desc")
,"//doc[1]/double[.='NaN'] "
,"//doc[last()]/double[.='-Infinity']"
);
@ -693,27 +692,27 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
,"*[count(//doc)=6]"
);
assertQ(req("id:44; a_i1 asc,b_i1 desc")
assertQ(req("q","id:44", "sort", "a_i1 asc,b_i1 desc")
,"*[count(//doc)=6] "
,"//doc[3]/int[.='100'] "
,"//doc[4]/int[.='50']"
);
assertQ(req("id:44;a_i1 asc , b_i1 asc;")
assertQ(req("q","id:44", "sort", "a_i1 asc , b_i1 asc")
,"*[count(//doc)=6] "
,"//doc[3]/int[.='50'] "
,"//doc[4]/int[.='100']"
);
assertQ(req("id:44;a_i1 asc;")
assertQ(req("q", "id:44", "sort", "a_i1 asc")
,"*[count(//doc)=6] "
,"//doc[1]/int[.='-1'] "
,"//doc[last()]/int[.='15']"
);
assertQ(req("id:44;a_i1 asc , score top;")
assertQ(req("q","id:44","sort","a_i1 asc , score top")
,"*[count(//doc)=6] "
,"//doc[1]/int[.='-1'] "
,"//doc[last()]/int[.='15']"
);
assertQ(req("id:44; score top , a_i1 top, b_i1 bottom ;")
assertQ(req("q","id:44","sort","score top , a_i1 top, b_i1 bottom ")
,"*[count(//doc)=6] "
,"//doc[last()]/int[.='-1'] "
,"//doc[1]/int[.='15'] "
@ -736,36 +735,36 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
assertQ(req("id_i:[1000 TO 1010]")
,"*[count(//doc)=7]"
);
assertQ(req("id_i:[1000 TO 1010]; b_i1 asc")
assertQ(req("q","id_i:[1000 TO 1010]","sort","b_i1 asc")
,"*[count(//doc)=7] "
,"//doc[1]/int[.='50'] "
,"//doc[2]/int[.='100']"
);
assertQ(req("id_i:[1000 TO 1010]; b_i1 desc")
assertQ(req("q","id_i:[1000 TO 1010]","sort"," b_i1 desc")
,"*[count(//doc)=7] "
,"//doc[1]/int[.='100'] "
,"//doc[2]/int[.='50']"
);
assertQ(req("id_i:[1000 TO 1010]; a_i1 asc,b_i1 desc")
assertQ(req("q","id_i:[1000 TO 1010]","sort"," a_i1 asc,b_i1 desc")
,"*[count(//doc)=7] "
,"//doc[3]/int[@name='b_i1' and .='100'] "
,"//doc[4]/int[@name='b_i1' and .='50'] "
,"//doc[5]/arr[@name='id_i' and .='1000']"
);
assertQ(req("id_i:[1000 TO 1010]; a_i1 asc,b_i1 asc")
assertQ(req("q","id_i:[1000 TO 1010]","sort"," a_i1 asc,b_i1 asc")
,"*[count(//doc)=7] "
,"//doc[3]/int[@name='b_i1' and .='50'] "
,"//doc[4]/int[@name='b_i1' and .='100'] "
,"//doc[5]/arr[@name='id_i' and .='1000']"
);
// nullfirst tests
assertQ(req("id_i:[1000 TO 1002]; nullfirst asc")
assertQ(req("q","id_i:[1000 TO 1002]","sort"," nullfirst asc")
,"*[count(//doc)=3] "
,"//doc[1]/arr[@name='id_i' and .='1002']"
,"//doc[2]/arr[@name='id_i' and .='1001'] "
,"//doc[3]/arr[@name='id_i' and .='1000']"
);
assertQ(req("id_i:[1000 TO 1002]; nullfirst desc")
assertQ(req("q","id_i:[1000 TO 1002]","sort"," nullfirst desc")
,"*[count(//doc)=3] "
,"//doc[1]/arr[@name='id_i' and .='1002']"
,"//doc[2]/arr[@name='id_i' and .='1000'] "
@ -779,16 +778,16 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
// Sort parsing exception tests. (SOLR-6, SOLR-99)
assertQEx( "can not sort unindexed fields",
req( "id_i:1000; shouldbeunindexed asc" ), 400 );
req( "q","id_i:1000", "sort", "shouldbeunindexed asc" ), 400 );
assertQEx( "invalid query format",
req( "id_i:1000; nullfirst" ), 400 );
req( "q","id_i:1000", "sort", "nullfirst" ), 400 );
assertQEx( "unknown sort field",
req( "id_i:1000; abcde12345 asc" ), 400 );
req( "q","id_i:1000", "sort", "abcde12345 asc" ), 400 );
assertQEx( "unknown sort order",
req( "id_i:1000; nullfirst aaa" ), 400 );
req( "q","id_i:1000", "sort", "nullfirst aaa" ), 400 );
resetExceptionIgnores();
@ -1166,32 +1165,31 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
);
args = new HashMap<>();
args.put("fl","score ");
args.put("defType","lucenePlusSort");
req = new LocalSolrQueryRequest(h.getCore(), "id:44;id desc;",
args.put("sort","id desc");
req = new LocalSolrQueryRequest(h.getCore(), "id:44",
"/select", 0, 10, args);
assertQ(req
,"//result[@maxScore>0]"
);
args = new HashMap<>();
args.put("fl","score ");
args.put("defType","lucenePlusSort");
req = new LocalSolrQueryRequest(h.getCore(), "id:44;",
req = new LocalSolrQueryRequest(h.getCore(), "id:44",
"/select", 0, 10, args);
assertQ(req
,"//@maxScore = //doc/float[@name='score']"
);
args = new HashMap<>();
args.put("fl","score ");
args.put("defType","lucenePlusSort");
req = new LocalSolrQueryRequest(h.getCore(), "id:44;id desc;",
args.put("sort","id desc");
req = new LocalSolrQueryRequest(h.getCore(), "id:44",
"/select", 0, 10, args);
assertQ(req
,"//@maxScore = //doc/float[@name='score']"
);
args = new HashMap<>();
args.put("fl","*,score");
args.put("defType","lucenePlusSort");
req = new LocalSolrQueryRequest(h.getCore(), "id:44;id desc;",
args.put("sort","id desc");
req = new LocalSolrQueryRequest(h.getCore(), "id:44",
"/select", 0, 0 , args);
assertQ(req
,"//result[@maxScore>0]"

8
solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java
View File

@ -30,10 +30,10 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase {
schemaString = "schema-distributed-missing-sort.xml";
}
String sint1_ml = "one_i1_ml"; // TrieIntField, sortMissingLast=true, multiValued=false
String sint1_mf = "two_i1_mf"; // TrieIntField, sortMissingFirst=true, multiValued=false
String long1_ml = "three_l1_ml"; // TrieLongField, sortMissingLast=true, multiValued=false
String long1_mf = "four_l1_mf"; // TrieLongField, sortMissingFirst=true, multiValued=false
String sint1_ml = "one_i1_ml"; // int field, sortMissingLast=true, multiValued=false
String sint1_mf = "two_i1_mf"; // int field, sortMissingFirst=true, multiValued=false
String long1_ml = "three_l1_ml"; // long field, sortMissingLast=true, multiValued=false
String long1_mf = "four_l1_mf"; // long field, sortMissingFirst=true, multiValued=false
String string1_ml = "five_s1_ml"; // StringField, sortMissingLast=true, multiValued=false
String string1_mf = "six_s1_mf"; // StringField, sortMissingFirst=true, multiValued=false

1
solr/core/src/test/org/apache/solr/cloud/CdcrBootstrapTest.java
View File

@ -239,6 +239,7 @@ public class CdcrBootstrapTest extends SolrTestCaseJ4 {
}
}
@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-11278")
public void testBootstrapWithContinousIndexingOnSourceCluster() throws Exception {
// start the target first so that we know its zkhost
MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr"));

4
solr/core/src/test/org/apache/solr/core/TestLazyCores.java
View File

@ -781,7 +781,6 @@ public class TestLazyCores extends SolrTestCaseJ4 {
}
}
@BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-10101")
// Insure that when a core is aged out of the transient cache, any uncommitted docs are preserved.
// Note, this needs FS-based indexes to persist!
// Cores 2, 3, 6, 7, 8, 9 are transient
@ -814,7 +813,8 @@ public class TestLazyCores extends SolrTestCaseJ4 {
openCores.clear();
// We still should have 6, 7, 8, 9 loaded, their reference counts have NOT dropped to zero
checkInCores(cc, "collection6", "collection7", "collection8", "collection9");
checkInCores(cc, "collection1", "collection5",
"collection6", "collection7", "collection8", "collection9");
for (String coreName : coreList) {
// The point of this test is to insure that when cores are aged out and re-opened

4
solr/core/src/test/org/apache/solr/handler/SearchHandlerTest.java
View File

@ -63,14 +63,14 @@ public class SearchHandlerTest extends AbstractSolrTestCase {
);
// Using legacy ';' param
assertQ(req("q", "title:test; val_s1 desc", "defType","lucenePlusSort")
assertQ(req("q", "title:test", "sort","val_s1 desc")
,"//*[@numFound='3']"
,"//result/doc[1]/str[@name='id'][.='12']"
,"//result/doc[2]/str[@name='id'][.='11']"
,"//result/doc[3]/str[@name='id'][.='10']"
);
assertQ(req("q", "title:test; val_s1 asc", "defType","lucenePlusSort")
assertQ(req("q", "title:test", "sort", "val_s1 asc")
,"//*[@numFound='3']"
,"//result/doc[1]/str[@name='id'][.='10']"
,"//result/doc[2]/str[@name='id'][.='11']"

1
solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
View File

@ -37,7 +37,6 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.Before;

16
solr/core/src/test/org/apache/solr/schema/EnumFieldTest.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.solr.schema;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Set;
import java.util.regex.Matcher;
@ -385,6 +386,21 @@ public class EnumFieldTest extends SolrTestCaseJ4 {
"//doc[4]/str[@name='" + FIELD_NAME + "']/text()='High'",
"//doc[5]/str[@name='" + FIELD_NAME + "']/text()='Critical'"
);
// missing first....
for (String dir : Arrays.asList("asc", "desc")) {
assertQ(req("fl", "id", "q", "*:*", "sort", FIELD_NAME + "_missingFirst " + dir + ", id desc")
, "//doc[1]/str[@name='id']/text()='9'"
, "//doc[2]/str[@name='id']/text()='8'"
);
}
// missing last...
for (String dir : Arrays.asList("asc", "desc")) {
assertQ(req("fl", "id", "q", "*:*", "sort", FIELD_NAME + "_missingLast " + dir + ", id desc")
, "//doc[6]/str[@name='id']/text()='9'"
, "//doc[7]/str[@name='id']/text()='8'"
);
}
}
@Test

9
solr/core/src/test/org/apache/solr/schema/TestPointFields.java
View File

@ -3330,7 +3330,14 @@ public class TestPointFields extends SolrTestCaseJ4 {
private void doTestDoublePointFunctionQuery(String field) throws Exception {
assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof PointField);
int numVals = 10 * RANDOM_MULTIPLIER;
List<Double> values = getRandomDoubles(numVals, false);
// Restrict values to float range; otherwise conversion to float will cause truncation -> undefined results
List<Double> values = getRandomList(numVals, false, () -> {
Float f = Float.NaN;
while (f.isNaN()) {
f = Float.intBitsToFloat(random().nextInt());
}
return f.doubleValue();
});
String assertNumFound = "//*[@numFound='" + numVals + "']";
String[] idAscXpathChecks = new String[numVals + 1];
String[] idAscNegXpathChecks = new String[numVals + 1];

10
solr/core/src/test/org/apache/solr/search/CursorMarkTest.java
View File

@ -230,15 +230,15 @@ public class CursorMarkTest extends SolrTestCaseJ4 {
random().nextBytes(randBytes);
val = new BytesRef(randBytes);
} else if (fieldName.contains("int")) {
val = random().nextInt(); // TrieIntField
val = random().nextInt();
} else if (fieldName.contains("long")) {
val = random().nextLong(); // TrieLongField
val = random().nextLong();
} else if (fieldName.contains("float")) {
val = random().nextFloat() * random().nextInt(); // TrieFloatField
val = random().nextFloat() * random().nextInt();
} else if (fieldName.contains("double")) {
val = random().nextDouble() * random().nextInt(); // TrieDoubleField
val = random().nextDouble() * random().nextInt();
} else if (fieldName.contains("date")) {
val = random().nextLong(); // TrieDateField
val = random().nextLong();
} else if (fieldName.startsWith("currency")) {
val = random().nextDouble();
} else if (fieldName.startsWith("uuid")) {

8
solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
View File

@ -94,14 +94,6 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
" +apache +solr");
}
@Deprecated
public void testQueryLucenePlusSort() throws Exception {
assertQueryEquals("lucenePlusSort",
"apache solr", "apache solr", "apache solr ; score desc");
assertQueryEquals("lucenePlusSort",
"+apache +solr", "apache AND solr", " +apache +solr; score desc");
}
public void testQueryPrefix() throws Exception {
SolrQueryRequest req = req("myField","foo_s");
try {

1
solr/core/src/test/org/apache/solr/search/QueryParsingTest.java
View File

@ -43,7 +43,6 @@ public class QueryParsingTest extends SolrTestCaseJ4 {
SolrQueryRequest req = req("df", "text");
final String[] parsersTested = new String[] {
OldLuceneQParserPlugin.NAME,
LuceneQParserPlugin.NAME,
DisMaxQParserPlugin.NAME,
ExtendedDismaxQParserPlugin.NAME

2
solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
View File

@ -278,7 +278,7 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 {
q = qParser.getQuery();
assertEquals(26, ((TermInSetQuery)q).getTermData().size());
// large numeric filter query should use TermsQuery (for trie fields)
// large numeric filter query should use TermsQuery
qParser = QParser.getParser("foo_ti:(1 2 3 4 5 6 7 8 9 10 20 19 18 17 16 15 14 13 12 11)", req);
qParser.setIsFilter(true); // this may change in the future
qParser.setParams(params);

21
solr/core/src/test/org/apache/solr/search/TestSort.java
View File

@ -271,9 +271,9 @@ public class TestSort extends SolrTestCaseJ4 {
if (r.nextBoolean()) sfields.add( new SortField(null, SortField.Type.SCORE));
// hit both use-cases of sort-missing-last
sfields.add( Sorting.getStringSortField("f", reverse, sortMissingLast, sortMissingFirst) );
sfields.add( getStringSortField("f", reverse, sortMissingLast, sortMissingFirst) );
if (secondary) {
sfields.add( Sorting.getStringSortField("f2", reverse2, sortMissingLast2, sortMissingFirst2) );
sfields.add( getStringSortField("f2", reverse2, sortMissingLast2, sortMissingFirst2) );
}
if (r.nextBoolean()) sfields.add( new SortField(null, SortField.Type.SCORE));
@ -355,5 +355,20 @@ public class TestSort extends SolrTestCaseJ4 {
return new BitDocIdSet(obs);
}
private static SortField getStringSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
SortField sortField = new SortField(fieldName, SortField.Type.STRING, reverse);
// 4 cases:
// missingFirst / forward: default lucene behavior
// missingFirst / reverse: set sortMissingLast
// missingLast / forward: set sortMissingLast
// missingLast / reverse: default lucene behavior
if (nullFirst && reverse) {
sortField.setMissingValue(SortField.STRING_LAST);
} else if (nullLast && !reverse) {
sortField.setMissingValue(SortField.STRING_LAST);
}
return sortField;
}
}

Some files were not shown because too many files have changed in this diff Show More