Merge branch 'master' into feature/autoscaling

# Conflicts:
#	solr/CHANGES.txt

build.xml

@@ -374,7 +374,23 @@
     </subant>
   </target>
 
-  <target name="validate-maven-dependencies" depends="generate-maven-artifacts"
+  <target name="-install-maven-artifacts" depends="resolve,resolve-groovy,resolve-markdown,install-maven-tasks">
+    <ant dir="lucene" inheritall="false">
+      <target name="-unpack-lucene-tgz"/>
+      <target name="-filter-pom-templates"/>
+      <propertyset refid="uptodate.and.compiled.properties"/>
+    </ant>
+    <ant dir="solr" target="-unpack-solr-tgz" inheritall="false">
+      <propertyset refid="uptodate.and.compiled.properties"/>
+    </ant>
+    <subant target="-install-to-maven-local-repo" inheritall="false" failonerror="true">
+      <fileset dir="lucene" includes="build.xml" />
+      <fileset dir="solr" includes="build.xml" />
+      <propertyset refid="uptodate.and.compiled.properties"/>
+    </subant>
+  </target>
+
+  <target name="validate-maven-dependencies" depends="-install-maven-artifacts"
           description="Validates maven dependencies, licenses, etc.">
     <subant target="-validate-maven-dependencies" inheritall="false" failonerror="true">
       <fileset dir="lucene" includes="build.xml"/>

dev-tools/doap/lucene.rdf

@@ -66,6 +66,13 @@
       </foaf:Person>
     </maintainer>
 
+    <release>
+      <Version>
+        <name>lucene-6.6.1</name>
+        <created>2017-09-07</created>
+	      <revision>6.6.1</revision>
+      </Version>
+    </release>
     <release>
       <Version>
         <name>lucene-6.6.0</name>

dev-tools/doap/solr.rdf

@@ -66,6 +66,13 @@
       </foaf:Person>
     </maintainer>
 
+    <release>
+      <Version>
+        <name>solr-6.6.1</name>
+        <created>2017-09-07</created>
+        <revision>6.6.1</revision>
+      </Version>
+    </release>
     <release>
       <Version>
         <name>solr-6.6.0</name>

lucene/CHANGES.txt

@@ -23,6 +23,8 @@ New Features
 * LUCENE-7927: Add LongValueFacetCounts, to compute facet counts for individual
   numeric values (Mike McCandless)
 
+* LUCENE-7940: Add BengaliAnalyzer. (Md. Abdulla-Al-Sun via Robert Muir)
+
 Optimizations
 
 * LUCENE-7905: Optimize how OrdinalMap (used by
@@ -52,6 +54,17 @@ Bug Fixes
   not recommended, lucene-analyzers-icu contains binary data structures
   specific to ICU/Unicode versions it is built against. (Chris Koenig, Robert Muir)
 
+* LUCENE-7891: Lucene's taxonomy facets now uses a non-buggy LRU cache
+  by default.  (Jan-Willem van den Broek via Mike McCandless)
+
+* LUCENE-7959: Improve NativeFSLockFactory's exception message if it cannot create
+  write.lock for an empty index due to bad permissions/read-only filesystem/etc.
+  (Erick Erickson, Shawn Heisey, Robert Muir)
+
+* LUCENE-7968: AnalyzingSuggester would sometimes order suggestions incorrectly,
+  it did not properly break ties on the surface forms when both the weights and
+  the analyzed forms were equal. (Robert Muir)
+
 Build
 
 * SOLR-11181: Switch order of maven artifact publishing procedure: deploy first
@@ -192,6 +205,13 @@ Bug Fixes
 * LUCENE-7864: IndexMergeTool is not using intermediate hard links (even
   if possible). (Dawid Weiss)
 
+* LUCENE-7956: Fixed potential stack overflow error in ICUNormalizer2CharFilter.
+  (Adrien Grand)
+
+* LUCENE-7963: Remove useless getAttribute() in DefaultIndexingChain that
+  causes performance drop, introduced by LUCENE-7626.  (Daniel Mitterdorfer
+  via Uwe Schindler)
+
 Improvements
 
 * LUCENE-7489: Better storage of sparse doc-values fields with the default

lucene/NOTICE.txt

@@ -54,13 +54,14 @@ The KStem stemmer in
 was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst)
 under the BSD-license.
 
-The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default
+The Arabic,Persian,Romanian,Bulgarian, Hindi and Bengali analyzers (common) come with a default
 stopword list that is BSD-licensed created by Jacques Savoy.  These files reside in:
 analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt,
 analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt,
 analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt,
 analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt,
-analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt
+analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt,
+analysis/common/src/resources/org/apache/lucene/analysis/bn/stopwords.txt
 See http://members.unine.ch/jacques.savoy/clef/index.html.
 
 The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers

lucene/analysis/build.xml

@@ -125,6 +125,10 @@
     <forall-analyzers target="-dist-maven"/>
   </target>
 
+  <target name="-install-to-maven-local-repo">
+    <forall-analyzers target="-install-to-maven-local-repo"/>
+  </target>
+
   <target name="-validate-maven-dependencies">
     <forall-analyzers target="-validate-maven-dependencies"/>
   </target>

lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliAnalyzer.java

@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+
+import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.core.DecimalDigitFilter;
+import org.apache.lucene.analysis.in.IndicNormalizationFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * Analyzer for Bengali.
+ */
+public final class BengaliAnalyzer extends StopwordAnalyzerBase {
+  private final CharArraySet stemExclusionSet;
+  
+  /**
+   * File containing default Bengali stopwords.
+   * 
+   * Default stopword list is from http://members.unine.ch/jacques.savoy/clef/bengaliST.txt
+   * The stopword list is BSD-Licensed.
+   */
+  public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
+  private static final String STOPWORDS_COMMENT = "#";
+  
+  /**
+   * Returns an unmodifiable instance of the default stop-words set.
+   * @return an unmodifiable instance of the default stop-words set.
+   */
+  public static CharArraySet getDefaultStopSet(){
+    return DefaultSetHolder.DEFAULT_STOP_SET;
+  }
+  
+  /**
+   * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class 
+   * accesses the static final set the first time.;
+   */
+  private static class DefaultSetHolder {
+    static final CharArraySet DEFAULT_STOP_SET;
+
+    static {
+      try {
+        DEFAULT_STOP_SET = loadStopwordSet(false, BengaliAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
+      } catch (IOException ex) {
+        throw new RuntimeException("Unable to load default stopword set");
+      }
+    }
+  }
+  
+  /**
+   * Builds an analyzer with the given stop words
+   * 
+   * @param stopwords a stopword set
+   * @param stemExclusionSet a stemming exclusion set
+   */
+  public BengaliAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
+    super(stopwords);
+    this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
+  }
+  
+  /**
+   * Builds an analyzer with the given stop words 
+   * 
+   * @param stopwords a stopword set
+   */
+  public BengaliAnalyzer(CharArraySet stopwords) {
+    this(stopwords, CharArraySet.EMPTY_SET);
+  }
+  
+  /**
+   * Builds an analyzer with the default stop words:
+   * {@link #DEFAULT_STOPWORD_FILE}.
+   */
+  public BengaliAnalyzer() {
+    this(DefaultSetHolder.DEFAULT_STOP_SET);
+  }
+
+  /**
+   * Creates
+   * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
+   * used to tokenize all the text in the provided {@link Reader}.
+   * 
+   * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
+   *         built from a {@link StandardTokenizer} filtered with
+   *         {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link IndicNormalizationFilter},
+   *         {@link BengaliNormalizationFilter}, {@link SetKeywordMarkerFilter}
+   *         if a stem exclusion set is provided, {@link BengaliStemFilter}, and
+   *         Bengali Stop words
+   */
+  @Override
+  protected TokenStreamComponents createComponents(String fieldName) {
+    final Tokenizer source = new StandardTokenizer();
+    TokenStream result = new LowerCaseFilter(source);
+    result = new DecimalDigitFilter(result);
+    if (!stemExclusionSet.isEmpty())
+      result = new SetKeywordMarkerFilter(result, stemExclusionSet);
+    result = new IndicNormalizationFilter(result);
+    result = new BengaliNormalizationFilter(result);
+    result = new StopFilter(result, stopwords);
+    result = new BengaliStemFilter(result);
+    return new TokenStreamComponents(source, result);
+  }
+
+  @Override
+  protected TokenStream normalize(String fieldName, TokenStream in) {
+    TokenStream result = new StandardFilter(in);
+    result = new LowerCaseFilter(result);
+    result = new DecimalDigitFilter(result);
+    result = new IndicNormalizationFilter(result);
+    result = new BengaliNormalizationFilter(result);
+    return result;
+  }
+}

lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilter.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+
+import java.io.IOException;
+
+/**
+ * A {@link TokenFilter} that applies {@link BengaliNormalizer} to normalize the
+ * orthography.
+ * <p>
+ * In some cases the normalization may cause unrelated terms to conflate, so
+ * to prevent terms from being normalized use an instance of
+ * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets
+ * the {@link KeywordAttribute} before this {@link TokenStream}.
+ * </p>
+ * @see BengaliNormalizer
+ */
+public final class BengaliNormalizationFilter extends TokenFilter {
+
+  private final BengaliNormalizer normalizer = new BengaliNormalizer();
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
+  
+  public BengaliNormalizationFilter(TokenStream input) {
+    super(input);
+  }
+
+  @Override
+  public boolean incrementToken() throws IOException {
+    if (input.incrementToken()) {
+      if (!keywordAtt.isKeyword())
+        termAtt.setLength(normalizer.normalize(termAtt.buffer(), 
+            termAtt.length()));
+      return true;
+    } 
+    return false;
+  }
+}

lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
+import org.apache.lucene.analysis.util.MultiTermAwareComponent;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+
+import java.util.Map;
+
+/** 
+ * Factory for {@link BengaliNormalizationFilter}. 
+ * <pre class="prettyprint">
+ * &lt;fieldType name="text_bnnormal" class="solr.TextField" positionIncrementGap="100"&gt;
+ *   &lt;analyzer&gt;
+ *     &lt;tokenizer class="solr.StandardTokenizerFactory"/&gt;
+ *     &lt;filter class="solr.BengaliNormalizationFilterFactory"/&gt;
+ *   &lt;/analyzer&gt;
+ * &lt;/fieldType&gt;</pre>
+ */
+public class BengaliNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+
+  public BengaliNormalizationFilterFactory(Map<String,String> args) {
+    super(args);
+    if (!args.isEmpty()) {
+      throw new IllegalArgumentException("Unknown parameters: " + args);
+    }
+  }
+  
+  @Override
+  public TokenStream create(TokenStream input) {
+    return new BengaliNormalizationFilter(input);
+  }
+  
+  @Override
+  public AbstractAnalysisFactory getMultiTermComponent() {
+    return this;
+  }
+}

lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizer.java

@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+
+import static org.apache.lucene.analysis.util.StemmerUtil.delete;
+
+/**
+ * Normalizer for Bengali.
+ * <p>
+ * Implements the Bengali-language specific algorithm specified in:
+ * <i>A Double Metaphone encoding for Bangla and its application in spelling checker</i>
+ * Naushad UzZaman and Mumit Khan.
+ * http://www.panl10n.net/english/final%20reports/pdf%20files/Bangladesh/BAN16.pdf
+ * </p>
+ */
+public class BengaliNormalizer {
+  /**
+   * Normalize an input buffer of Bengali text
+   *
+   * @param s   input buffer
+   * @param len length of input buffer
+   * @return length of input buffer after normalization
+   */
+  public int normalize(char s[], int len) {
+
+    for (int i = 0; i < len; i++) {
+      switch (s[i]) {
+        // delete Chandrabindu
+        case '\u0981':
+          len = delete(s, i, len);
+          i--;
+          break;
+
+        // DirghoI kar -> RosshoI kar
+        case '\u09C0':
+          s[i] = '\u09BF';
+          break;
+
+        // DirghoU kar -> RosshoU kar
+        case '\u09C2':
+          s[i] = '\u09C1';
+          break;
+
+        // Khio (Ka + Hoshonto + Murdorno Sh)
+        case '\u0995':
+          if(i + 2 < len && s[i+1] == '\u09CD' && s[i+2] == '\u09BF') {
+            if (i == 0) {
+              s[i] = '\u0996';
+              len = delete(s, i + 2, len);
+              len = delete(s, i + 1, len);
+            } else {
+              s[i+1] = '\u0996';
+              len = delete(s, i + 2, len);
+            }
+          }
+          break;
+
+        // Nga to Anusvara
+        case '\u0999':
+          s[i] = '\u0982';
+          break;
+
+        // Ja Phala
+        case '\u09AF':
+          if(i - 2 == 0 && s[i-1] == '\u09CD') {
+            s[i - 1] = '\u09C7';
+
+            if(i + 1 < len && s[i+1] == '\u09BE') {
+              len = delete(s, i+1, len);
+            }
+            len = delete(s, i, len);
+            i --;
+          } else if(i - 1 >= 0 && s[i-1] == '\u09CD' ){
+            len = delete(s, i, len);
+            len = delete(s, i-1, len);
+            i -=2;
+          }
+          break;
+
+        // Ba Phalaa
+        case '\u09AC':
+          if((i >= 1 && s[i-1] != '\u09CD') || i == 0)
+            break;
+          if(i - 2 == 0) {
+            len = delete(s, i, len);
+            len = delete(s, i - 1, len);
+            i -= 2;
+          } else if(i - 5 >= 0 && s[i - 3] == '\u09CD') {
+            len = delete(s, i, len);
+            len = delete(s, i-1, len);
+            i -=2;
+          } else if(i - 2 >= 0){
+            s[i - 1] = s[i - 2];
+            len = delete(s, i, len);
+            i --;
+          }
+          break;
+
+        // Visarga
+        case '\u0983':
+          if(i == len -1) {
+            if(len <= 3) {
+              s[i] = '\u09B9';
+            } else {
+              len = delete(s, i, len);
+            }
+          } else {
+            s[i] = s[i+1];
+          }
+          break;
+
+        //All sh
+        case '\u09B6':
+        case '\u09B7':
+          s[i] = '\u09B8';
+          break;
+
+        //check na
+        case '\u09A3':
+          s[i] = '\u09A8';
+          break;
+
+        //check ra
+        case '\u09DC':
+        case '\u09DD':
+          s[i] = '\u09B0';
+          break;
+
+        case '\u09CE':
+          s[i] = '\u09A4';
+          break;
+
+        default:
+          break;
+      }
+    }
+
+    return len;
+  }
+}

lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilter.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+
+import java.io.IOException;
+
+/**
+ * A {@link TokenFilter} that applies {@link BengaliStemmer} to stem Bengali words.
+ */
+public final class BengaliStemFilter extends TokenFilter {
+  private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAttribute = addAttribute(KeywordAttribute.class);
+  private final BengaliStemmer bengaliStemmer = new BengaliStemmer();
+  
+  public BengaliStemFilter(TokenStream input) {
+    super(input);
+  }
+  
+  @Override
+  public boolean incrementToken() throws IOException {
+    if (input.incrementToken()) {
+      if (!keywordAttribute.isKeyword())
+        termAttribute.setLength(bengaliStemmer.stem(termAttribute.buffer(), termAttribute.length()));
+      return true;
+    } else {
+      return false;
+    }
+  }
+}

lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilterFactory.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+
+import java.util.Map;
+
+/** 
+ * Factory for {@link BengaliStemFilter}. 
+ * <pre class="prettyprint">
+ * &lt;fieldType name="text_histem" class="solr.TextField" positionIncrementGap="100"&gt;
+ *   &lt;analyzer&gt;
+ *     &lt;tokenizer class="solr.StandardTokenizerFactory"/&gt;
+ *     &lt;filter class="solr.BengaliStemFilterFactory"/&gt;
+ *   &lt;/analyzer&gt;
+ * &lt;/fieldType&gt;</pre>
+ */
+public class BengaliStemFilterFactory extends TokenFilterFactory {
+
+  public BengaliStemFilterFactory(Map<String,String> args) {
+    super(args);
+    if (!args.isEmpty()) {
+      throw new IllegalArgumentException("Unknown parameters: " + args);
+    }
+  }
+  
+  @Override
+  public TokenStream create(TokenStream input) {
+    return new BengaliStemFilter(input);
+  }
+}

lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemmer.java

@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+
+import static org.apache.lucene.analysis.util.StemmerUtil.endsWith;
+
+/**
+ * Stemmer for Bengali.
+ * <p>
+ * The algorithm is based on the report in:
+ * <i>Natural Language Processing in an Indian Language (Bengali)-I: Verb Phrase Analysis</i>
+ * P Sengupta and B B Chaudhuri
+ * </p>
+ *
+ * <p>
+ *   Few Stemmer criteria are taken from:
+ *   <i>http://members.unine.ch/jacques.savoy/clef/BengaliStemmerLight.java.txt</i>
+ * </p>
+ */
+public class BengaliStemmer {
+  public int stem(char buffer[], int len) {
+
+    // 8
+    if (len > 9 && (endsWith(buffer, len, "িয়াছিলাম")
+        || endsWith(buffer, len, "িতেছিলাম")
+        || endsWith(buffer, len, "িতেছিলেন")
+        || endsWith(buffer, len, "ইতেছিলেন")
+        || endsWith(buffer, len, "িয়াছিলেন")
+        || endsWith(buffer, len, "ইয়াছিলেন")
+    ))
+      return len - 8;
+
+    // 7
+    if ((len > 8) && (endsWith(buffer, len, "িতেছিলি")
+        || endsWith(buffer, len, "িতেছিলে")
+        || endsWith(buffer, len, "িয়াছিলা")
+        || endsWith(buffer, len, "িয়াছিলে")
+        || endsWith(buffer, len, "িতেছিলা")
+        || endsWith(buffer, len, "িয়াছিলি")
+
+        || endsWith(buffer, len, "য়েদেরকে")
+    ))
+      return len - 7;
+
+    // 6
+    if ((len > 7) && (endsWith(buffer, len, "িতেছিস")
+        || endsWith(buffer, len, "িতেছেন")
+        || endsWith(buffer, len, "িয়াছিস")
+        || endsWith(buffer, len, "িয়াছেন")
+        || endsWith(buffer, len, "েছিলাম")
+        || endsWith(buffer, len, "েছিলেন")
+
+        || endsWith(buffer, len, "েদেরকে")
+    ))
+      return len - 6;
+
+    // 5
+    if ((len > 6) && (endsWith(buffer, len, "িতেছি")
+        || endsWith(buffer, len, "িতেছা")
+        || endsWith(buffer, len, "িতেছে")
+        || endsWith(buffer, len, "ছিলাম")
+        || endsWith(buffer, len, "ছিলেন")
+        || endsWith(buffer, len, "িয়াছি")
+        || endsWith(buffer, len, "িয়াছা")
+        || endsWith(buffer, len, "িয়াছে")
+        || endsWith(buffer, len, "েছিলে")
+        || endsWith(buffer, len, "েছিলা")
+
+        || endsWith(buffer, len, "য়েদের")
+        || endsWith(buffer, len, "দেরকে")
+    ))
+      return len - 5;
+
+    // 4
+    if ((len > 5) && (endsWith(buffer, len, "িলাম")
+        || endsWith(buffer, len, "িলেন")
+        || endsWith(buffer, len, "িতাম")
+        || endsWith(buffer, len, "িতেন")
+        || endsWith(buffer, len, "িবেন")
+        || endsWith(buffer, len, "ছিলি")
+        || endsWith(buffer, len, "ছিলে")
+        || endsWith(buffer, len, "ছিলা")
+        || endsWith(buffer, len, "তেছে")
+        || endsWith(buffer, len, "িতেছ")
+
+        || endsWith(buffer, len, "খানা")
+        || endsWith(buffer, len, "খানি")
+        || endsWith(buffer, len, "গুলো")
+        || endsWith(buffer, len, "গুলি")
+        || endsWith(buffer, len, "য়েরা")
+        || endsWith(buffer, len, "েদের")
+    ))
+      return len - 4;
+
+    // 3
+    if ((len > 4) && (endsWith(buffer, len, "লাম")
+        || endsWith(buffer, len, "িলি")
+        || endsWith(buffer, len, "ইলি")
+        || endsWith(buffer, len, "িলে")
+        || endsWith(buffer, len, "ইলে")
+        || endsWith(buffer, len, "লেন")
+        || endsWith(buffer, len, "িলা")
+        || endsWith(buffer, len, "ইলা")
+        || endsWith(buffer, len, "তাম")
+        || endsWith(buffer, len, "িতি")
+        || endsWith(buffer, len, "ইতি")
+        || endsWith(buffer, len, "িতে")
+        || endsWith(buffer, len, "ইতে")
+        || endsWith(buffer, len, "তেন")
+        || endsWith(buffer, len, "িতা")
+        || endsWith(buffer, len, "িবা")
+        || endsWith(buffer, len, "ইবা")
+        || endsWith(buffer, len, "িবি")
+        || endsWith(buffer, len, "ইবি")
+        || endsWith(buffer, len, "বেন")
+        || endsWith(buffer, len, "িবে")
+        || endsWith(buffer, len, "ইবে")
+        || endsWith(buffer, len, "ছেন")
+
+        || endsWith(buffer, len, "য়োন")
+        || endsWith(buffer, len, "য়ের")
+        || endsWith(buffer, len, "েরা")
+        || endsWith(buffer, len, "দের")
+    ))
+      return len - 3;
+
+    // 2
+    if ((len > 3) && (endsWith(buffer, len, "িস")
+        || endsWith(buffer, len, "েন")
+        || endsWith(buffer, len, "লি")
+        || endsWith(buffer, len, "লে")
+        || endsWith(buffer, len, "লা")
+        || endsWith(buffer, len, "তি")
+        || endsWith(buffer, len, "তে")
+        || endsWith(buffer, len, "তা")
+        || endsWith(buffer, len, "বি")
+        || endsWith(buffer, len, "বে")
+        || endsWith(buffer, len, "বা")
+        || endsWith(buffer, len, "ছি")
+        || endsWith(buffer, len, "ছা")
+        || endsWith(buffer, len, "ছে")
+        || endsWith(buffer, len, "ুন")
+        || endsWith(buffer, len, "ুক")
+
+        || endsWith(buffer, len, "টা")
+        || endsWith(buffer, len, "টি")
+        || endsWith(buffer, len, "নি")
+        || endsWith(buffer, len, "ের")
+        || endsWith(buffer, len, "তে")
+        || endsWith(buffer, len, "রা")
+        || endsWith(buffer, len, "কে")
+    ))
+      return len - 2;
+
+    // 1
+    if ((len > 2) && (endsWith(buffer, len, "ি")
+        || endsWith(buffer, len, "ী")
+        || endsWith(buffer, len, "া")
+        || endsWith(buffer, len, "ো")
+        || endsWith(buffer, len, "ে")
+        || endsWith(buffer, len, "ব")
+        || endsWith(buffer, len, "ত")
+    ))
+      return len - 1;
+
+    return len;
+  }
+}

lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/package-info.java

@@ -14,24 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.search;
-
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.request.SolrQueryRequest;
 
 /**
- * Parse Solr's variant of Lucene QueryParser syntax, including the
+ * Analyzer for Bengali Language.
- * deprecated sort specification after the query.
- * <br>Example: <code>{!lucenePlusSort}myfield:foo +bar -baz;price asc</code>
- *
- * @deprecated This class should have been removed a long time ago, it will be removed in Solr 8.0
  */
-@Deprecated
+package org.apache.lucene.analysis.bn;
-public class OldLuceneQParserPlugin extends QParserPlugin {
-  public static final String NAME = "lucenePlusSort";
-
-  @Override
-  public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
-    return new OldLuceneQParser(qstr, localParams, params, req);
-  }
-}

lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory

@@ -17,6 +17,8 @@ org.apache.lucene.analysis.tr.ApostropheFilterFactory
 org.apache.lucene.analysis.ar.ArabicNormalizationFilterFactory
 org.apache.lucene.analysis.ar.ArabicStemFilterFactory
 org.apache.lucene.analysis.bg.BulgarianStemFilterFactory
+org.apache.lucene.analysis.bn.BengaliNormalizationFilterFactory
+org.apache.lucene.analysis.bn.BengaliStemFilterFactory
 org.apache.lucene.analysis.br.BrazilianStemFilterFactory
 org.apache.lucene.analysis.cjk.CJKBigramFilterFactory
 org.apache.lucene.analysis.cjk.CJKWidthFilterFactory

lucene/analysis/common/src/resources/org/apache/lucene/analysis/bn/stopwords.txt

@@ -0,0 +1,121 @@
+# See http://members.unine.ch/jacques.savoy/clef/index.html.
+# This file was created by Jacques Savoy and is distributed under the BSD license
+এই
+ও
+থেকে
+করে
+এ
+না
+ওই
+এক্
+নিয়ে
+করা
+বলেন
+সঙ্গে
+যে
+এব
+তা
+আর
+কোনো
+বলে
+সেই
+দিন
+হয়
+কি
+দু
+পরে
+সব
+দেওয়া
+মধ্যে
+এর
+সি
+শুরু
+কাজ
+কিছু
+কাছে
+সে
+তবে
+বা
+বন
+আগে
+জ্নজন
+পি
+পর
+তো
+ছিল
+এখন
+আমরা
+প্রায়
+দুই
+আমাদের
+তাই
+অন্য
+গিয়ে
+প্রযন্ত
+মনে
+নতুন
+মতো
+কেখা
+প্রথম
+আজ
+টি
+ধামার
+অনেক
+বিভিন্ন
+র
+হাজার
+জানা
+নয়
+অবশ্য
+বেশি
+এস
+করে
+কে
+হতে
+বি
+কয়েক
+সহ
+বেশ
+এমন
+এমনি
+কেন
+কেউ
+নেওয়া
+চেষ্টা
+লক্ষ
+বলা
+কারণ
+আছে
+শুধু
+তখন
+যা
+এসে
+চার
+ছিল
+যদি
+আবার
+কোটি
+উত্তর
+সামনে
+উপর
+বক্তব্য
+এত
+প্রাথমিক
+উপরে
+আছে
+প্রতি
+কাজে
+যখন
+খুব
+বহু
+গেল
+পেয়্র্
+চালু
+ই
+নাগাদ
+থাকা
+পাচ
+যাওয়া
+রকম
+সাধারণ
+কমনে

lucene/analysis/common/src/test/org/apache/lucene/analysis/bn/TestBengaliAnalyzer.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+
+/**
+ * Tests the BengaliAnalyzer
+ */
+public class TestBengaliAnalyzer extends BaseTokenStreamTestCase {
+
+  public void testResourcesAvailable() {
+    new BengaliAnalyzer().close();
+  }
+  
+  public void testBasics() throws Exception {
+    Analyzer a = new BengaliAnalyzer();
+
+    checkOneTerm(a, "বাড়ী", "বার");
+    checkOneTerm(a, "বারী", "বার");
+    a.close();
+  }
+  /**
+   * test Digits
+   */
+  public void testDigits() throws Exception {
+    BengaliAnalyzer a = new BengaliAnalyzer();
+    checkOneTerm(a, "১২৩৪৫৬৭৮৯০", "1234567890");
+    a.close();
+  }
+  
+  /** blast some random strings through the analyzer */
+  public void testRandomStrings() throws Exception {
+    Analyzer analyzer = new BengaliAnalyzer();
+    checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
+    analyzer.close();
+  }
+}

lucene/analysis/common/src/test/org/apache/lucene/analysis/bn/TestBengaliFilters.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+
+import java.io.Reader;
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+
+/**
+ * Test Bengali Filter Factory
+ */
+public class TestBengaliFilters extends BaseTokenStreamFactoryTestCase {
+  /**
+   * Test IndicNormalizationFilterFactory
+   */
+  public void testIndicNormalizer() throws Exception {
+    Reader reader = new StringReader("ত্‍ আমি");
+    TokenStream stream = whitespaceMockTokenizer(reader);
+    stream = tokenFilterFactory("IndicNormalization").create(stream);
+    assertTokenStreamContents(stream, new String[] { "ৎ", "আমি" });
+  }
+  
+  /**
+   * Test BengaliNormalizationFilterFactory
+   */
+  public void testBengaliNormalizer() throws Exception {
+    Reader reader = new StringReader("বাড়ী");
+    TokenStream stream = whitespaceMockTokenizer(reader);
+    stream = tokenFilterFactory("IndicNormalization").create(stream);
+    stream = tokenFilterFactory("BengaliNormalization").create(stream);
+    assertTokenStreamContents(stream, new String[] {"বারি"});
+  }
+  
+  /**
+   * Test BengaliStemFilterFactory
+   */
+  public void testStemmer() throws Exception {
+    Reader reader = new StringReader("বাড়ী");
+    TokenStream stream = whitespaceMockTokenizer(reader);
+    stream = tokenFilterFactory("IndicNormalization").create(stream);
+    stream = tokenFilterFactory("BengaliNormalization").create(stream);
+    stream = tokenFilterFactory("BengaliStem").create(stream);
+    assertTokenStreamContents(stream, new String[] {"বার"});
+  }
+  
+  /** Test that bogus arguments result in exception */
+  public void testBogusArguments() throws Exception {
+    IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+      tokenFilterFactory("IndicNormalization", "bogusArg", "bogusValue");
+    });
+    assertTrue(expected.getMessage().contains("Unknown parameters"));
+    
+    expected = expectThrows(IllegalArgumentException.class, () -> {
+      tokenFilterFactory("BengaliNormalization", "bogusArg", "bogusValue");
+    });
+    assertTrue(expected.getMessage().contains("Unknown parameters"));
+    
+    expected = expectThrows(IllegalArgumentException.class, () -> {
+      tokenFilterFactory("BengaliStem", "bogusArg", "bogusValue");
+    });
+    assertTrue(expected.getMessage().contains("Unknown parameters"));
+  }
+}

lucene/analysis/common/src/test/org/apache/lucene/analysis/bn/TestBengaliNormalizer.java

@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.util.TestUtil;
+
+import java.io.IOException;
+
+/**
+ * Test BengaliNormalizer
+ */
+public class TestBengaliNormalizer extends BaseTokenStreamTestCase {
+  /**
+   * Test some basic normalization, with an example from the paper.
+   */
+  public void testChndrobindu() throws IOException {
+    check("চাঁদ", "চাদ");
+  }
+
+  public void testRosshoIKar() throws IOException {
+    check("বাড়ী", "বারি");
+    check("তীর", "তির");
+  }
+
+  public void testRosshoUKar() throws IOException {
+    check("ভূল", "ভুল");
+    check("অনূপ", "অনুপ");
+  }
+
+  public void testNga() throws IOException {
+    check("বাঙলা", "বাংলা");
+  }
+
+  public void testJaPhaala() throws IOException {
+    check("ব্যাক্তি", "বেক্তি");
+    check( "সন্ধ্যা", "সন্ধা");
+  }
+
+  public void testBaPhalaa() throws IOException {
+    check("স্বদেশ", "সদেস");
+    check("তত্ত্ব", "তত্ত");
+    check("বিশ্ব", "বিসস");
+  }
+
+  public void testVisarga() throws IOException {
+    check("দুঃখ", "দুখখ");
+    check("উঃ", "উহ");
+    check("পুনঃ", "পুন");
+  }
+
+  public void testBasics() throws IOException {
+    check("কণা", "কনা");
+    check("শরীর", "সরির");
+    check("বাড়ি", "বারি");
+  }
+
+  /** creates random strings in the bengali block and ensures the normalizer doesn't trip up on them */
+  public void testRandom() throws IOException {
+    BengaliNormalizer normalizer = new BengaliNormalizer();
+    for (int i = 0; i < 100000; i++) {
+      String randomBengali = TestUtil.randomSimpleStringRange(random(), '\u0980', '\u09FF', 7);
+      try {
+        int newLen = normalizer.normalize(randomBengali.toCharArray(), randomBengali.length());
+        assertTrue(newLen >= 0); // should not return negative length
+        assertTrue(newLen <= randomBengali.length()); // should not increase length of string
+      } catch (Exception e) {
+        System.err.println("normalizer failed on input: '" + randomBengali + "' (" + escape(randomBengali) + ")");
+        throw e;
+      }
+    }
+  }
+
+  private void check(String input, String output) throws IOException {
+    Tokenizer tokenizer = whitespaceMockTokenizer(input);
+    TokenFilter tf = new BengaliNormalizationFilter(tokenizer);
+    assertTokenStreamContents(tf, new String[] { output });
+  }
+  
+  public void testEmptyTerm() throws IOException {
+    Analyzer a = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName) {
+        Tokenizer tokenizer = new KeywordTokenizer();
+        return new TokenStreamComponents(tokenizer, new BengaliNormalizationFilter(tokenizer));
+      }
+    };
+    checkOneTerm(a, "", "");
+    a.close();
+  }
+}

lucene/analysis/common/src/test/org/apache/lucene/analysis/bn/TestBengaliStemmer.java

@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.bn;
+
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.KeywordTokenizer;
+
+import java.io.IOException;
+
+/**
+ * Test Codes for BengaliStemmer
+ */
+public class TestBengaliStemmer extends BaseTokenStreamTestCase {
+
+  /**
+   * Testing few verbal words
+   */
+  public void testVerbsInShadhuForm() throws IOException {
+    check("করেছিলাম", "কর");
+    check("করিতেছিলে", "কর");
+    check("খাইতাম", "খাই");
+    check("যাইবে", "যা");
+  }
+
+  public void testVerbsInCholitoForm() throws IOException {
+    check("করছিলাম", "কর");
+    check("করছিলে", "কর");
+    check("করতাম", "কর");
+    check("যাব", "যা");
+    check("যাবে", "যা");
+    check("করি", "কর");
+    check("করো", "কর");
+  }
+
+  public void testNouns() throws IOException {
+    check("মেয়েরা", "মে");
+    check("মেয়েদেরকে", "মে");
+    check("মেয়েদের", "মে");
+
+    check("একটি", "এক");
+    check("মানুষগুলি", "মানুষ");
+  }
+
+  private void check(String input, String output) throws IOException {
+    Tokenizer tokenizer = whitespaceMockTokenizer(input);
+    TokenFilter tf = new BengaliStemFilter(tokenizer);
+    assertTokenStreamContents(tf, new String[] { output });
+  }
+  
+  public void testEmptyTerm() throws IOException {
+    Analyzer a = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName) {
+        Tokenizer tokenizer = new KeywordTokenizer();
+        return new TokenStreamComponents(tokenizer, new BengaliStemFilter(tokenizer));
+      }
+    };
+    checkOneTerm(a, "", "");
+    a.close();
+  }
+}

lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilter.java

@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.io.Reader;
 import java.util.Objects;
 
+import org.apache.lucene.analysis.CharacterUtils;
 import org.apache.lucene.analysis.charfilter.BaseCharFilter;
 
 import com.ibm.icu.text.Normalizer2;
@@ -61,7 +62,7 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
   ICUNormalizer2CharFilter(Reader in, Normalizer2 normalizer, int bufferSize) {
     super(in);
     this.normalizer = Objects.requireNonNull(normalizer);
-    this.tmpBuffer = new char[bufferSize];
+    this.tmpBuffer = CharacterUtils.newCharacterBuffer(bufferSize);
   }
 
   @Override
@@ -94,23 +95,31 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
     return -1;
   }
 
-  private final char[] tmpBuffer;
+  private final CharacterUtils.CharacterBuffer tmpBuffer;
 
-  private int readInputToBuffer() throws IOException {
+  private void readInputToBuffer() throws IOException {
-    final int len = input.read(tmpBuffer);
+    while (true) {
-    if (len == -1) {
+      // CharacterUtils.fill is supplementary char aware
+      final boolean hasRemainingChars = CharacterUtils.fill(tmpBuffer, input);
+
+      assert tmpBuffer.getOffset() == 0;
+      inputBuffer.append(tmpBuffer.getBuffer(), 0, tmpBuffer.getLength());
+
+      if (hasRemainingChars == false) {
         inputFinished = true;
-      return 0;
+        break;
+      }
+
+      final int lastCodePoint = Character.codePointBefore(tmpBuffer.getBuffer(), tmpBuffer.getLength(), 0);
+      if (normalizer.isInert(lastCodePoint)) {
+        // we require an inert char so that we can normalize content before and
+        // after this character independently
+        break;
+      }
     }
-    inputBuffer.append(tmpBuffer, 0, len);
 
     // if checkedInputBoundary was at the end of a buffer, we need to check that char again
     checkedInputBoundary = Math.max(checkedInputBoundary - 1, 0);
-    // this loop depends on 'isInert' (changes under normalization) but looks only at characters.
-    // so we treat all surrogates as non-inert for simplicity
-    if (normalizer.isInert(tmpBuffer[len - 1]) && !Character.isSurrogate(tmpBuffer[len-1])) {
-      return len;
-    } else return len + readInputToBuffer();
   }
 
   private int readAndNormalizeFromInput() {

lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java

@@ -20,12 +20,14 @@ package org.apache.lucene.analysis.icu;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
+import java.util.Arrays;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.CharFilter;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.KeywordTokenizer;
 import org.apache.lucene.analysis.ngram.NGramTokenizer;
 import org.apache.lucene.util.TestUtil;
 
@@ -418,4 +420,23 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase {
     }
     a.close();
   }
+
+  // https://issues.apache.org/jira/browse/LUCENE-7956
+  public void testVeryLargeInputOfNonInertChars() throws Exception {
+    char[] text = new char[1000000];
+    Arrays.fill(text, 'a');
+    try (Analyzer a = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName) {
+        return new TokenStreamComponents(new KeywordTokenizer());
+      }
+
+      @Override
+      protected Reader initReader(String fieldName, Reader reader) {
+        return new ICUNormalizer2CharFilter(reader, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
+      }
+    }) {
+      checkAnalysisConsistency(random(), a, false, new String(text));
+    }
+  }
 }

lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java

@@ -432,7 +432,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
       "6.5.1-cfs",
       "6.5.1-nocfs",
       "6.6.0-cfs",
-      "6.6.0-nocfs"
+      "6.6.0-nocfs",
+      "6.6.1-cfs",
+      "6.6.1-nocfs"
   };
 
   // TODO: on 6.0.0 release, gen the single segment indices and add here:

lucene/build.xml

@@ -428,6 +428,19 @@
     </sequential>
   </target>
 
+  <target name="-install-to-maven-local-repo" depends="install-maven-tasks">
+    <sequential>
+      <m2-install pom.xml="${filtered.pom.templates.dir}/pom.xml"/>        <!-- Lucene/Solr grandparent POM -->
+      <m2-install pom.xml="${filtered.pom.templates.dir}/lucene/pom.xml"/> <!-- Lucene parent POM -->
+      <subant target="-install-to-maven-local-repo" failonerror="true" inheritall="false">
+        <propertyset refid="uptodate.and.compiled.properties"/>
+        <fileset dir="${common.dir}/core" includes="build.xml"/>
+        <fileset dir="${common.dir}/test-framework" includes="build.xml"/>
+      </subant>
+      <modules-crawl target="-install-to-maven-local-repo"/>
+    </sequential>
+  </target>
+
   <target name="generate-maven-artifacts" depends="-unpack-lucene-tgz">
     <ant dir=".." target="resolve" inheritall="false"/>
     <antcall target="-filter-pom-templates" inheritall="false"/>

lucene/codecs/build.xml

@@ -23,4 +23,6 @@
   <import file="../module-build.xml"/>
 
   <target name="-dist-maven" depends="-dist-maven-src-java"/>
+
+  <target name="-install-to-maven-local-repo" depends="-install-src-java-to-maven-local-repo"/>
 </project>

lucene/common-build.xml

@@ -596,6 +596,19 @@
     </sequential>
   </macrodef>
 
+  <macrodef name="m2-install" description="Installs a Maven artifact into the local repository">
+    <element name="parent-poms" optional="yes"/>
+    <attribute name="pom.xml"/>
+    <attribute name="jar.file" default="${dist.jar.dir.prefix}-${version}/${dist.jar.dir.suffix}/${final.name}.jar"/>
+    <sequential>
+      <parent-poms/>
+      <artifact:pom id="maven.project" file="@{pom.xml}"/>
+      <artifact:install file="@{jar.file}">
+        <pom refid="maven.project"/>
+      </artifact:install>
+    </sequential>
+  </macrodef>
+
   <!-- validate maven dependencies -->
   <macrodef name="m2-validate-dependencies">
       <attribute name="pom.xml"/>
@@ -1713,6 +1726,44 @@ ${tests-output}/junit4-*.suites     - per-JVM executed suites
     </sequential>
   </target>
 
+  <target name="-install-to-maven-local-repo" depends="install-maven-tasks">
+    <sequential>
+      <property name="top.level.dir" location="${common.dir}/.."/>
+      <pathconvert property="pom.xml">
+        <mapper>
+          <chainedmapper>
+            <globmapper from="${top.level.dir}*" to="${filtered.pom.templates.dir}*"/>
+            <globmapper from="*build.xml" to="*pom.xml"/>
+          </chainedmapper>
+        </mapper>
+        <path location="${ant.file}"/>
+      </pathconvert>
+      <artifact:pom id="maven.project" file="${pom.xml}"/>
+      <artifact:install file="${dist.jar.dir.prefix}-${version}/${dist.jar.dir.suffix}/${final.name}.jar">
+        <pom refid="maven.project"/>
+      </artifact:install>
+    </sequential>
+  </target>
+
+  <target name="-install-src-java-to-maven-local-repo" depends="install-maven-tasks">
+    <sequential>
+      <property name="top.level.dir" location="${common.dir}/.."/>
+      <pathconvert property="pom.xml">
+        <mapper>
+          <chainedmapper>
+            <globmapper from="${top.level.dir}*" to="${filtered.pom.templates.dir}*"/>
+            <globmapper from="*build.xml" to="*/src/java/pom.xml"/>
+          </chainedmapper>
+        </mapper>
+        <path location="${ant.file}"/>
+      </pathconvert>
+      <artifact:pom id="maven.project" file="${pom.xml}"/>
+      <artifact:install file="${dist.jar.dir.prefix}-${version}/${dist.jar.dir.suffix}/${final.name}.jar">
+        <pom refid="maven.project"/>
+      </artifact:install>
+    </sequential>
+  </target>
+
   <target name="-dist-maven-src-java" depends="install-maven-tasks, jar-src, javadocs">
     <sequential>
       <property name="top.level.dir" location="${common.dir}/.."/>

lucene/core/build.xml

@@ -64,6 +64,8 @@
 
   <target name="-dist-maven" depends="-dist-maven-src-java"/>
 
+  <target name="-install-to-maven-local-repo" depends="-install-src-java-to-maven-local-repo"/>
+
   <macrodef name="createLevAutomaton">
       <attribute name="n"/>
       <sequential>

lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java

@@ -27,7 +27,6 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.codecs.DocValuesConsumer;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.NormsConsumer;
@@ -733,7 +732,6 @@ final class DefaultIndexingChain extends DocConsumer {
         stream.reset();
         invertState.setAttributeSource(stream);
         termsHashPerField.start(field, first);
-        CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
 
         while (stream.incrementToken()) {
 

lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java

@@ -93,15 +93,27 @@ public final class NativeFSLockFactory extends FSLockFactory {
     
     Path lockFile = lockDir.resolve(lockName);
 
+    IOException creationException = null;
     try {
       Files.createFile(lockFile);
     } catch (IOException ignore) {
       // we must create the file to have a truly canonical path.
       // if it's already created, we don't care. if it cant be created, it will fail below.
+      creationException = ignore;
     }
     
     // fails if the lock file does not exist
-    final Path realPath = lockFile.toRealPath();
+    final Path realPath;
+    try {
+      realPath = lockFile.toRealPath();
+    } catch (IOException e) {
+      // if we couldn't resolve the lock file, it might be because we couldn't create it.
+      // so append any exception from createFile as a suppressed exception, in case its useful
+      if (creationException != null) {
+        e.addSuppressed(creationException);
+      }
+      throw e;
+    }
     
     // used as a best-effort check, to see if the underlying file has changed
     final FileTime creationTime = Files.readAttributes(realPath, BasicFileAttributes.class).creationTime();

lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java

@@ -18,9 +18,17 @@ package org.apache.lucene.store;
 
 
 import java.io.IOException;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.FileSystem;
 import java.nio.file.Files;
+import java.nio.file.OpenOption;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
+import java.util.Set;
 
+import org.apache.lucene.mockfile.FilterFileSystemProvider;
+import org.apache.lucene.mockfile.FilterPath;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.TestUtil;
 
@@ -89,4 +97,38 @@ public class TestNativeFSLockFactory extends BaseLockFactoryTestCase {
       IOUtils.closeWhileHandlingException(lock);
     }
   }
+
+  /** MockFileSystem that throws AccessDeniedException on creating test.lock */
+  static class MockBadPermissionsFileSystem extends FilterFileSystemProvider {
+    public MockBadPermissionsFileSystem(FileSystem delegateInstance) {
+      super("mockbadpermissions://", delegateInstance);
+    }
+
+    @Override
+    public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
+      if (path.getFileName().toString().equals("test.lock")) {
+        throw new AccessDeniedException(path.toString(), null, "fake access denied");
+      }
+      return super.newByteChannel(path, options, attrs);
+    }
+  }
+
+  public void testBadPermissions() throws IOException {
+    // create a mock filesystem that will throw exc on creating test.lock
+    Path tmpDir = createTempDir();
+    tmpDir = FilterPath.unwrap(tmpDir).toRealPath();
+    FileSystem mock = new MockBadPermissionsFileSystem(tmpDir.getFileSystem()).getFileSystem(null);
+    Path mockPath = mock.getPath(tmpDir.toString());
+
+    // we should get an IOException (typically NoSuchFileException but no guarantee) with
+    // our fake AccessDenied added as suppressed.
+    Directory dir = getDirectory(mockPath.resolve("indexDir"));
+    IOException expected = expectThrows(IOException.class, () -> {
+      dir.obtainLock("test.lock");
+    });
+    AccessDeniedException suppressed = (AccessDeniedException) expected.getSuppressed()[0];
+    assertTrue(suppressed.getMessage().contains("fake access denied"));
+
+    dir.close();
+  }
 }

lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/LruTaxonomyWriterCache.java

@@ -32,8 +32,12 @@ public class LruTaxonomyWriterCache implements TaxonomyWriterCache {
    * function, LRU_STRING should be used.
    */
   public enum LRUType {
-    /** Use the label's hash as the key; this can lead to
+    /** Use only the label's 64 bit longHashCode as the hash key. Do not
-     *  silent conflicts! */
+     *  check equals, unlike most hash maps.
+     *  Note that while these hashes are very likely to be unique, the chance
+     *  of a collision is still greater than zero. If such an unlikely event
+     *  occurs, your document will get an incorrect facet.
+     */
     LRU_HASHED,
 
     /** Use the label as the hash key; this is always
@@ -43,15 +47,15 @@ public class LruTaxonomyWriterCache implements TaxonomyWriterCache {
 
   private NameIntCacheLRU cache;
 
-  /** Creates this with {@link LRUType#LRU_HASHED} method. */
+  /** Creates this with {@link LRUType#LRU_STRING} method. */
   public LruTaxonomyWriterCache(int cacheSize) {
     // TODO (Facet): choose between NameHashIntCacheLRU and NameIntCacheLRU.
     // For guaranteed correctness - not relying on no-collisions in the hash
     // function, NameIntCacheLRU should be used:
     // On the other hand, NameHashIntCacheLRU takes less RAM but if there
-    // are collisions (which we never found) two different paths would be
+    // are collisions two different paths would be mapped to the same
-    // mapped to the same ordinal...
+    // ordinal...
-    this(cacheSize, LRUType.LRU_HASHED);
+    this(cacheSize, LRUType.LRU_STRING);
   }
 
   /** Creates this with the specified method. */
@@ -60,8 +64,8 @@ public class LruTaxonomyWriterCache implements TaxonomyWriterCache {
     // For guaranteed correctness - not relying on no-collisions in the hash
     // function, NameIntCacheLRU should be used:
     // On the other hand, NameHashIntCacheLRU takes less RAM but if there
-    // are collisions (which we never found) two different paths would be
+    // are collisions two different paths would be mapped to the same
-    // mapped to the same ordinal...
+    // ordinal...
     if (lruType == LRUType.LRU_HASHED) {
       this.cache = new NameHashIntCacheLRU(cacheSize);
     } else {

lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestLruTaxonomyWriterCache.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.facet.taxonomy.writercache;
+
+import org.apache.lucene.facet.FacetTestCase;
+import org.apache.lucene.facet.taxonomy.FacetLabel;
+import org.junit.Test;
+
+public class TestLruTaxonomyWriterCache extends FacetTestCase {
+
+  @Test
+  public void testDefaultLRUTypeIsCollisionSafe() {
+    // These labels are clearly different, but have identical longHashCodes.
+    // Note that these labels are clearly contrived. We did encounter
+    // collisions in actual production data, but we aren't allowed to publish
+    // those.
+    final FacetLabel a = new FacetLabel("\0", "\u0003\uFFE2");
+    final FacetLabel b = new FacetLabel("\1", "\0");
+    // If this fails, then the longHashCode implementation has changed. This
+    // cannot prevent collisions. (All hashes must allow for collisions.) It
+    // will however stop the rest of this test from making sense. To fix, find
+    // new colliding labels, or make a subclass of FacetLabel that produces
+    // collisions.
+    assertEquals(a.longHashCode(), b.longHashCode());
+    // Make a cache with capacity > 2 so both our labels will fit. Don't
+    // specify an LRUType, since we want to check if the default is
+    // collision-safe.
+    final LruTaxonomyWriterCache cache = new LruTaxonomyWriterCache(10);
+    cache.put(a, 0);
+    cache.put(b, 1);
+    assertEquals(cache.get(a), 0);
+    assertEquals(cache.get(b), 1);
+  }
+
+}

lucene/ivy-versions.properties

@@ -94,7 +94,7 @@ org.apache.calcite.version = 1.13.0
 /org.apache.commons/commons-compress = 1.11
 /org.apache.commons/commons-exec = 1.3
 /org.apache.commons/commons-lang3 = 3.6
-/org.apache.commons/commons-math3 = 3.4.1
+/org.apache.commons/commons-math3 = 3.6.1
 
 org.apache.curator.version = 2.8.0
 /org.apache.curator/curator-client = ${org.apache.curator.version}

lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java

@@ -82,6 +82,11 @@ public class GeoBBoxFactory {
     //System.err.println(" not vertical line");
     if (extent >= Math.PI) {
       if (Math.abs(topLat - bottomLat) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+        if (Math.abs(topLat - Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+          return new GeoDegeneratePoint(planetModel, topLat, 0.0);
+        } else if (Math.abs(bottomLat + Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+          return new GeoDegeneratePoint(planetModel, bottomLat, 0.0);
+        }
         //System.err.println(" wide degenerate line");
         return new GeoWideDegenerateHorizontalLine(planetModel, topLat, leftLon, rightLon);
       }
@@ -94,8 +99,10 @@ public class GeoBBoxFactory {
       return new GeoWideRectangle(planetModel, topLat, bottomLat, leftLon, rightLon);
     }
     if (Math.abs(topLat - bottomLat) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
-      if (Math.abs(topLat - Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION || Math.abs(topLat + Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+      if (Math.abs(topLat - Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
         return new GeoDegeneratePoint(planetModel, topLat, 0.0);
+      } else if (Math.abs(bottomLat + Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+        return new GeoDegeneratePoint(planetModel, bottomLat, 0.0);
       }
       //System.err.println(" horizontal line");
       return new GeoDegenerateHorizontalLine(planetModel, topLat, leftLon, rightLon);

lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java

@@ -0,0 +1,788 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.spatial3d.geom;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * GeoShape representing a path across the surface of the globe,
+ * with a specified half-width.  Path is described by a series of points.
+ * Distances are measured from the starting point along the path, and then at right
+ * angles to the path.
+ *
+ * @lucene.internal
+ */
+class GeoDegeneratePath extends GeoBasePath {
+
+  /** The original list of path points */
+  protected final List<GeoPoint> points = new ArrayList<GeoPoint>();
+  
+  /** A list of SegmentEndpoints */
+  protected List<SegmentEndpoint> endPoints;
+  /** A list of PathSegments */
+  protected List<PathSegment> segments;
+
+  /** A point on the edge */
+  protected GeoPoint[] edgePoints;
+
+  /** Set to true if path has been completely constructed */
+  protected boolean isDone = false;
+  
+  /** Constructor.
+   *@param planetModel is the planet model.
+   *@param pathPoints are the points in the path.
+   */
+  public GeoDegeneratePath(final PlanetModel planetModel, final GeoPoint[] pathPoints) {
+    this(planetModel);
+    Collections.addAll(points, pathPoints);
+    done();
+  }
+
+  /** Piece-wise constructor.  Use in conjunction with addPoint() and done().
+   *@param planetModel is the planet model.
+   */
+  public GeoDegeneratePath(final PlanetModel planetModel) {
+    super(planetModel);
+  }
+
+  /** Add a point to the path.
+   *@param lat is the latitude of the point.
+   *@param lon is the longitude of the point.
+   */
+  public void addPoint(final double lat, final double lon) {
+    if (isDone)
+      throw new IllegalStateException("Can't call addPoint() if done() already called");
+    points.add(new GeoPoint(planetModel, lat, lon));
+  }
+  
+  /** Complete the path.
+   */
+  public void done() {
+    if (isDone)
+      throw new IllegalStateException("Can't call done() twice");
+    if (points.size() == 0)
+      throw new IllegalArgumentException("Path must have at least one point");
+    isDone = true;
+
+    endPoints = new ArrayList<>(points.size());
+    segments = new ArrayList<>(points.size());
+    
+    // First, build all segments.  We'll then go back and build corresponding segment endpoints.
+    GeoPoint lastPoint = null;
+    for (final GeoPoint end : points) {
+      if (lastPoint != null) {
+        final Plane normalizedConnectingPlane = new Plane(lastPoint, end);
+        if (normalizedConnectingPlane == null) {
+          continue;
+        }
+        segments.add(new PathSegment(planetModel, lastPoint, end, normalizedConnectingPlane));
+      }
+      lastPoint = end;
+    }
+    
+    if (segments.size() == 0) {
+      // Simple circle
+      final GeoPoint point = points.get(0);
+      
+      final SegmentEndpoint onlyEndpoint = new SegmentEndpoint(point);
+      endPoints.add(onlyEndpoint);
+      this.edgePoints = new GeoPoint[]{point};
+      return;
+    }
+    
+    // Create segment endpoints.  Use an appropriate constructor for the start and end of the path.
+    for (int i = 0; i < segments.size(); i++) {
+      final PathSegment currentSegment = segments.get(i);
+      
+      if (i == 0) {
+        // Starting endpoint
+        final SegmentEndpoint startEndpoint = new SegmentEndpoint(currentSegment.start,
+          currentSegment.startCutoffPlane);
+        endPoints.add(startEndpoint);
+        this.edgePoints = new GeoPoint[]{currentSegment.start};
+        continue;
+      }
+      
+      endPoints.add(new SegmentEndpoint(currentSegment.start,
+        segments.get(i-1).endCutoffPlane,
+        currentSegment.startCutoffPlane));
+    }
+    // Do final endpoint
+    final PathSegment lastSegment = segments.get(segments.size()-1);
+    endPoints.add(new SegmentEndpoint(lastSegment.end,
+      lastSegment.endCutoffPlane));
+
+  }
+
+  /**
+   * Constructor for deserialization.
+   * @param planetModel is the planet model.
+   * @param inputStream is the input stream.
+   */
+  public GeoDegeneratePath(final PlanetModel planetModel, final InputStream inputStream) throws IOException {
+    this(planetModel, 
+      SerializableObject.readPointArray(planetModel, inputStream));
+  }
+
+  @Override
+  public void write(final OutputStream outputStream) throws IOException {
+    SerializableObject.writePointArray(outputStream, points);
+  }
+
+  @Override
+  public double computePathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+    // Walk along path and keep track of the closest distance we find
+    double closestDistance = Double.POSITIVE_INFINITY;
+    // Segments first
+    for (PathSegment segment : segments) {
+      final double segmentDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
+      if (segmentDistance < closestDistance) {
+        closestDistance = segmentDistance;
+      }
+    }
+    // Now, endpoints
+    for (SegmentEndpoint endpoint : endPoints) {
+      final double endpointDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
+      if (endpointDistance < closestDistance) {
+        closestDistance = endpointDistance;
+      }
+    }
+    return closestDistance;
+  }
+
+  @Override
+  public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+    double currentDistance = 0.0;
+    double minPathCenterDistance = Double.POSITIVE_INFINITY;
+    double bestDistance = Double.POSITIVE_INFINITY;
+    int segmentIndex = 0;
+    
+    for (SegmentEndpoint endpoint : endPoints) {
+      final double endpointPathCenterDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
+      if (endpointPathCenterDistance < minPathCenterDistance) {
+        // Use this endpoint
+        minPathCenterDistance = endpointPathCenterDistance;
+        bestDistance = currentDistance;
+      }
+      // Look at the following segment, if any
+      if (segmentIndex < segments.size()) {
+        final PathSegment segment = segments.get(segmentIndex++);
+        final double segmentPathCenterDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
+        if (segmentPathCenterDistance < minPathCenterDistance) {
+          minPathCenterDistance = segmentPathCenterDistance;
+          bestDistance = distanceStyle.aggregateDistances(currentDistance, segment.nearestPathDistance(planetModel, distanceStyle, x, y, z));
+        }
+        currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
+      }
+    }
+    return bestDistance;
+  }
+
+  @Override
+  protected double distance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+    // Algorithm:
+    // (1) If the point is within any of the segments along the path, return that value.
+    // (2) If the point is within any of the segment end circles along the path, return that value.
+    double currentDistance = 0.0;
+    for (PathSegment segment : segments) {
+      double distance = segment.pathDistance(planetModel, distanceStyle, x,y,z);
+      if (distance != Double.POSITIVE_INFINITY)
+        return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
+      currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
+    }
+
+    int segmentIndex = 0;
+    currentDistance = 0.0;
+    for (SegmentEndpoint endpoint : endPoints) {
+      double distance = endpoint.pathDistance(distanceStyle, x, y, z);
+      if (distance != Double.POSITIVE_INFINITY)
+        return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
+      if (segmentIndex < segments.size())
+        currentDistance = distanceStyle.aggregateDistances(currentDistance, segments.get(segmentIndex++).fullPathDistance(distanceStyle));
+    }
+
+    return Double.POSITIVE_INFINITY;
+  }
+
+  @Override
+  protected void distanceBounds(final Bounds bounds, final DistanceStyle distanceStyle, final double distanceValue) {
+    // TBD: Compute actual bounds based on distance
+    getBounds(bounds);
+  }
+
+  @Override
+  protected double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+    double minDistance = Double.POSITIVE_INFINITY;
+    for (final SegmentEndpoint endpoint : endPoints) {
+      final double newDistance = endpoint.outsideDistance(distanceStyle, x,y,z);
+      if (newDistance < minDistance)
+        minDistance = newDistance;
+    }
+    for (final PathSegment segment : segments) {
+      final double newDistance = segment.outsideDistance(planetModel, distanceStyle, x, y, z);
+      if (newDistance < minDistance)
+        minDistance = newDistance;
+    }
+    return minDistance;
+  }
+
+  @Override
+  public boolean isWithin(final double x, final double y, final double z) {
+    for (SegmentEndpoint pathPoint : endPoints) {
+      if (pathPoint.isWithin(x, y, z)) {
+        return true;
+      }
+    }
+    for (PathSegment pathSegment : segments) {
+      if (pathSegment.isWithin(x, y, z)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public GeoPoint[] getEdgePoints() {
+    return edgePoints;
+  }
+
+  @Override
+  public boolean intersects(final Plane plane, final GeoPoint[] notablePoints, final Membership... bounds) {
+    // We look for an intersection with any of the exterior edges of the path.
+    // We also have to look for intersections with the cones described by the endpoints.
+    // Return "true" if any such intersections are found.
+
+    // For plane intersections, the basic idea is to come up with an equation of the line that is
+    // the intersection (if any).  Then, find the intersections with the unit sphere (if any).  If
+    // any of the intersection points are within the bounds, then we've detected an intersection.
+    // Well, sort of.  We can detect intersections also due to overlap of segments with each other.
+    // But that's an edge case and we won't be optimizing for it.
+    //System.err.println(" Looking for intersection of plane "+plane+" with path "+this);
+    
+    // Since the endpoints are included in the path segments, we only need to do this if there are
+    // no path segments
+    if (endPoints.size() == 1) {
+      return endPoints.get(0).intersects(planetModel, plane, notablePoints, bounds);
+    }
+
+    for (final PathSegment pathSegment : segments) {
+      if (pathSegment.intersects(planetModel, plane, notablePoints, bounds)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  @Override
+  public boolean intersects(GeoShape geoShape) {
+    // Since the endpoints are included in the path segments, we only need to do this if there are
+    // no path segments
+    if (endPoints.size() == 1) {
+      return endPoints.get(0).intersects(geoShape);
+    }
+
+    for (final PathSegment pathSegment : segments) {
+      if (pathSegment.intersects(geoShape)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  @Override
+  public void getBounds(Bounds bounds) {
+    super.getBounds(bounds);
+    // For building bounds, order matters.  We want to traverse
+    // never more than 180 degrees longitude at a pop or we risk having the
+    // bounds object get itself inverted.  So do the edges first.
+    for (PathSegment pathSegment : segments) {
+      pathSegment.getBounds(planetModel, bounds);
+    }
+    if (endPoints.size() == 1) {
+      endPoints.get(0).getBounds(planetModel, bounds);
+    }
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!(o instanceof GeoDegeneratePath))
+      return false;
+    GeoDegeneratePath p = (GeoDegeneratePath) o;
+    if (!super.equals(p))
+      return false;
+    return points.equals(p.points);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = super.hashCode();
+    result = 31 * result + points.hashCode();
+    return result;
+  }
+
+  @Override
+  public String toString() {
+    return "GeoDegeneratePath: {planetmodel=" + planetModel+", points={" + points + "}}";
+  }
+
+  /**
+   * This is precalculated data for segment endpoint.  Since the path is degenerate, there are several different cases:
+   * (1) The path consists of a single endpoint.  In this case, the degenerate path consists of this one point.
+   * (2) This is the end of a path.  There is a bounding plane passed in which describes the part of the world that is considered
+   *    to belong to this endpoint.
+   * (3) Intersection.  There are two cutoff planes, one for each end of the intersection.
+   */
+  private static class SegmentEndpoint {
+    /** The center point of the endpoint */
+    public final GeoPoint point;
+    /** Pertinent cutoff planes from adjoining segments */
+    public final Membership[] cutoffPlanes;
+    /** Notable points for this segment endpoint */
+    public final GeoPoint[] notablePoints;
+    /** No notable points from the circle itself */
+    public final static GeoPoint[] circlePoints = new GeoPoint[0];
+    /** Null membership */
+    public final static Membership[] NO_MEMBERSHIP = new Membership[0];
+    
+    /** Constructor for case (1).
+     *@param point is the center point.
+     */
+    public SegmentEndpoint(final GeoPoint point) {
+      this.point = point;
+      this.cutoffPlanes = NO_MEMBERSHIP;
+      this.notablePoints = circlePoints;
+    }
+    
+    /** Constructor for case (2).
+     * Generate an endpoint, given a single cutoff plane plus upper and lower edge points.
+     *@param point is the center point.
+     *@param cutoffPlane is the plane from the adjoining path segment marking the boundary between this endpoint and that segment.
+     */
+    public SegmentEndpoint(final GeoPoint point, final SidedPlane cutoffPlane) {
+      this.point = point;
+      this.cutoffPlanes = new Membership[]{new SidedPlane(cutoffPlane)};
+      this.notablePoints = new GeoPoint[]{point};
+    }
+
+    /** Constructor for case (3).
+     * Generate an endpoint, given two cutoff planes.
+     *@param point is the center.
+     *@param cutoffPlane1 is one adjoining path segment cutoff plane.
+     *@param cutoffPlane2 is another adjoining path segment cutoff plane.
+     */
+    public SegmentEndpoint(final GeoPoint point,
+      final SidedPlane cutoffPlane1, final SidedPlane cutoffPlane2) {
+      this.point = point;
+      this.cutoffPlanes = new Membership[]{new SidedPlane(cutoffPlane1), new SidedPlane(cutoffPlane2)};
+      this.notablePoints = new GeoPoint[]{point};
+    }
+    
+    /** Check if point is within this endpoint.
+     *@param point is the point.
+     *@return true of within.
+     */
+    public boolean isWithin(final Vector point) {
+      return this.point.isIdentical(point.x, point.y, point.z);
+    }
+
+    /** Check if point is within this endpoint.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return true of within.
+     */
+    public boolean isWithin(final double x, final double y, final double z) {
+      return this.point.isIdentical(x, y, z);
+    }
+
+    /** Compute interior path distance.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric, in aggregation form.
+     */
+    public double pathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      if (!isWithin(x,y,z))
+        return Double.POSITIVE_INFINITY;
+      return distanceStyle.toAggregationForm(distanceStyle.computeDistance(this.point, x, y, z));
+    }
+
+    /** Compute nearest path distance.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric (always value zero), in aggregation form, or POSITIVE_INFINITY
+     * if the point is not within the bounds of the endpoint.
+     */
+    public double nearestPathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      for (final Membership m : cutoffPlanes) {
+        if (!m.isWithin(x,y,z)) {
+          return Double.POSITIVE_INFINITY;
+        }
+      }
+      return distanceStyle.toAggregationForm(0.0);
+    }
+
+    /** Compute path center distance.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric, or POSITIVE_INFINITY
+     * if the point is not within the bounds of the endpoint.
+     */
+    public double pathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      for (final Membership m : cutoffPlanes) {
+        if (!m.isWithin(x,y,z)) {
+          return Double.POSITIVE_INFINITY;
+        }
+      }
+      return distanceStyle.computeDistance(this.point, x, y, z);
+    }
+
+    /** Compute external distance.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric.
+     */
+    public double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      return distanceStyle.computeDistance(this.point, x, y, z);
+    }
+
+    /** Determine if this endpoint intersects a specified plane.
+     *@param planetModel is the planet model.
+     *@param p is the plane.
+     *@param notablePoints are the points associated with the plane.
+     *@param bounds are any bounds which the intersection must lie within.
+     *@return true if there is a matching intersection.
+     */
+    public boolean intersects(final PlanetModel planetModel, final Plane p, final GeoPoint[] notablePoints, final Membership[] bounds) {
+    // If not on the plane, no intersection
+    if (!p.evaluateIsZero(point))
+      return false;
+
+    for (Membership m : bounds) {
+      if (!m.isWithin(point))
+        return false;
+    }
+    return true;
+    }
+
+    /** Determine if this endpoint intersects a GeoShape.
+     *@param geoShape is the GeoShape.
+     *@return true if there is shape intersect this endpoint.
+     */
+    public boolean intersects(final GeoShape geoShape) {
+      return geoShape.isWithin(point);
+    }
+
+    /** Get the bounds for a segment endpoint.
+     *@param planetModel is the planet model.
+     *@param bounds are the bounds to be modified.
+     */
+    public void getBounds(final PlanetModel planetModel, Bounds bounds) {
+      bounds.addPoint(point);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (!(o instanceof SegmentEndpoint))
+        return false;
+      SegmentEndpoint other = (SegmentEndpoint) o;
+      return point.equals(other.point);
+    }
+
+    @Override
+    public int hashCode() {
+      return point.hashCode();
+    }
+
+    @Override
+    public String toString() {
+      return point.toString();
+    }
+  }
+
+  /**
+   * This is the pre-calculated data for a path segment.
+   */
+  private static class PathSegment {
+    /** Starting point of the segment */
+    public final GeoPoint start;
+    /** End point of the segment */
+    public final GeoPoint end;
+    /** Place to keep any complete segment distances we've calculated so far */
+    public final Map<DistanceStyle,Double> fullDistanceCache = new HashMap<DistanceStyle,Double>();
+    /** Normalized plane connecting the two points and going through world center */
+    public final Plane normalizedConnectingPlane;
+    /** Plane going through the center and start point, marking the start edge of the segment */
+    public final SidedPlane startCutoffPlane;
+    /** Plane going through the center and end point, marking the end edge of the segment */
+    public final SidedPlane endCutoffPlane;
+    /** Notable points for the connecting plane */
+    public final GeoPoint[] connectingPlanePoints;
+
+    /** Construct a path segment.
+     *@param planetModel is the planet model.
+     *@param start is the starting point.
+     *@param end is the ending point.
+     *@param normalizedConnectingPlane is the connecting plane.
+     */
+    public PathSegment(final PlanetModel planetModel, final GeoPoint start, final GeoPoint end,
+      final Plane normalizedConnectingPlane) {
+      this.start = start;
+      this.end = end;
+      this.normalizedConnectingPlane = normalizedConnectingPlane;
+      
+      // Cutoff planes use opposite endpoints as correct side examples
+      startCutoffPlane = new SidedPlane(end, normalizedConnectingPlane, start);
+      endCutoffPlane = new SidedPlane(start, normalizedConnectingPlane, end);
+      connectingPlanePoints = new GeoPoint[]{start, end};
+    }
+
+    /** Compute the full distance along this path segment.
+     *@param distanceStyle is the distance style.
+     *@return the distance metric, in aggregation form.
+     */
+    public double fullPathDistance(final DistanceStyle distanceStyle) {
+      synchronized (fullDistanceCache) {
+        Double dist = fullDistanceCache.get(distanceStyle);
+        if (dist == null) {
+          dist = new Double(distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, end.x, end.y, end.z)));
+          fullDistanceCache.put(distanceStyle, dist);
+        }
+        return dist.doubleValue();
+      }
+    }
+  
+    /** Check if point is within this segment.
+     *@param point is the point.
+     *@return true of within.
+     */
+    public boolean isWithin(final Vector point) {
+      return startCutoffPlane.isWithin(point) &&
+          endCutoffPlane.isWithin(point) &&
+          normalizedConnectingPlane.evaluateIsZero(point);
+    }
+
+    /** Check if point is within this segment.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return true of within.
+     */
+    public boolean isWithin(final double x, final double y, final double z) {
+      return startCutoffPlane.isWithin(x, y, z) &&
+          endCutoffPlane.isWithin(x, y, z) &&
+          normalizedConnectingPlane.evaluateIsZero(x, y, z);
+    }
+
+    /** Compute path center distance.
+     *@param planetModel is the planet model.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric, or Double.POSITIVE_INFINITY if outside this segment
+     */
+    public double pathCenterDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      // First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
+      if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
+        return Double.POSITIVE_INFINITY;
+      }
+      // (1) Compute normalizedPerpPlane.  If degenerate, then there is no such plane, which means that the point given
+      // is insufficient to distinguish between a family of such planes.  This can happen only if the point is one of the
+      // "poles", imagining the normalized plane to be the "equator".  In that case, the distance returned should be zero.
+      // Want no allocations or expensive operations!  so we do this the hard way
+      final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
+      final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
+      final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
+      final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
+      if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
+        return distanceStyle.computeDistance(start, x, y, z);
+      final double normFactor = 1.0/magnitude;
+      final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
+      
+      final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
+      GeoPoint thePoint;
+      if (intersectionPoints.length == 0)
+        throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+      else if (intersectionPoints.length == 1)
+        thePoint = intersectionPoints[0];
+      else {
+        if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
+          thePoint = intersectionPoints[0];
+        else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
+          thePoint = intersectionPoints[1];
+        else
+          throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+      }
+      return distanceStyle.computeDistance(thePoint, x, y, z);
+    }
+    
+    /** Compute nearest path distance.
+     *@param planetModel is the planet model.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric, in aggregation form, or Double.POSITIVE_INFINITY if outside this segment
+     */
+    public double nearestPathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      // First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
+      if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
+        return Double.POSITIVE_INFINITY;
+      }
+      // (1) Compute normalizedPerpPlane.  If degenerate, then there is no such plane, which means that the point given
+      // is insufficient to distinguish between a family of such planes.  This can happen only if the point is one of the
+      // "poles", imagining the normalized plane to be the "equator".  In that case, the distance returned should be zero.
+      // Want no allocations or expensive operations!  so we do this the hard way
+      final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
+      final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
+      final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
+      final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
+      if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
+        return distanceStyle.toAggregationForm(0.0);
+      final double normFactor = 1.0/magnitude;
+      final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
+      
+      final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
+      GeoPoint thePoint;
+      if (intersectionPoints.length == 0)
+        throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+      else if (intersectionPoints.length == 1)
+        thePoint = intersectionPoints[0];
+      else {
+        if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
+          thePoint = intersectionPoints[0];
+        else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
+          thePoint = intersectionPoints[1];
+        else
+          throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+      }
+      return distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z));
+    }
+      
+
+    /** Compute interior path distance.
+     *@param planetModel is the planet model.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric, in aggregation form.
+     */
+    public double pathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      if (!isWithin(x,y,z))
+        return Double.POSITIVE_INFINITY;
+
+      // (1) Compute normalizedPerpPlane.  If degenerate, then return point distance from start to point.
+      // Want no allocations or expensive operations!  so we do this the hard way
+      final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
+      final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
+      final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
+      final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
+      if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
+        return distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, x,y,z));
+      final double normFactor = 1.0/magnitude;
+      final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
+      
+      // Old computation: too expensive, because it calculates the intersection point twice.
+      //return distanceStyle.computeDistance(planetModel, normalizedConnectingPlane, x, y, z, startCutoffPlane, endCutoffPlane) +
+      //  distanceStyle.computeDistance(planetModel, normalizedPerpPlane, start.x, start.y, start.z, upperConnectingPlane, lowerConnectingPlane);
+
+      final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
+      GeoPoint thePoint;
+      if (intersectionPoints.length == 0)
+        throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+      else if (intersectionPoints.length == 1)
+        thePoint = intersectionPoints[0];
+      else {
+        if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
+          thePoint = intersectionPoints[0];
+        else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
+          thePoint = intersectionPoints[1];
+        else
+          throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+      }
+      return distanceStyle.aggregateDistances(distanceStyle.toAggregationForm(distanceStyle.computeDistance(thePoint, x, y, z)),
+        distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z)));
+    }
+
+    /** Compute external distance.
+     *@param planetModel is the planet model.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric.
+     */
+    public double outsideDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      final double distance = distanceStyle.computeDistance(planetModel, normalizedConnectingPlane, x,y,z, startCutoffPlane, endCutoffPlane);
+      final double startDistance = distanceStyle.computeDistance(start, x,y,z);
+      final double endDistance = distanceStyle.computeDistance(end, x,y,z);
+      return Math.min(
+        Math.min(startDistance, endDistance),
+        distance);
+    }
+
+    /** Determine if this endpoint intersects a specified plane.
+     *@param planetModel is the planet model.
+     *@param p is the plane.
+     *@param notablePoints are the points associated with the plane.
+     *@param bounds are any bounds which the intersection must lie within.
+     *@return true if there is a matching intersection.
+     */
+    public boolean intersects(final PlanetModel planetModel, final Plane p, final GeoPoint[] notablePoints, final Membership[] bounds) {
+      return normalizedConnectingPlane.intersects(planetModel, p, connectingPlanePoints, notablePoints, bounds, startCutoffPlane, endCutoffPlane);
+    }
+
+    /** Determine if this endpoint intersects a specified GeoShape.
+     *@param geoShape is the GeoShape.
+     *@return true if there GeoShape intersects this endpoint.
+     */
+    public boolean intersects(final GeoShape geoShape) {
+      return geoShape.intersects(normalizedConnectingPlane, connectingPlanePoints, startCutoffPlane, endCutoffPlane);
+    }
+
+    /** Get the bounds for a segment endpoint.
+     *@param planetModel is the planet model.
+     *@param bounds are the bounds to be modified.
+     */
+    public void getBounds(final PlanetModel planetModel, Bounds bounds) {
+      // We need to do all bounding planes as well as corner points
+      bounds.addPoint(start).addPoint(end)
+        .addPlane(planetModel, normalizedConnectingPlane, startCutoffPlane, endCutoffPlane);
+    }
+
+  }
+
+}

lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java

@@ -125,7 +125,7 @@ class GeoDegeneratePoint extends GeoPoint implements GeoPointShape {
 
   @Override
   public boolean isWithin(final double x, final double y, final double z) {
-    return x == this.x && y == this.y && z == this.z;
+    return this.isIdentical(x, y, z);
   }
 
   @Override

lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java

@@ -22,4 +22,62 @@ package org.apache.lucene.spatial3d.geom;
  * @lucene.experimental
  */
 public interface GeoPath extends GeoDistanceShape {
+
+  // The following methods compute distances along the path from the shape to a point
+  // that doesn't need to be inside the shape.  The perpendicular distance from the path
+  // itself to the point is not included in the calculation.
+
+  /**
+   * Compute the nearest path distance to the GeoPoint.
+   * The path distance will not include the distance from the path itself to the
+   * point, but just the distance along the path to the nearest point on the path.
+   *
+   * @param distanceStyle is the distance style.
+   * @param point is the point to compute the distance to.
+   * @return the distance to the nearest path point.
+   */
+  public default double computeNearestDistance(final DistanceStyle distanceStyle, final GeoPoint point) {
+    return computeNearestDistance(distanceStyle, point.x, point.y, point.z);
+  }
+
+  /**
+   * Compute the nearest path distance to the GeoPoint.
+   * The path distance will not include the distance from the path itself to the
+   * point, but just the distance along the path to the nearest point on the path.
+   *
+   * @param x is the point's unit x coordinate (using U.S. convention).
+   * @param y is the point's unit y coordinate (using U.S. convention).
+   * @param z is the point's unit z coordinate (using U.S. convention).
+   * @return the distance to the nearest path point.
+   */
+  public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z);
+
+  // The following methods compute the best distance from the path center to the point.
+  
+  /**
+   * Compute the shortest distance from the path center to the GeoPoint.
+   * The distance is meant to allow comparisons between different
+   * paths to find the one that goes closest to a point.
+   *
+   * @param distanceStyle is the distance style.
+   * @param point is the point to compute the distance to.
+   * @return the shortest distance from the path center to the point.
+   */
+  public default double computePathCenterDistance(final DistanceStyle distanceStyle, final GeoPoint point) {
+    return computePathCenterDistance(distanceStyle, point.x, point.y, point.z);
+  }
+
+  /**
+   * Compute the shortest distance from the path center to the GeoPoint.
+   * The distance is meant to allow comparisons between different
+   * paths to find the one that goes closest to a point.
+   *
+   * @param distanceStyle is the distance style.
+   * @param x is the point's unit x coordinate (using U.S. convention).
+   * @param y is the point's unit y coordinate (using U.S. convention).
+   * @param z is the point's unit z coordinate (using U.S. convention).
+   * @return the shortest distance from the path center to the point.
+   */
+  public double computePathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z);
+
 }

lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java

@@ -33,6 +33,9 @@ public class GeoPathFactory {
    * @return a GeoPath corresponding to what was specified.
    */
   public static GeoPath makeGeoPath(final PlanetModel planetModel, final double maxCutoffAngle, final GeoPoint[] pathPoints) {
+    if (maxCutoffAngle < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+      return new GeoDegeneratePath(planetModel, pathPoints);
+    }
     return new GeoStandardPath(planetModel, maxCutoffAngle, pathPoints);
   }
 

lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java

@@ -214,6 +214,55 @@ class GeoStandardPath extends GeoBasePath {
     SerializableObject.writePointArray(outputStream, points);
   }
 
+  @Override
+  public double computePathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+    // Walk along path and keep track of the closest distance we find
+    double closestDistance = Double.POSITIVE_INFINITY;
+    // Segments first
+    for (PathSegment segment : segments) {
+      final double segmentDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
+      if (segmentDistance < closestDistance) {
+        closestDistance = segmentDistance;
+      }
+    }
+    // Now, endpoints
+    for (SegmentEndpoint endpoint : endPoints) {
+      final double endpointDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
+      if (endpointDistance < closestDistance) {
+        closestDistance = endpointDistance;
+      }
+    }
+    return closestDistance;
+  }
+
+  @Override
+  public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+    double currentDistance = 0.0;
+    double minPathCenterDistance = Double.POSITIVE_INFINITY;
+    double bestDistance = Double.POSITIVE_INFINITY;
+    int segmentIndex = 0;
+    
+    for (SegmentEndpoint endpoint : endPoints) {
+      final double endpointPathCenterDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
+      if (endpointPathCenterDistance < minPathCenterDistance) {
+        // Use this endpoint
+        minPathCenterDistance = endpointPathCenterDistance;
+        bestDistance = currentDistance;
+      }
+      // Look at the following segment, if any
+      if (segmentIndex < segments.size()) {
+        final PathSegment segment = segments.get(segmentIndex++);
+        final double segmentPathCenterDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
+        if (segmentPathCenterDistance < minPathCenterDistance) {
+          minPathCenterDistance = segmentPathCenterDistance;
+          bestDistance = distanceStyle.aggregateDistances(currentDistance, segment.nearestPathDistance(planetModel, distanceStyle, x, y, z));
+        }
+        currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
+      }
+    }
+    return bestDistance;
+  }
+
   @Override
   protected double distance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
     // Algorithm:
@@ -562,6 +611,40 @@ class GeoStandardPath extends GeoBasePath {
       return distanceStyle.toAggregationForm(distanceStyle.computeDistance(this.point, x, y, z));
     }
 
+    /** Compute nearest path distance.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric (always value zero), in aggregation form, or POSITIVE_INFINITY
+     * if the point is not within the bounds of the endpoint.
+     */
+    public double nearestPathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      for (final Membership m : cutoffPlanes) {
+        if (!m.isWithin(x,y,z)) {
+          return Double.POSITIVE_INFINITY;
+        }
+      }
+      return distanceStyle.toAggregationForm(0.0);
+    }
+
+    /** Compute path center distance.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric, or POSITIVE_INFINITY
+     * if the point is not within the bounds of the endpoint.
+     */
+    public double pathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      for (final Membership m : cutoffPlanes) {
+        if (!m.isWithin(x,y,z)) {
+          return Double.POSITIVE_INFINITY;
+        }
+      }
+      return distanceStyle.computeDistance(this.point, x, y, z);
+    }
+
     /** Compute external distance.
      *@param distanceStyle is the distance style.
      *@param x is the point x.
@@ -766,6 +849,93 @@ class GeoStandardPath extends GeoBasePath {
           lowerConnectingPlane.isWithin(x, y, z);
     }
 
+    /** Compute path center distance.
+     *@param planetModel is the planet model.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric, or Double.POSITIVE_INFINITY if outside this segment
+     */
+    public double pathCenterDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      // First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
+      if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
+        return Double.POSITIVE_INFINITY;
+      }
+      // (1) Compute normalizedPerpPlane.  If degenerate, then there is no such plane, which means that the point given
+      // is insufficient to distinguish between a family of such planes.  This can happen only if the point is one of the
+      // "poles", imagining the normalized plane to be the "equator".  In that case, the distance returned should be zero.
+      // Want no allocations or expensive operations!  so we do this the hard way
+      final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
+      final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
+      final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
+      final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
+      if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
+        return distanceStyle.computeDistance(start, x, y, z);
+      final double normFactor = 1.0/magnitude;
+      final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
+      
+      final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
+      GeoPoint thePoint;
+      if (intersectionPoints.length == 0)
+        throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+      else if (intersectionPoints.length == 1)
+        thePoint = intersectionPoints[0];
+      else {
+        if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
+          thePoint = intersectionPoints[0];
+        else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
+          thePoint = intersectionPoints[1];
+        else
+          throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+      }
+      return distanceStyle.computeDistance(thePoint, x, y, z);
+    }
+
+    /** Compute nearest path distance.
+     *@param planetModel is the planet model.
+     *@param distanceStyle is the distance style.
+     *@param x is the point x.
+     *@param y is the point y.
+     *@param z is the point z.
+     *@return the distance metric, in aggregation form, or Double.POSITIVE_INFINITY if outside this segment
+     */
+    public double nearestPathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+      // First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
+      if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
+        return Double.POSITIVE_INFINITY;
+      }
+      // (1) Compute normalizedPerpPlane.  If degenerate, then there is no such plane, which means that the point given
+      // is insufficient to distinguish between a family of such planes.  This can happen only if the point is one of the
+      // "poles", imagining the normalized plane to be the "equator".  In that case, the distance returned should be zero.
+      // Want no allocations or expensive operations!  so we do this the hard way
+      final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
+      final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
+      final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
+      final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
+      if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
+        return distanceStyle.toAggregationForm(0.0);
+      final double normFactor = 1.0/magnitude;
+      final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
+      
+      final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
+      GeoPoint thePoint;
+      if (intersectionPoints.length == 0)
+        throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+      else if (intersectionPoints.length == 1)
+        thePoint = intersectionPoints[0];
+      else {
+        if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
+          thePoint = intersectionPoints[0];
+        else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
+          thePoint = intersectionPoints[1];
+        else
+          throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+      }
+      return distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z));
+    }
+      
+
     /** Compute interior path distance.
      *@param planetModel is the planet model.
      *@param distanceStyle is the distance style.

lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardObjects.java

@@ -73,6 +73,7 @@ class StandardObjects {
    classRegsitry.put(XYdZSolid.class, 33);
    classRegsitry.put(StandardXYZSolid.class, 34);
    classRegsitry.put(PlanetModel.class, 35);
+   classRegsitry.put(GeoDegeneratePath.class, 36);
 
    for (Class<?> clazz : classRegsitry.keySet()){
      codeRegsitry.put(classRegsitry.get(clazz), clazz);

lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java

@@ -56,8 +56,10 @@ public class GeoPathTest {
     p.done();
     gp = new GeoPoint(PlanetModel.SPHERE, 0.05, 0.15);
     assertEquals(0.15 + 0.05, p.computeDistance(DistanceStyle.ARC,gp), 0.000001);
+    assertEquals(0.15, p.computeNearestDistance(DistanceStyle.ARC,gp), 0.000001);
     gp = new GeoPoint(PlanetModel.SPHERE, 0.0, 0.12);
     assertEquals(0.12, p.computeDistance(DistanceStyle.ARC,gp), 0.000001);
+    assertEquals(0.12, p.computeNearestDistance(DistanceStyle.ARC,gp), 0.000001);
 
     // Now try a vertical path, and make sure distances are as expected
     p = new GeoStandardPath(PlanetModel.SPHERE, 0.1);
@@ -302,4 +304,77 @@ public class GeoPathTest {
     assertTrue(solid.isWithin(point));
   }
   
+  @Test
+  public void testInterpolation() {
+    final double lat = 52.51607;
+    final double lon = 13.37698;
+    final double[] pathLats = new double[] {52.5355,52.54,52.5626,52.5665,52.6007,52.6135,52.6303,52.6651,52.7074};
+    final double[] pathLons = new double[] {13.3634,13.3704,13.3307,13.3076,13.2806,13.2484,13.2406,13.241,13.1926};
+
+    // Set up a point in the right way
+    final GeoPoint carPoint = new GeoPoint(PlanetModel.SPHERE, toRadians(lat), toRadians(lon));
+    // Create the path, but use a tiny width (e.g. zero)
+    final GeoPoint[] pathPoints = new GeoPoint[pathLats.length];
+    for (int i = 0; i < pathPoints.length; i++) {
+      pathPoints[i] = new GeoPoint(PlanetModel.SPHERE, toRadians(pathLats[i]), toRadians(pathLons[i]));
+    }
+    // Construct a path with no width
+    final GeoPath thisPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 0.0, pathPoints);
+    // Construct a path with a width
+    final GeoPath legacyPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 1e-6, pathPoints);
+    // Compute the inside distance to the atPoint using zero-width path
+    final double distance = thisPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
+    // Compute the inside distance using legacy path
+    final double legacyDistance = legacyPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
+    // Compute the inside distance using the legacy formula
+    final double oldFormulaDistance = thisPath.computeDistance(DistanceStyle.ARC, carPoint);
+    // Compute the inside distance using the legacy formula with the legacy shape
+    final double oldFormulaLegacyDistance = legacyPath.computeDistance(DistanceStyle.ARC, carPoint);
+
+    // These should be about the same
+    assertEquals(legacyDistance, distance, 1e-12);
+    assertEquals(oldFormulaLegacyDistance, oldFormulaDistance, 1e-12);
+    // This isn't true because example search center is off of the path.
+    //assertEquals(oldFormulaDistance, distance, 1e-12);
+
+  }
+
+  @Test
+  public void testInterpolation2() {
+    final double lat = 52.5665;
+    final double lon = 13.3076;
+    final double[] pathLats = new double[] {52.5355,52.54,52.5626,52.5665,52.6007,52.6135,52.6303,52.6651,52.7074};
+    final double[] pathLons = new double[] {13.3634,13.3704,13.3307,13.3076,13.2806,13.2484,13.2406,13.241,13.1926};
+
+    final GeoPoint carPoint = new GeoPoint(PlanetModel.SPHERE, toRadians(lat), toRadians(lon));
+    final GeoPoint[] pathPoints = new GeoPoint[pathLats.length];
+    for (int i = 0; i < pathPoints.length; i++) {
+      pathPoints[i] = new GeoPoint(PlanetModel.SPHERE, toRadians(pathLats[i]), toRadians(pathLons[i]));
+    }
+    
+    // Construct a path with no width
+    final GeoPath thisPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 0.0, pathPoints);
+    // Construct a path with a width
+    final GeoPath legacyPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 1e-6, pathPoints);
+    
+    // Compute the inside distance to the atPoint using zero-width path
+    final double distance = thisPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
+    // Compute the inside distance using legacy path
+    final double legacyDistance = legacyPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
+    
+    // Compute the inside distance using the legacy formula
+    final double oldFormulaDistance = thisPath.computeDistance(DistanceStyle.ARC, carPoint);
+    // Compute the inside distance using the legacy formula with the legacy shape
+    final double oldFormulaLegacyDistance = legacyPath.computeDistance(DistanceStyle.ARC, carPoint);
+
+    // These should be about the same
+    assertEquals(legacyDistance, distance, 1e-12);
+    
+    assertEquals(oldFormulaLegacyDistance, oldFormulaDistance, 1e-12);
+    
+    // Since the point we picked is actually on the path, this should also be true
+    assertEquals(oldFormulaDistance, distance, 1e-12);
+
+  }
+
 }

lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomBinaryCodecTest.java

@@ -27,18 +27,18 @@ import org.junit.Test;
 /**
  * Test to check Serialization
  */
-public class RandomBinaryCodecTest extends RandomGeoShapeGenerator{
+public class RandomBinaryCodecTest extends RandomGeo3dShapeGenerator {
 
   @Test
   @Repeat(iterations = 10)
   public void testRandomPointCodec() throws IOException{
     PlanetModel planetModel = randomPlanetModel();
-    GeoPoint shape = randomGeoPoint(planetModel, getEmptyConstraint());
+    GeoPoint shape = randomGeoPoint(planetModel);
     ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
     SerializableObject.writeObject(outputStream, shape);
     ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
     SerializableObject shapeCopy = SerializableObject.readObject(planetModel, inputStream);
-    assertEquals(shape, shapeCopy);
+    assertEquals(shape.toString(), shape, shapeCopy);
   }
 
   @Test
@@ -51,7 +51,7 @@ public class RandomBinaryCodecTest extends RandomGeoShapeGenerator{
     SerializableObject.writePlanetObject(outputStream, shape);
     ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
     SerializableObject shapeCopy = SerializableObject.readPlanetObject(inputStream);
-    assertEquals(shape, shapeCopy);
+    assertEquals(shape.toString(), shape, shapeCopy);
   }
 
   @Test
@@ -64,6 +64,6 @@ public class RandomBinaryCodecTest extends RandomGeoShapeGenerator{
     SerializableObject.writeObject(outputStream, shape);
     ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
     SerializableObject shapeCopy = SerializableObject.readObject(planetModel, inputStream);
-    assertEquals(shape, shapeCopy);
+    assertEquals(shape.toString(), shape, shapeCopy);
   }
 }

lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java

@@ -36,7 +36,7 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomDouble;
  * created using GeoConvexPolygons and GeoConcavePolygons.
  *
  */
-public class RandomGeoShapeGenerator extends LuceneTestCase {
+public class RandomGeo3dShapeGenerator extends LuceneTestCase {
 
   /* Max num of iterations to find right shape under given constrains */
   final private static int MAX_SHAPE_ITERATIONS = 50;
@@ -53,19 +53,31 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
   final protected static int RECTANGLE = 6;
   final protected static int PATH = 7;
   final protected static int COLLECTION = 8;
+  final protected static int POINT = 9;
+  final protected static int LINE = 10;
 
   /* Helper shapes for generating constraints whch are just three sided polygons */
   final protected static int CONVEX_SIMPLE_POLYGON = 500;
   final protected static int CONCAVE_SIMPLE_POLYGON = 501;
 
-
   /**
-   * Method that returns empty Constraints object..
+   * Method that returns a random generated Planet model from the supported
+   * Planet models. currently SPHERE and WGS84
    *
-   * @return an empty Constraints object
+   * @return a random generated Planet model
    */
-  public Constraints getEmptyConstraint(){
+  public PlanetModel randomPlanetModel() {
-    return new Constraints();
+    final int shapeType = random().nextInt(2);
+    switch (shapeType) {
+      case 0: {
+        return PlanetModel.SPHERE;
+      }
+      case 1: {
+        return PlanetModel.WGS84;
+      }
+      default:
+        throw new IllegalStateException("Unexpected planet model");
+    }
   }
 
   /**
@@ -75,7 +87,20 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
    * @return a random generated shape code
    */
   public int randomShapeType(){
-    return random().nextInt(9);
+    return random().nextInt(11);
+  }
+
+  /**
+   * Method that returns a random generated GeoAreaShape code from all
+   * supported GeoAreaShapes.
+   *
+   * We are removing Collections because it is difficult to create shapes
+   * with properties in some cases.
+   *
+   * @return a random generated polygon code
+   */
+  public int randomGeoAreaShapeType(){
+    return random().nextInt(11);
   }
 
   /**
@@ -106,19 +131,6 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
     return shapeType;
   }
 
-  /**
-   * Method that returns a random generated GeoAreaShape code from all
-   * supported GeoAreaShapes.
-   *
-   * We are removing Collections because it is difficult to create shapes
-   * with properties in some cases.
-   *
-   * @return a random generated polygon code
-   */
-  public int randomGeoAreaShapeType(){
-    return random().nextInt(8);
-  }
-
   /**
    * Check if a shape code represents a concave shape
    *
@@ -129,23 +141,26 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
   }
 
   /**
-   * Method that returns a random generated Planet model from the supported
+   * Method that returns empty Constraints object..
-   * Planet models. currently SPHERE and WGS84
    *
-   * @return a random generated Planet model
+   * @return an empty Constraints object
    */
-  public PlanetModel randomPlanetModel() {
+  public Constraints getEmptyConstraint(){
-    final int shapeType = random().nextInt(2);
+    return new Constraints();
-    switch (shapeType) {
-      case 0: {
-        return PlanetModel.SPHERE;
   }
-      case 1: {
+
-        return PlanetModel.WGS84;
+  /**
-      }
+   * Method that returns a random generated GeoPoint.
-      default:
+   *
-        throw new IllegalStateException("Unexpected planet model");
+   * @param planetModel The planet model.
+   * @return The random generated GeoPoint.
+   */
+  public GeoPoint randomGeoPoint(PlanetModel planetModel) {
+    GeoPoint point = null;
+    while (point == null) {
+      point = randomGeoPoint(planetModel, getEmptyConstraint());
     }
+    return point;
   }
 
   /**
@@ -159,13 +174,13 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
   public GeoPoint randomGeoPoint(PlanetModel planetModel, Constraints constraints) {
     int iterations = 0;
     while (iterations < MAX_POINT_ITERATIONS) {
-      double lat = randomDouble();
+      double lat = randomDouble() * Math.PI/2;
-      if (Math.PI/2 - Math.abs(lat)  <0){
+      if (random().nextBoolean()) {
-        continue;
+        lat = (-1)*lat;
       }
-      double lon =  randomDouble();
+      double lon =  randomDouble() * Math.PI;
-      if (Math.PI - Math.abs(lat)   <0){
+      if (random().nextBoolean()) {
-        continue;
+        lon = (-1)*lon;
       }
       iterations++;
       GeoPoint point = new GeoPoint(planetModel, lat, lon);
@@ -257,6 +272,12 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
       case COLLECTION: {
         return collection(planetModel, constraints);
       }
+      case POINT: {
+        return point(planetModel, constraints);
+      }
+      case LINE: {
+        return line(planetModel, constraints);
+      }
       case CONVEX_SIMPLE_POLYGON: {
         return simpleConvexPolygon(planetModel, constraints);
       }
@@ -268,6 +289,36 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
     }
   }
 
+  /**
+   * Method that returns a random generated a GeoPointShape under given constraints. Returns
+   * NULL if it cannot build the GeoCircle under the given constraints.
+   *
+   * @param planetModel The planet model.
+   * @param constraints The given constraints.
+   * @return The random generated GeoPointShape.
+   */
+  private GeoPointShape point(PlanetModel planetModel , Constraints constraints) {
+    int iterations=0;
+    while (iterations < MAX_SHAPE_ITERATIONS) {
+      iterations++;
+      final GeoPoint point = randomGeoPoint(planetModel, constraints);
+      if (point == null){
+        continue;
+      }
+      try {
+
+        GeoPointShape pointShape = GeoPointShapeFactory.makeGeoPointShape(planetModel, point.getLatitude(), point.getLongitude());
+        if (!constraints.valid(pointShape)) {
+          continue;
+        }
+        return pointShape;
+      } catch (IllegalArgumentException e) {
+        continue;
+      }
+    }
+    return null;
+  }
+
   /**
    * Method that returns a random generated a GeoCircle under given constraints. Returns
    * NULL if it cannot build the GeoCircle under the given constraints.
@@ -339,6 +390,33 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
     return null;
   }
 
+  /**
+   * Method that returns a random generated degenerate GeoPath under given constraints. Returns
+   * NULL if it cannot build the degenerate GeoPath under the given constraints.
+   *
+   * @param planetModel The planet model.
+   * @param constraints The given constraints.
+   * @return The random generated degenerated GeoPath.
+   */
+  private GeoPath line(PlanetModel planetModel, Constraints constraints) {
+    int iterations = 0;
+    while (iterations < MAX_SHAPE_ITERATIONS) {
+      iterations++;
+      int vertexCount =  random().nextInt(2) + 2;
+      List<GeoPoint> geoPoints = points(vertexCount, planetModel, constraints);
+      try {
+        GeoPath path = GeoPathFactory.makeGeoPath(planetModel, 0, geoPoints.toArray(new GeoPoint[geoPoints.size()]));
+        if (!constraints.valid(path)) {
+          continue;
+        }
+        return path;
+      } catch (IllegalArgumentException e) {
+        continue;
+      }
+    }
+    return null;
+  }
+
   /**
    * Method that returns a random generated a GeoPath under given constraints. Returns
    * NULL if it cannot build the GeoPath under the given constraints.
@@ -732,13 +810,7 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
    * @return the cutoff angle.
    */
   private double randomCutoffAngle() {
-    while(true) {
+    return randomDouble() * Math.PI;
-      double radius = randomDouble();
-      if (radius <0 || radius > Math.PI){
-        continue;
-      }
-      return radius;
-    }
   }
 
   /**
@@ -939,6 +1011,3 @@ public class RandomGeoShapeGenerator extends LuceneTestCase {
     }
   }
 }
-
-
-

lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeRelationshipTest.java

@@ -23,8 +23,7 @@ import org.junit.Test;
 /**
  * Random test to check relationship between GeoAreaShapes and GeoShapes.
  */
-public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
+public class RandomGeoShapeRelationshipTest extends RandomGeo3dShapeGenerator {
-
 
   /**
    * Test for WITHIN points. We build a WITHIN shape with respect the geoAreaShape
@@ -38,6 +37,9 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
     int referenceShapeType = CONVEX_POLYGON;
     PlanetModel planetModel = randomPlanetModel();
     int shapeType = randomShapeType();
+    while (shapeType == POINT || shapeType == LINE) {
+      shapeType = randomShapeType();
+    }
     GeoAreaShape shape = null;
     GeoPoint point = null;
     while (point == null) {
@@ -51,7 +53,10 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
         point = randomGeoPoint(planetModel, constraints);
       }
     }
-    assertTrue(shape.isWithin(point));
+    StringBuilder b = new StringBuilder();
+    b.append("shape: " + shape + "\n");
+    b.append("point: " + point);
+    assertTrue(b.toString(), shape.isWithin(point));
   }
 
   /**
@@ -78,7 +83,10 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
         point = randomGeoPoint(planetModel, constraints);
       }
     }
-    assertFalse(shape.isWithin(point));
+    StringBuilder b = new StringBuilder();
+    b.append("shape: " + shape + "\n");
+    b.append("point: " + point);
+    assertFalse(b.toString(), shape.isWithin(point));
   }
 
   /**
@@ -109,11 +117,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
         shape = randomGeoShape(shapeType, planetModel, constraints);
       }
     }
+    StringBuilder b = new StringBuilder();
+    b.append("geoAreaShape: " + geoAreaShape + "\n");
+    b.append("shape: " + shape);
     int rel = geoAreaShape.getRelationship(shape);
-    assertEquals(GeoArea.DISJOINT, rel);
+    assertEquals(b.toString(), GeoArea.DISJOINT, rel);
     if (shape instanceof GeoArea) {
       rel = ((GeoArea)shape).getRelationship(geoAreaShape);
-      assertEquals(GeoArea.DISJOINT, rel);
+      assertEquals(b.toString(), GeoArea.DISJOINT, rel);
     }
   }
 
@@ -129,7 +140,11 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
   public void testRandomWithIn() {
     PlanetModel planetModel = randomPlanetModel();
     int geoAreaShapeType = randomGeoAreaShapeType();
-    int shapeType =randomShapeType();
+    //shapes cannot be point or line -- no area!
+    while(geoAreaShapeType == POINT || geoAreaShapeType == LINE) {
+      geoAreaShapeType = randomGeoAreaShapeType();
+    }
+    int shapeType = LINE;//randomShapeType();
     int referenceShapeType = CONVEX_SIMPLE_POLYGON;
     if (!isConcave(geoAreaShapeType)){
       shapeType =randomConvexShapeType();
@@ -150,11 +165,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
         shape = randomGeoShape(shapeType, planetModel, constraints);
       }
     }
+    StringBuilder b = new StringBuilder();
+    b.append("geoAreaShape: " + geoAreaShape + "\n");
+    b.append("shape: " + shape);
     int rel = geoAreaShape.getRelationship(shape);
-    assertEquals(GeoArea.WITHIN, rel);
+    assertEquals(b.toString(), GeoArea.WITHIN, rel);
     if (shape instanceof GeoArea) {
       rel = ((GeoArea)shape).getRelationship(geoAreaShape);
-      assertEquals(GeoArea.CONTAINS, rel);
+      assertEquals(b.toString(), GeoArea.CONTAINS, rel);
     }
   }
 
@@ -178,6 +196,9 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
       geoAreaShapeType = randomGeoAreaShapeType();
     }
     int shapeType = randomShapeType();
+    while (shapeType == POINT || shapeType == LINE) {
+      shapeType = randomShapeType();
+    }
     if (isConcave(geoAreaShapeType)){
       shapeType = randomConcaveShapeType();
     }
@@ -197,11 +218,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
         shape = randomGeoShape(shapeType, planetModel, constraints);
       }
     }
+    StringBuilder b = new StringBuilder();
+    b.append("geoAreaShape: " + geoAreaShape + "\n");
+    b.append("shape: " + shape);
     int rel = geoAreaShape.getRelationship(shape);
-    assertEquals(GeoArea.CONTAINS, rel);
+    assertEquals(b.toString(), GeoArea.CONTAINS, rel);
     if (shape instanceof GeoArea) {
       rel = ((GeoArea)shape).getRelationship(geoAreaShape);
-      assertEquals(GeoArea.WITHIN, rel);
+      assertEquals(b.toString(), GeoArea.WITHIN, rel);
     }
   }
 
@@ -216,8 +240,13 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
   public void testRandomOverlaps() {
     PlanetModel planetModel = randomPlanetModel();
     int geoAreaShapeType = randomGeoAreaShapeType();
+    while (geoAreaShapeType == POINT || geoAreaShapeType == LINE) {
+      geoAreaShapeType = randomGeoAreaShapeType();
+    }
     int shapeType = randomShapeType();
-
+    while (shapeType == POINT || shapeType == LINE) {
+      shapeType = randomShapeType();
+    }
     GeoShape shape = null;
     GeoAreaShape geoAreaShape = null;
     while (shape == null) {
@@ -246,12 +275,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
         shape = randomGeoShape(shapeType, planetModel, constraints);
       }
     }
+    StringBuilder b = new StringBuilder();
+    b.append("geoAreaShape: " + geoAreaShape + "\n");
+    b.append("shape: " + shape);
     int rel = geoAreaShape.getRelationship(shape);
-    assertEquals(GeoArea.OVERLAPS, rel);
+    assertEquals(b.toString(), GeoArea.OVERLAPS, rel);
     if (shape instanceof GeoArea) {
       rel = ((GeoArea)shape).getRelationship(geoAreaShape);
-      assertEquals(GeoArea.OVERLAPS, rel);
+      assertEquals(b.toString(), GeoArea.OVERLAPS, rel);
     }
   }
 }
-

lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java

@@ -24,6 +24,7 @@ import java.util.List;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Check relationship between polygon and GeoShapes of basic polygons. Normally we construct
@@ -736,6 +737,29 @@ public class SimpleGeoPolygonRelationshipsTest {
     assertEquals(GeoArea.CONTAINS,  rel);
   }
 
+  @Test
+  public void testDegeneratedPointInPole(){
+    GeoBBox bBox1 = GeoBBoxFactory.makeGeoBBox(PlanetModel.SPHERE, Math.PI*0.5, Math.PI*0.5, 0, 0);
+    GeoPoint point = new GeoPoint(PlanetModel.SPHERE, Math.PI*0.5,  Math.PI);
+    System.out.println("bbox1 = "+bBox1+"; point = "+point);
+    assertTrue(bBox1.isWithin(point));
+  }
+
+  @Test
+  public void testDegeneratePathShape(){
+    GeoPoint point1 = new GeoPoint(PlanetModel.SPHERE, 0, 0);
+    GeoPoint point2 = new GeoPoint(PlanetModel.SPHERE, 0, 1);
+    GeoPoint[] pointPath1 = new GeoPoint[] {point1, point2};
+    GeoPath path1 = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 0, pointPath1);
+    GeoPath path2 = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 1, pointPath1);
+    int rel = path1.getRelationship(path2);
+    //if an end point is inside the shape it will always return intersects
+    assertEquals(GeoArea.CONTAINS, rel); //should be contains?
+    rel = path2.getRelationship(path1);
+    assertEquals(GeoArea.WITHIN, rel);
+  }
+
+
   private GeoPolygon buildConvexGeoPolygon(double lon1, double lat1,
                                            double lon2, double lat2,
                                            double lon3, double lat3,

lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java

@@ -390,9 +390,11 @@ public class AnalyzingSuggester extends Lookup implements Accountable {
       } else {
         scratchA.offset = readerA.getPosition();
         scratchB.offset = readerB.getPosition();
-        scratchA.length = a.length - scratchA.offset;
+        scratchA.length = readerA.length() - readerA.getPosition();
-        scratchB.length = b.length - scratchB.offset;
+        scratchB.length = readerB.length() - readerB.getPosition();
       }
+      assert scratchA.isValid();
+      assert scratchB.isValid();
    
       return scratchA.compareTo(scratchB);
     }

lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java

@@ -1111,6 +1111,66 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
     IOUtils.close(a, tempDir);
   }
 
+  /**
+   * Adds 50 random keys, that all analyze to the same thing (dog), with the same cost,
+   * and checks that they come back in surface-form order.
+   */
+  public void testTieBreakOnSurfaceForm() throws Exception {
+    Analyzer a = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName) {
+        Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
+
+        return new TokenStreamComponents(tokenizer) {
+          @Override
+          public TokenStream getTokenStream() {
+            return new CannedTokenStream(new Token[] {
+                token("dog", 1, 1)
+              });
+          }
+          @Override
+          protected void setReader(final Reader reader) {
+          }
+        };
+      }
+    };
+
+    Directory tempDir = getDirectory();
+    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
+
+    // make 50 inputs all with the same cost of 1, random strings
+    Input[] inputs = new Input[100];
+    for (int i = 0; i < inputs.length; i++) {
+      inputs[i] = new Input(TestUtil.randomSimpleString(random()), 1);
+    }
+
+    suggester.build(new InputArrayIterator(inputs));
+
+    // Try to save/load:
+    Path tmpDir = createTempDir("AnalyzingSuggesterTest");
+    Path path = tmpDir.resolve("suggester");
+
+    OutputStream os = Files.newOutputStream(path);
+    suggester.store(os);
+    os.close();
+
+    InputStream is = Files.newInputStream(path);
+    suggester.load(is);
+    is.close();
+
+    // now suggest everything, and check that stuff comes back in order
+    List<LookupResult> results = suggester.lookup("", false, 50);
+    assertEquals(50, results.size());
+    for (int i = 1; i < 50; i++) {
+      String previous = results.get(i-1).toString();
+      String current = results.get(i).toString();
+      assertTrue("surface forms out of order: previous=" + previous + ",current=" + current,
+                 current.compareTo(previous) >= 0);
+    }
+
+    IOUtils.close(a, tempDir);
+  }
+
   public void test0ByteKeys() throws Exception {
     final Analyzer a = new Analyzer() {
         @Override

solr/CHANGES.txt

@@ -100,6 +100,13 @@ New Features
 
 * SOLR-11072: Implement trigger for searchRate event type. (ab)
 
+* SOLR-11317: JSON Facet API: min/max aggregations on numeric fields are now typed better so int/long
+  fields return an appropriate integral type rather than a double. (yonik)
+
+* SOLR-11316: JSON Facet API: min/max aggregations are now supported on single-valued date fields.
+  (yonik)
+
+
 Bug Fixes
 ----------------------
 
@@ -142,6 +149,12 @@ Bug Fixes
 
 * SOLR-11293: Potential data loss in TLOG replicas after replication failures (noble)
 
+* SOLR-10101: TestLazyCores hangs (Erick Erickson)
+
+* SOLR-11332: Fix sorting on 'enum' fieldTypes that use sortMissingFirst or sortMissingLast (hossman)
+
+* SOLR-11348: Fix the DIH database example (James Dyer)
+
 Optimizations
 ----------------------
 
@@ -154,6 +167,10 @@ Optimizations
 
 * SOLR-10769: Allow nodeAdded / nodeLost events to report multiple nodes in one event. (ab)
 
+* SOLR-11242: QueryParser: re-use the LookaheadSuccess exception. (Michael Braun via David Smiley)
+
+* SOLR-11314: FastCharStream: re-use the READ_PAST_EOF exception. (Michael Braun via David Smiley)
+
 Other Changes
 ----------------------
 
@@ -216,6 +233,15 @@ Other Changes
 
 * SOLR-11209: Upgrade HttpClient to 4.5.3. (Hrishikesh Gadre via Mark Miller)
 
+* SOLR-11322: JSON Facet API: instead of returning NaN, min & max aggregations omit
+  the value for any bucket with no values in the numeric field. (yonik)
+
+* SOLR-10990: Breakup QueryComponent.process method for readability. (Christine Poerschke)
+
+* SOLR-11132: Refactor common getSortField logic in various FieldTypes (Jason Gerlowski, hossman)
+
+* SOLR-11351: Make LTRScoringModel model more extensible. (Christine Poerschke)
+
 ==================  7.0.0 ==================
 
 Versions of Major Components
@@ -848,7 +874,7 @@ Other Changes
 * SOLR-10494: Make default response format JSON (wt=json), and also indent text responses formats
   (indent=on) by default (Trey Grainger & Cassandra Targett via hossman)
 
-* SOLR-10760: Remove trie field types and fields from example schemas. (Steve Rowe)
+* SOLR-10760,SOLR-11315,SOLR-11313: Remove trie field types and fields from example schemas. (Steve Rowe)
 
 * SOLR-11056: Add random range query test that compares results across Trie*, *Point and DocValue-only fields
   (Tomás Fernández Löbbe)
@@ -942,6 +968,8 @@ Other Changes
 * SOLR-11261, SOLR-10966: Upgrade to Hadoop 2.7.4 to fix incompatibility with Java 9.
   (Uwe Schindler)
 
+* SOLR-11324: Clean up mention of trie fields in documentation and source comments. (Steve Rowe)
+
 ==================  6.6.1 ==================
 
 Bug Fixes

solr/build.xml

@@ -710,6 +710,20 @@
     </sequential>
   </target>
 
+  <target name="-install-to-maven-local-repo" depends="install-maven-tasks">
+    <sequential>
+      <m2-install pom.xml="${filtered.pom.templates.dir}/solr/pom.xml"/>    <!-- Solr parent POM -->
+      <subant target="-install-to-maven-local-repo" inheritall="false">
+        <propertyset refid="uptodate.and.compiled.properties"/>
+        <fileset dir="core" includes="build.xml"/>
+        <fileset dir="solrj" includes="build.xml"/>
+        <fileset dir="test-framework" includes="build.xml"/>
+        <fileset dir="webapp" includes="build.xml"/>
+      </subant>
+      <contrib-crawl target="-install-to-maven-local-repo"/>
+    </sequential>
+  </target>
+
   <target name="generate-maven-artifacts" depends="-unpack-solr-tgz">
     <ant dir=".." target="resolve" inheritall="false"/>
     <antcall target="-filter-pom-templates" inheritall="false"/>

solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java

@@ -80,7 +80,7 @@ public abstract class LTRScoringModel {
   protected final List<Feature> features;
   private final List<Feature> allFeatures;
   private final Map<String,Object> params;
-  private final List<Normalizer> norms;
+  protected final List<Normalizer> norms;
 
   public static LTRScoringModel getInstance(SolrResourceLoader solrResourceLoader,
       String className, String name, List<Feature> features,
@@ -123,6 +123,8 @@ public abstract class LTRScoringModel {
    * {@link ModelException} if they do not make sense.
    */
   protected void validate() throws ModelException {
+    final List<Feature> features = getFeatures();
+    final List<Normalizer> norms = getNorms();
     if (features.isEmpty()) {
       throw new ModelException("no features declared for model "+name);
     }

solr/contrib/ltr/src/test/org/apache/solr/ltr/TestRerankBase.java

@@ -239,12 +239,14 @@ public class TestRerankBase extends RestTestBase {
         .append(",\n");
     sb.append("\"class\":").append('"').append(type).append('"').append(",\n");
     sb.append("\"features\":").append('[');
+    if (features.length > 0) {
       for (final String feature : features) {
         sb.append("\n\t{ ");
         sb.append("\"name\":").append('"').append(feature).append('"')
         .append("},");
       }
       sb.deleteCharAt(sb.length() - 1);
+    }
     sb.append("\n]\n");
     if (params != null) {
       sb.append(",\n");

solr/core/build.xml

@@ -60,6 +60,8 @@
 
   <target name="-dist-maven" depends="-dist-maven-src-java"/>
 
+  <target name="-install-to-maven-local-repo" depends="-install-src-java-to-maven-local-repo"/>
+
   <target name="resolve" depends="ivy-availability-check,ivy-fail,ivy-configure">
     <sequential>
       <ivy:retrieve conf="compile,compile.hadoop" type="jar,bundle" sync="${ivy.sync}" log="download-only" symlink="${ivy.symlink}"/>
@@ -83,6 +85,11 @@
                      byline="true"
                      match="public QueryParser\(QueryParserTokenManager "
                      replace="protected QueryParser(QueryParserTokenManager "/>
+      <!-- change an exception used for signaling to be static -->
+      <replaceregexp file="src/java/org/apache/solr/parser/QueryParser.java"
+                     byline="true"
+                     match="final private LookaheadSuccess jj_ls ="
+                     replace="static final private LookaheadSuccess jj_ls =" />
       <replace token="StringBuffer" value="StringBuilder" encoding="UTF-8">
          <fileset dir="src/java/org/apache/solr/parser" includes="ParseException.java TokenMgrError.java"/>
       </replace>

solr/core/src/java/org/apache/solr/handler/StreamHandler.java

@@ -33,84 +33,7 @@ import org.apache.solr.client.solrj.io.ModelCache;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
-import org.apache.solr.client.solrj.io.eval.AbsoluteValueEvaluator;
+import org.apache.solr.client.solrj.io.eval.*;
-import org.apache.solr.client.solrj.io.eval.AddEvaluator;
-import org.apache.solr.client.solrj.io.eval.AndEvaluator;
-import org.apache.solr.client.solrj.io.eval.AnovaEvaluator;
-import org.apache.solr.client.solrj.io.eval.AppendEvaluator;
-import org.apache.solr.client.solrj.io.eval.ArcCosineEvaluator;
-import org.apache.solr.client.solrj.io.eval.ArcSineEvaluator;
-import org.apache.solr.client.solrj.io.eval.ArcTangentEvaluator;
-import org.apache.solr.client.solrj.io.eval.ArrayEvaluator;
-import org.apache.solr.client.solrj.io.eval.AscEvaluator;
-import org.apache.solr.client.solrj.io.eval.CeilingEvaluator;
-import org.apache.solr.client.solrj.io.eval.CoalesceEvaluator;
-import org.apache.solr.client.solrj.io.eval.ColumnEvaluator;
-import org.apache.solr.client.solrj.io.eval.ConversionEvaluator;
-import org.apache.solr.client.solrj.io.eval.ConvolutionEvaluator;
-import org.apache.solr.client.solrj.io.eval.CopyOfEvaluator;
-import org.apache.solr.client.solrj.io.eval.CopyOfRangeEvaluator;
-import org.apache.solr.client.solrj.io.eval.CorrelationEvaluator;
-import org.apache.solr.client.solrj.io.eval.CosineEvaluator;
-import org.apache.solr.client.solrj.io.eval.CovarianceEvaluator;
-import org.apache.solr.client.solrj.io.eval.CubedRootEvaluator;
-import org.apache.solr.client.solrj.io.eval.CumulativeProbabilityEvaluator;
-import org.apache.solr.client.solrj.io.eval.DescribeEvaluator;
-import org.apache.solr.client.solrj.io.eval.DivideEvaluator;
-import org.apache.solr.client.solrj.io.eval.EmpiricalDistributionEvaluator;
-import org.apache.solr.client.solrj.io.eval.EqualToEvaluator;
-import org.apache.solr.client.solrj.io.eval.EuclideanDistanceEvaluator;
-import org.apache.solr.client.solrj.io.eval.ExclusiveOrEvaluator;
-import org.apache.solr.client.solrj.io.eval.FindDelayEvaluator;
-import org.apache.solr.client.solrj.io.eval.FloorEvaluator;
-import org.apache.solr.client.solrj.io.eval.GreaterThanEqualToEvaluator;
-import org.apache.solr.client.solrj.io.eval.GreaterThanEvaluator;
-import org.apache.solr.client.solrj.io.eval.HistogramEvaluator;
-import org.apache.solr.client.solrj.io.eval.HyperbolicCosineEvaluator;
-import org.apache.solr.client.solrj.io.eval.HyperbolicSineEvaluator;
-import org.apache.solr.client.solrj.io.eval.HyperbolicTangentEvaluator;
-import org.apache.solr.client.solrj.io.eval.IfThenElseEvaluator;
-import org.apache.solr.client.solrj.io.eval.KolmogorovSmirnovEvaluator;
-import org.apache.solr.client.solrj.io.eval.LengthEvaluator;
-import org.apache.solr.client.solrj.io.eval.LessThanEqualToEvaluator;
-import org.apache.solr.client.solrj.io.eval.LessThanEvaluator;
-import org.apache.solr.client.solrj.io.eval.ModuloEvaluator;
-import org.apache.solr.client.solrj.io.eval.MovingAverageEvaluator;
-import org.apache.solr.client.solrj.io.eval.MultiplyEvaluator;
-import org.apache.solr.client.solrj.io.eval.NaturalLogEvaluator;
-import org.apache.solr.client.solrj.io.eval.NormalDistributionEvaluator;
-import org.apache.solr.client.solrj.io.eval.NormalizeEvaluator;
-import org.apache.solr.client.solrj.io.eval.NotEvaluator;
-import org.apache.solr.client.solrj.io.eval.OrEvaluator;
-import org.apache.solr.client.solrj.io.eval.PercentileEvaluator;
-import org.apache.solr.client.solrj.io.eval.PowerEvaluator;
-import org.apache.solr.client.solrj.io.eval.PredictEvaluator;
-import org.apache.solr.client.solrj.io.eval.RankEvaluator;
-import org.apache.solr.client.solrj.io.eval.RawValueEvaluator;
-import org.apache.solr.client.solrj.io.eval.RegressionEvaluator;
-import org.apache.solr.client.solrj.io.eval.ResidualsEvaluator;
-import org.apache.solr.client.solrj.io.eval.ReverseEvaluator;
-import org.apache.solr.client.solrj.io.eval.RoundEvaluator;
-import org.apache.solr.client.solrj.io.eval.SampleEvaluator;
-import org.apache.solr.client.solrj.io.eval.ScaleEvaluator;
-import org.apache.solr.client.solrj.io.eval.SequenceEvaluator;
-import org.apache.solr.client.solrj.io.eval.SineEvaluator;
-import org.apache.solr.client.solrj.io.eval.SquareRootEvaluator;
-import org.apache.solr.client.solrj.io.eval.SubtractEvaluator;
-import org.apache.solr.client.solrj.io.eval.TangentEvaluator;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDay;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDayOfQuarter;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDayOfYear;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorEpoch;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorHour;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorMinute;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorMonth;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorQuarter;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorSecond;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorWeek;
-import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorYear;
-import org.apache.solr.client.solrj.io.eval.UniformDistributionEvaluator;
-import org.apache.solr.client.solrj.io.eval.UuidEvaluator;
 import org.apache.solr.client.solrj.io.graph.GatherNodesStream;
 import org.apache.solr.client.solrj.io.graph.ShortestPathStream;
 import org.apache.solr.client.solrj.io.ops.ConcatOperation;
@@ -326,8 +249,15 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
         .withFunctionName("copyOfRange", CopyOfRangeEvaluator.class)
         .withFunctionName("copyOf", CopyOfEvaluator.class)
         .withFunctionName("cov", CovarianceEvaluator.class)
+        .withFunctionName("corr", CorrelationEvaluator.class)
+        .withFunctionName("kendallsCorr", KendallsCorrelationEvaluator.class)
+        .withFunctionName("spearmansCorr", SpearmansCorrelationEvaluator.class)
         .withFunctionName("describe", DescribeEvaluator.class)
         .withFunctionName("distance", EuclideanDistanceEvaluator.class)
+        .withFunctionName("manhattanDistance", ManhattanDistanceEvaluator.class)
+        .withFunctionName("earthMoversDistance", EarthMoversDistanceEvaluator.class)
+        .withFunctionName("canberraDistance", CanberraDistanceEvaluator.class)
+        .withFunctionName("chebyshevDistance", ChebyshevDistanceEvaluator.class)
         .withFunctionName("empiricalDistribution", EmpiricalDistributionEvaluator.class)
         .withFunctionName("finddelay", FindDelayEvaluator.class)
         .withFunctionName("hist", HistogramEvaluator.class)
@@ -352,8 +282,26 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
         .withFunctionName("ks", KolmogorovSmirnovEvaluator.class)
         .withFunctionName("asc", AscEvaluator.class)
         .withFunctionName("cumulativeProbability", CumulativeProbabilityEvaluator.class)
+        .withFunctionName("ebeAdd", EBEAddEvaluator.class)
+        .withFunctionName("ebeSubtract", EBESubtractEvaluator.class)
+        .withFunctionName("ebeMultiply", EBEMultiplyEvaluator.class)
+        .withFunctionName("ebeDivide", EBEDivideEvaluator.class)
+        .withFunctionName("dotProduct", DotProductEvaluator.class)
+        .withFunctionName("cosineSimilarity", CosineSimilarityEvaluator.class)
+        .withFunctionName("freqTable", FrequencyTableEvaluator.class)
+        .withFunctionName("uniformIntegerDistribution", UniformIntegerDistributionEvaluator.class)
+        .withFunctionName("binomialDistribution", BinomialDistributionEvaluator.class)
+        .withFunctionName("poissonDistribution", PoissonDistributionEvaluator.class)
+        .withFunctionName("enumeratedDistribution", EnumeratedDistributionEvaluator.class)
+        .withFunctionName("probability", ProbabilityEvaluator.class)
+        .withFunctionName("sumDifference", SumDifferenceEvaluator.class)
+        .withFunctionName("meanDifference", MeanDifferenceEvaluator.class)
+        .withFunctionName("primes", PrimesEvaluator.class)
 
         // Boolean Stream Evaluators
+
+
+
                 .withFunctionName("and", AndEvaluator.class)
         .withFunctionName("eor", ExclusiveOrEvaluator.class)
         .withFunctionName("eq", EqualToEvaluator.class)
@@ -402,7 +350,6 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
         .withFunctionName("cbrt", CubedRootEvaluator.class)
         .withFunctionName("coalesce", CoalesceEvaluator.class)
         .withFunctionName("uuid", UuidEvaluator.class)
-        .withFunctionName("corr", CorrelationEvaluator.class)
 
         // Conditional Stream Evaluators
         .withFunctionName("if", IfThenElseEvaluator.class)

solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java

@@ -306,12 +306,12 @@ public class QueryComponent extends SearchComponent
     if (!params.getBool(COMPONENT_NAME, true)) {
       return;
     }
-    SolrIndexSearcher searcher = req.getSearcher();
 
     StatsCache statsCache = req.getCore().getStatsCache();
     
     int purpose = params.getInt(ShardParams.SHARDS_PURPOSE, ShardRequest.PURPOSE_GET_TOP_IDS);
     if ((purpose & ShardRequest.PURPOSE_GET_TERM_STATS) != 0) {
+      SolrIndexSearcher searcher = req.getSearcher();
       statsCache.returnLocalStats(rb, searcher);
       return;
     }
@@ -321,50 +321,11 @@ public class QueryComponent extends SearchComponent
       statsCache.receiveGlobalStats(req);
     }
 
-    SolrQueryResponse rsp = rb.rsp;
-    IndexSchema schema = searcher.getSchema();
-
     // Optional: This could also be implemented by the top-level searcher sending
     // a filter that lists the ids... that would be transparent to
     // the request handler, but would be more expensive (and would preserve score
     // too if desired).
-    String ids = params.get(ShardParams.IDS);
+    if (doProcessSearchByIds(rb)) {
-    if (ids != null) {
-      SchemaField idField = schema.getUniqueKeyField();
-      List<String> idArr = StrUtils.splitSmart(ids, ",", true);
-      int[] luceneIds = new int[idArr.size()];
-      int docs = 0;
-      if (idField.getType().isPointField()) {
-        for (int i=0; i<idArr.size(); i++) {
-          int id = searcher.search(
-              idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
-          if (id >= 0) {
-            luceneIds[docs++] = id;
-          }
-        }
-      } else {
-        for (int i=0; i<idArr.size(); i++) {
-          int id = searcher.getFirstMatch(
-                  new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
-          if (id >= 0)
-            luceneIds[docs++] = id;
-        }
-      }
-
-      DocListAndSet res = new DocListAndSet();
-      res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
-      if (rb.isNeedDocSet()) {
-        // TODO: create a cache for this!
-        List<Query> queries = new ArrayList<>();
-        queries.add(rb.getQuery());
-        List<Query> filters = rb.getFilters();
-        if (filters != null) queries.addAll(filters);
-        res.docSet = searcher.getDocSet(queries);
-      }
-      rb.setResults(res);
-
-      ResultContext ctx = new BasicResultContext(rb);
-      rsp.addResponse(ctx);
       return;
     }
 
@@ -395,145 +356,15 @@ public class QueryComponent extends SearchComponent
     if (groupingSpec != null) {
       cmd.setSegmentTerminateEarly(false); // not supported, silently ignore any segmentTerminateEarly flag
       try {
-        boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
         if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
-          CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
+          doProcessGroupedDistributedSearchFirstPhase(rb, cmd, result);
-              .setQueryCommand(cmd)
-              .setNeedDocSet(false) // Order matters here
-              .setIncludeHitCount(true)
-              .setSearcher(searcher);
-
-          for (String field : groupingSpec.getFields()) {
-            topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
-                .setField(schema.getField(field))
-                .setGroupSort(groupingSpec.getGroupSort())
-                .setTopNGroups(cmd.getOffset() + cmd.getLen())
-                .setIncludeGroupCount(groupingSpec.isIncludeGroupCount())
-                .build()
-            );
-          }
-
-          CommandHandler commandHandler = topsGroupsActionBuilder.build();
-          commandHandler.execute();
-          SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
-          rsp.add("firstPhase", commandHandler.processResult(result, serializer));
-          rsp.add("totalHitCount", commandHandler.getTotalHitCount());
-          rb.setResult(result);
           return;
         } else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) {
-          CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
+          doProcessGroupedDistributedSearchSecondPhase(rb, cmd, result);
-              .setQueryCommand(cmd)
-              .setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
-              .setSearcher(searcher);
-
-          int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
-          docsToCollect = Math.max(docsToCollect, 1);
-
-          for (String field : groupingSpec.getFields()) {
-            SchemaField schemaField = schema.getField(field);
-            String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
-            if (topGroupsParam == null) {
-              topGroupsParam = new String[0];
-            }
-
-            List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
-            for (String topGroup : topGroupsParam) {
-              SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
-              if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
-                BytesRefBuilder builder = new BytesRefBuilder();
-                schemaField.getType().readableToIndexed(topGroup, builder);
-                searchGroup.groupValue = builder.get();
-              }
-              topGroups.add(searchGroup);
-            }
-
-            secondPhaseBuilder.addCommandField(
-                new TopGroupsFieldCommand.Builder()
-                    .setField(schemaField)
-                    .setGroupSort(groupingSpec.getGroupSort())
-                    .setSortWithinGroup(groupingSpec.getSortWithinGroup())
-                    .setFirstPhaseGroups(topGroups)
-                    .setMaxDocPerGroup(docsToCollect)
-                    .setNeedScores(needScores)
-                    .setNeedMaxScore(needScores)
-                    .build()
-            );
-          }
-
-          for (String query : groupingSpec.getQueries()) {
-            secondPhaseBuilder.addCommandField(new Builder()
-                .setDocsToCollect(docsToCollect)
-                .setSort(groupingSpec.getGroupSort())
-                .setQuery(query, rb.req)
-                .setDocSet(searcher)
-                .build()
-            );
-          }
-
-          CommandHandler commandHandler = secondPhaseBuilder.build();
-          commandHandler.execute();
-          TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
-          rsp.add("secondPhase", commandHandler.processResult(result, serializer));
-          rb.setResult(result);
           return;
         }
 
-        int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
+        doProcessGroupedSearch(rb, cmd, result);
-        boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
-        Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
-            Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
-        int limitDefault = cmd.getLen(); // this is normally from "rows"
-        Grouping grouping =
-            new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
-        grouping.setGroupSort(groupingSpec.getGroupSort())
-            .setWithinGroupSort(groupingSpec.getSortWithinGroup())
-            .setDefaultFormat(groupingSpec.getResponseFormat())
-            .setLimitDefault(limitDefault)
-            .setDefaultTotalCount(defaultTotalCount)
-            .setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
-            .setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
-            .setGetGroupedDocSet(groupingSpec.isTruncateGroups());
-
-        if (groupingSpec.getFields() != null) {
-          for (String field : groupingSpec.getFields()) {
-            grouping.addFieldCommand(field, rb.req);
-          }
-        }
-
-        if (groupingSpec.getFunctions() != null) {
-          for (String groupByStr : groupingSpec.getFunctions()) {
-            grouping.addFunctionCommand(groupByStr, rb.req);
-          }
-        }
-
-        if (groupingSpec.getQueries() != null) {
-          for (String groupByStr : groupingSpec.getQueries()) {
-            grouping.addQueryCommand(groupByStr, rb.req);
-          }
-        }
-
-        if( rb.isNeedDocList() || rb.isDebug() ){
-          // we need a single list of the returned docs
-          cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
-        }
-
-        grouping.execute();
-        if (grouping.isSignalCacheWarning()) {
-          rsp.add(
-              "cacheWarning",
-              String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
-          );
-        }
-        rb.setResult(result);
-
-        if (grouping.mainResult != null) {
-          ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
-          rsp.addResponse(ctx);
-          rsp.getToLog().add("hits", grouping.mainResult.matches());
-        } else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
-          rsp.add("grouped", result.groupedResults);
-          rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
-        }
         return;
       } catch (SyntaxError e) {
         throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
@@ -541,27 +372,7 @@ public class QueryComponent extends SearchComponent
     }
 
     // normal search result
-    searcher.search(result, cmd);
+    doProcessUngroupedSearch(rb, cmd, result);
-    rb.setResult(result);
-
-    ResultContext ctx = new BasicResultContext(rb);
-    rsp.addResponse(ctx);
-    rsp.getToLog().add("hits", rb.getResults().docList.matches());
-
-    if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
-      if (null != rb.getNextCursorMark()) {
-        rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
-                   rb.getNextCursorMark().getSerializedTotem());
-      }
-    }
-
-    if(rb.mergeFieldHandler != null) {
-      rb.mergeFieldHandler.handleMergeFields(rb, searcher);
-    } else {
-      doFieldSortValues(rb, searcher);
-    }
-
-    doPrefetch(rb);
   }
 
   protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException
@@ -1385,6 +1196,265 @@ public class QueryComponent extends SearchComponent
     return Category.QUERY;
   }
 
+  private boolean doProcessSearchByIds(ResponseBuilder rb) throws IOException {
+
+    SolrQueryRequest req = rb.req;
+    SolrQueryResponse rsp = rb.rsp;
+
+    SolrParams params = req.getParams();
+
+    String ids = params.get(ShardParams.IDS);
+    if (ids == null) {
+      return false;
+    }
+
+    SolrIndexSearcher searcher = req.getSearcher();
+    IndexSchema schema = searcher.getSchema();
+    SchemaField idField = schema.getUniqueKeyField();
+    List<String> idArr = StrUtils.splitSmart(ids, ",", true);
+    int[] luceneIds = new int[idArr.size()];
+    int docs = 0;
+    if (idField.getType().isPointField()) {
+      for (int i=0; i<idArr.size(); i++) {
+        int id = searcher.search(
+            idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
+        if (id >= 0) {
+          luceneIds[docs++] = id;
+        }
+      }
+    } else {
+      for (int i=0; i<idArr.size(); i++) {
+        int id = searcher.getFirstMatch(
+            new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
+        if (id >= 0)
+          luceneIds[docs++] = id;
+      }
+    }
+
+    DocListAndSet res = new DocListAndSet();
+    res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
+    if (rb.isNeedDocSet()) {
+      // TODO: create a cache for this!
+      List<Query> queries = new ArrayList<>();
+      queries.add(rb.getQuery());
+      List<Query> filters = rb.getFilters();
+      if (filters != null) queries.addAll(filters);
+      res.docSet = searcher.getDocSet(queries);
+    }
+    rb.setResults(res);
+
+    ResultContext ctx = new BasicResultContext(rb);
+    rsp.addResponse(ctx);
+    return true;
+  }
+
+  private void doProcessGroupedDistributedSearchFirstPhase(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException {
+
+    GroupingSpecification groupingSpec = rb.getGroupingSpec();
+    assert null != groupingSpec : "GroupingSpecification is null";
+
+    SolrQueryRequest req = rb.req;
+    SolrQueryResponse rsp = rb.rsp;
+
+    SolrIndexSearcher searcher = req.getSearcher();
+    IndexSchema schema = searcher.getSchema();
+
+    CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
+        .setQueryCommand(cmd)
+        .setNeedDocSet(false) // Order matters here
+        .setIncludeHitCount(true)
+        .setSearcher(searcher);
+
+    for (String field : groupingSpec.getFields()) {
+      topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
+          .setField(schema.getField(field))
+          .setGroupSort(groupingSpec.getGroupSort())
+          .setTopNGroups(cmd.getOffset() + cmd.getLen())
+          .setIncludeGroupCount(groupingSpec.isIncludeGroupCount())
+          .build()
+      );
+    }
+
+    CommandHandler commandHandler = topsGroupsActionBuilder.build();
+    commandHandler.execute();
+    SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
+
+    rsp.add("firstPhase", commandHandler.processResult(result, serializer));
+    rsp.add("totalHitCount", commandHandler.getTotalHitCount());
+    rb.setResult(result);
+  }
+
+  private void doProcessGroupedDistributedSearchSecondPhase(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException, SyntaxError {
+
+    GroupingSpecification groupingSpec = rb.getGroupingSpec();
+    assert null != groupingSpec : "GroupingSpecification is null";
+
+    SolrQueryRequest req = rb.req;
+    SolrQueryResponse rsp = rb.rsp;
+
+    SolrParams params = req.getParams();
+
+    SolrIndexSearcher searcher = req.getSearcher();
+    IndexSchema schema = searcher.getSchema();
+
+    boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
+
+    CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
+        .setQueryCommand(cmd)
+        .setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
+        .setSearcher(searcher);
+
+    int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
+    docsToCollect = Math.max(docsToCollect, 1);
+
+    for (String field : groupingSpec.getFields()) {
+      SchemaField schemaField = schema.getField(field);
+      String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
+      if (topGroupsParam == null) {
+        topGroupsParam = new String[0];
+      }
+
+      List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
+      for (String topGroup : topGroupsParam) {
+        SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
+        if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
+          BytesRefBuilder builder = new BytesRefBuilder();
+          schemaField.getType().readableToIndexed(topGroup, builder);
+          searchGroup.groupValue = builder.get();
+        }
+        topGroups.add(searchGroup);
+      }
+
+      secondPhaseBuilder.addCommandField(
+          new TopGroupsFieldCommand.Builder()
+              .setField(schemaField)
+              .setGroupSort(groupingSpec.getGroupSort())
+              .setSortWithinGroup(groupingSpec.getSortWithinGroup())
+              .setFirstPhaseGroups(topGroups)
+              .setMaxDocPerGroup(docsToCollect)
+              .setNeedScores(needScores)
+              .setNeedMaxScore(needScores)
+              .build()
+      );
+    }
+
+    for (String query : groupingSpec.getQueries()) {
+      secondPhaseBuilder.addCommandField(new Builder()
+          .setDocsToCollect(docsToCollect)
+          .setSort(groupingSpec.getGroupSort())
+          .setQuery(query, rb.req)
+          .setDocSet(searcher)
+          .build()
+      );
+    }
+
+    CommandHandler commandHandler = secondPhaseBuilder.build();
+    commandHandler.execute();
+    TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
+    rsp.add("secondPhase", commandHandler.processResult(result, serializer));
+    rb.setResult(result);
+  }
+
+  private void doProcessGroupedSearch(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException, SyntaxError {
+
+    GroupingSpecification groupingSpec = rb.getGroupingSpec();
+    assert null != groupingSpec : "GroupingSpecification is null";
+
+    SolrQueryRequest req = rb.req;
+    SolrQueryResponse rsp = rb.rsp;
+
+    SolrParams params = req.getParams();
+
+    SolrIndexSearcher searcher = req.getSearcher();
+
+    int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
+    boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
+    Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
+        Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
+    int limitDefault = cmd.getLen(); // this is normally from "rows"
+    Grouping grouping =
+        new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
+    grouping.setGroupSort(groupingSpec.getGroupSort())
+        .setWithinGroupSort(groupingSpec.getSortWithinGroup())
+        .setDefaultFormat(groupingSpec.getResponseFormat())
+        .setLimitDefault(limitDefault)
+        .setDefaultTotalCount(defaultTotalCount)
+        .setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
+        .setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
+        .setGetGroupedDocSet(groupingSpec.isTruncateGroups());
+
+    if (groupingSpec.getFields() != null) {
+      for (String field : groupingSpec.getFields()) {
+        grouping.addFieldCommand(field, rb.req);
+      }
+    }
+
+    if (groupingSpec.getFunctions() != null) {
+      for (String groupByStr : groupingSpec.getFunctions()) {
+        grouping.addFunctionCommand(groupByStr, rb.req);
+      }
+    }
+
+    if (groupingSpec.getQueries() != null) {
+      for (String groupByStr : groupingSpec.getQueries()) {
+        grouping.addQueryCommand(groupByStr, rb.req);
+      }
+    }
+
+    if( rb.isNeedDocList() || rb.isDebug() ){
+      // we need a single list of the returned docs
+      cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
+    }
+
+    grouping.execute();
+    if (grouping.isSignalCacheWarning()) {
+      rsp.add(
+          "cacheWarning",
+          String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
+      );
+    }
+    rb.setResult(result);
+
+    if (grouping.mainResult != null) {
+      ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
+      rsp.addResponse(ctx);
+      rsp.getToLog().add("hits", grouping.mainResult.matches());
+    } else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
+      rsp.add("grouped", result.groupedResults);
+      rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
+    }
+  }
+
+  private void doProcessUngroupedSearch(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException {
+
+    SolrQueryRequest req = rb.req;
+    SolrQueryResponse rsp = rb.rsp;
+
+    SolrIndexSearcher searcher = req.getSearcher();
+
+    searcher.search(result, cmd);
+    rb.setResult(result);
+
+    ResultContext ctx = new BasicResultContext(rb);
+    rsp.addResponse(ctx);
+    rsp.getToLog().add("hits", rb.getResults().docList.matches());
+
+    if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
+      if (null != rb.getNextCursorMark()) {
+        rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
+                   rb.getNextCursorMark().getSerializedTotem());
+      }
+    }
+
+    if(rb.mergeFieldHandler != null) {
+      rb.mergeFieldHandler.handleMergeFields(rb, searcher);
+    } else {
+      doFieldSortValues(rb, searcher);
+    }
+
+    doPrefetch(rb);
+  }
+
   /**
    * Fake scorer for a single document
    *

solr/core/src/java/org/apache/solr/parser/FastCharStream.java

@@ -69,7 +69,7 @@ public final class FastCharStream implements CharStream {
     int charsRead =          // fill space in buffer
       input.read(buffer, newPosition, buffer.length-newPosition);
     if (charsRead == -1)
-      throw new IOException("read past eof");
+      throw READ_PAST_EOF;
     else
       bufferLength += charsRead;
   }
@@ -80,6 +80,11 @@ public final class FastCharStream implements CharStream {
     return readChar();
   }
 
+  /**
+   * This Exception is used as a signal rather than an exceptional state.
+   */
+  private static final IOException READ_PAST_EOF = new IOException("read past eof");
+
   @Override
   public final void backup(int amount) {
     bufferPosition -= amount;

solr/core/src/java/org/apache/solr/parser/QueryParser.java

@@ -767,7 +767,7 @@ public class QueryParser extends SolrQueryParserBase implements QueryParserConst
   }
 
   static private final class LookaheadSuccess extends java.lang.Error { }
-  final private LookaheadSuccess jj_ls = new LookaheadSuccess();
+  static final private LookaheadSuccess jj_ls = new LookaheadSuccess();
   private boolean jj_scan_token(int kind) {
     if (jj_scanpos == jj_lastpos) {
       jj_la--;

solr/core/src/java/org/apache/solr/schema/AbstractEnumField.java

@@ -250,11 +250,12 @@ public abstract class AbstractEnumField extends PrimitiveFieldType {
 
   @Override
   public SortField getSortField(SchemaField field, boolean top) {
-    field.checkSortability();
+    SortField result = getSortField(field, SortField.Type.INT, top, Integer.MIN_VALUE, Integer.MAX_VALUE);
-    final Object missingValue = Integer.MIN_VALUE;
+    if (null == result.getMissingValue()) {
-    SortField sf = new SortField(field.getName(), SortField.Type.INT, top);
+      // special case default behavior: assume missing values are "below" all enum values
-    sf.setMissingValue(missingValue);
+      result.setMissingValue(Integer.MIN_VALUE);
-    return sf;
+    }
+    return result;
   }
 
   @Override

solr/core/src/java/org/apache/solr/schema/DatePointField.java

@@ -190,20 +190,7 @@ public class DatePointField extends PointField implements DateValueFieldType {
 
   @Override
   public SortField getSortField(SchemaField field, boolean top) {
-    field.checkSortability();
+    return getSortField(field, SortField.Type.LONG, top, Long.MIN_VALUE, Long.MAX_VALUE);
-
-    Object missingValue = null;
-    boolean sortMissingLast = field.sortMissingLast();
-    boolean sortMissingFirst = field.sortMissingFirst();
-
-    if (sortMissingLast) {
-      missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
-    } else if (sortMissingFirst) {
-      missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
-    }
-    SortField sf = new SortField(field.getName(), SortField.Type.LONG, top);
-    sf.setMissingValue(missingValue);
-    return sf;
   }
 
   @Override

solr/core/src/java/org/apache/solr/schema/DateRangeField.java

@@ -39,7 +39,7 @@ import org.apache.solr.util.DateMathParser;
 import org.locationtech.spatial4j.shape.Shape;
 
 /**
- * A field for indexed dates and date ranges. It's mostly compatible with TrieDateField.  It has the potential to allow
+ * A field for indexed dates and date ranges. It's mostly compatible with DatePointField.  It has the potential to allow
  * efficient faceting, similar to facet.enum.
  *
  * @see NumberRangePrefixTreeStrategy
@@ -75,7 +75,7 @@ public class DateRangeField extends AbstractSpatialPrefixTreeFieldType<NumberRan
     if (shape instanceof UnitNRShape) {
       UnitNRShape unitShape = (UnitNRShape) shape;
       if (unitShape.getLevel() == tree.getMaxLevels()) {
-        //fully precise date. We can be fully compatible with TrieDateField (incl. 'Z')
+        //fully precise date. We can be fully compatible with DatePointField (incl. 'Z')
         return shape.toString() + 'Z';
       }
     }

solr/core/src/java/org/apache/solr/schema/DoublePointField.java

@@ -134,20 +134,7 @@ public class DoublePointField extends PointField implements DoubleValueFieldType
 
   @Override
   public SortField getSortField(SchemaField field, boolean top) {
-    field.checkSortability();
+    return getSortField(field, SortField.Type.DOUBLE, top, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY);
-
-    Object missingValue = null;
-    boolean sortMissingLast = field.sortMissingLast();
-    boolean sortMissingFirst = field.sortMissingFirst();
-
-    if (sortMissingLast) {
-      missingValue = top ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
-    } else if (sortMissingFirst) {
-      missingValue = top ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
-    }
-    SortField sf = new SortField(field.getName(), SortField.Type.DOUBLE, top);
-    sf.setMissingValue(missingValue);
-    return sf;
   }
 
   @Override

solr/core/src/java/org/apache/solr/schema/FieldType.java

@@ -48,6 +48,7 @@ import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedSetSortField;
 import org.apache.lucene.search.SortedNumericSelector;
 import org.apache.lucene.search.SortedSetSelector;
 import org.apache.lucene.search.TermInSetQuery;
@@ -69,7 +70,6 @@ import org.apache.solr.query.SolrRangeQuery;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
 import org.apache.solr.search.QueryUtils;
-import org.apache.solr.search.Sorting;
 import org.apache.solr.uninverting.UninvertingReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -662,17 +662,76 @@ public abstract class FieldType extends FieldProperties {
    * Returns the SortField instance that should be used to sort fields
    * of this type.
    * @see SchemaField#checkSortability
+   * @see #getSortField(SchemaField,SortField.Type,boolean,Object,Object)
    */
   public abstract SortField getSortField(SchemaField field, boolean top);
 
+  /**
+   * <p>A Helper utility method for use by subclasses.</p>
+   * <p>This method deals with:</p>
+   * <ul>
+   *  <li>{@link SchemaField#checkSortability}</li>
+   *  <li>Creating a {@link SortField} on <code>field</code> with the specified 
+   *      <code>reverse</code> &amp; <code>sortType</code></li>
+   *  <li>Setting the {@link SortField#setMissingValue} to <code>missingLow</code> or <code>missingHigh</code>
+   *      as appropriate based on the value of <code>reverse</code> and the 
+   *      <code>sortMissingFirst</code> &amp; <code>sortMissingLast</code> properties of the 
+   *      <code>field</code></li>
+   * </ul>
+   *
+   * @param field The SchemaField to sort on.  May use <code>sortMissingFirst</code> or <code>sortMissingLast</code> or neither.
+   * @param sortType The sort Type of the underlying values in the <code>field</code>
+   * @param reverse True if natural order of the <code>sortType</code> should be reversed
+   * @param missingLow The <code>missingValue</code> to be used if the other params indicate that docs w/o values should sort as "low" as possible.
+   * @param missingHigh The <code>missingValue</code> to be used if the other params indicate that docs w/o values should sort as "high" as possible.
+   * @see #getSortedSetSortField
+   */
+  protected static SortField getSortField(SchemaField field, SortField.Type sortType, boolean reverse,
+                                          Object missingLow, Object missingHigh) {
+    field.checkSortability();
+
+    SortField sf = new SortField(field.getName(), sortType, reverse);
+    applySetMissingValue(field, sf, missingLow, missingHigh);
+    
+    return sf;
+  }
+
+  /**
+   * Same as {@link #getSortField} but using {@link SortedSetSortField}
+   */
+  protected static SortField getSortedSetSortField(SchemaField field, SortedSetSelector.Type selector,
+                                                   boolean reverse, Object missingLow, Object missingHigh) {
+                                                   
+    field.checkSortability();
+
+    SortField sf = new SortedSetSortField(field.getName(), reverse, selector);
+    applySetMissingValue(field, sf, missingLow, missingHigh);
+    
+    return sf;
+  }
+  
+  /** 
+   * @see #getSortField 
+   * @see #getSortedSetSortField 
+   */
+  private static void applySetMissingValue(SchemaField field, SortField sortField, 
+                                           Object missingLow, Object missingHigh) {
+    final boolean reverse = sortField.getReverse();
+    
+    if (field.sortMissingLast()) {
+      sortField.setMissingValue(reverse ? missingLow : missingHigh);
+    } else if (field.sortMissingFirst()) {
+      sortField.setMissingValue(reverse ? missingHigh : missingLow);
+    }
+  }
+
   /**
    * Utility usable by subclasses when they want to get basic String sorting
    * using common checks.
    * @see SchemaField#checkSortability
    */
   protected SortField getStringSort(SchemaField field, boolean reverse) {
-    field.checkSortability();
+    return getSortField(field, SortField.Type.STRING, reverse, SortField.STRING_FIRST, SortField.STRING_LAST);
-    return Sorting.getStringSortField(field.name, reverse, field.sortMissingLast(),field.sortMissingFirst());
   }
 
   /** called to get the default value source (normally, from the

solr/core/src/java/org/apache/solr/schema/FloatPointField.java

@@ -134,20 +134,7 @@ public class FloatPointField extends PointField implements FloatValueFieldType {
 
   @Override
   public SortField getSortField(SchemaField field, boolean top) {
-    field.checkSortability();
+    return getSortField(field, SortField.Type.FLOAT, top, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY);
-
-    Object missingValue = null;
-    boolean sortMissingLast = field.sortMissingLast();
-    boolean sortMissingFirst = field.sortMissingFirst();
-
-    if (sortMissingLast) {
-      missingValue = top ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY;
-    } else if (sortMissingFirst) {
-      missingValue = top ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY;
-    }
-    SortField sf = new SortField(field.getName(), SortField.Type.FLOAT, top);
-    sf.setMissingValue(missingValue);
-    return sf;
   }
 
   @Override

solr/core/src/java/org/apache/solr/schema/IntPointField.java

@@ -132,20 +132,7 @@ public class IntPointField extends PointField implements IntValueFieldType {
 
   @Override
   public SortField getSortField(SchemaField field, boolean top) {
-    field.checkSortability();
+    return getSortField(field, SortField.Type.INT, top, Integer.MIN_VALUE, Integer.MAX_VALUE);
-
-    Object missingValue = null;
-    boolean sortMissingLast = field.sortMissingLast();
-    boolean sortMissingFirst = field.sortMissingFirst();
-
-    if (sortMissingLast) {
-      missingValue = top ? Integer.MIN_VALUE : Integer.MAX_VALUE;
-    } else if (sortMissingFirst) {
-      missingValue = top ? Integer.MAX_VALUE : Integer.MIN_VALUE;
-    }
-    SortField sf = new SortField(field.getName(), SortField.Type.INT, top);
-    sf.setMissingValue(missingValue);
-    return sf;
   }
 
   @Override

solr/core/src/java/org/apache/solr/schema/LongPointField.java

@@ -131,20 +131,7 @@ public class LongPointField extends PointField implements LongValueFieldType {
 
   @Override
   public SortField getSortField(SchemaField field, boolean top) {
-    field.checkSortability();
+    return getSortField(field, SortField.Type.LONG, top, Long.MIN_VALUE, Long.MAX_VALUE);
-
-    Object missingValue = null;
-    boolean sortMissingLast = field.sortMissingLast();
-    boolean sortMissingFirst = field.sortMissingFirst();
-
-    if (sortMissingLast) {
-      missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
-    } else if (sortMissingFirst) {
-      missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
-    }
-    SortField sf = new SortField(field.getName(), SortField.Type.LONG, top);
-    sf.setMissingValue(missingValue);
-    return sf;
   }
 
   @Override

solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java

@@ -34,13 +34,13 @@ import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedSetSelector;
 import org.apache.lucene.util.AttributeFactory;
 import org.apache.lucene.util.AttributeSource.State;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.solr.analysis.SolrAnalyzer;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
-import org.apache.solr.search.Sorting;
 import org.apache.solr.uninverting.UninvertingReader.Type;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -132,8 +132,8 @@ public class PreAnalyzedField extends TextField implements HasImplicitIndexAnaly
   
   @Override
   public SortField getSortField(SchemaField field, boolean top) {
-    field.checkSortability();
+    return getSortedSetSortField(field, SortedSetSelector.Type.MIN, top,
-    return Sorting.getTextSortField(field.getName(), top, field.sortMissingLast(), field.sortMissingFirst());
+                                 SortField.STRING_FIRST, SortField.STRING_LAST);
   }
   
   @Override

solr/core/src/java/org/apache/solr/schema/TextField.java

@@ -32,7 +32,6 @@ import org.apache.solr.common.SolrException;
 import org.apache.solr.query.SolrRangeQuery;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
-import org.apache.solr.search.Sorting;
 import org.apache.solr.uninverting.UninvertingReader.Type;
 
 /** <code>TextField</code> is the basic type for configurable text analysis.
@@ -108,8 +107,8 @@ public class TextField extends FieldType {
   @Override
   public SortField getSortField(SchemaField field, boolean reverse) {
     /* :TODO: maybe warn if isTokenized(), but doesn't use LimitTokenCountFilter in its chain? */
-    field.checkSortability();
+    return getSortedSetSortField(field, SortedSetSelector.Type.MIN, reverse,
-    return Sorting.getTextSortField(field.getName(), reverse, field.sortMissingLast(), field.sortMissingFirst());
+                                 SortField.STRING_FIRST, SortField.STRING_LAST);
   }
   
   @Override

solr/core/src/java/org/apache/solr/schema/TrieField.java

@@ -171,50 +171,14 @@ public class TrieField extends NumericFieldType {
 
     switch (type) {
       case INTEGER:
-        if( sortMissingLast ) {
+        return getSortField(field, SortField.Type.INT, top, Integer.MIN_VALUE, Integer.MAX_VALUE);
-          missingValue = top ? Integer.MIN_VALUE : Integer.MAX_VALUE;
-        }
-        else if( sortMissingFirst ) {
-          missingValue = top ? Integer.MAX_VALUE : Integer.MIN_VALUE;
-        }
-        sf = new SortField( field.getName(), SortField.Type.INT, top);
-        sf.setMissingValue(missingValue);
-        return sf;
-      
       case FLOAT:
-        if( sortMissingLast ) {
+        return getSortField(field, SortField.Type.FLOAT, top, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY);
-          missingValue = top ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY;
-        }
-        else if( sortMissingFirst ) {
-          missingValue = top ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY;
-        }
-        sf = new SortField( field.getName(), SortField.Type.FLOAT, top);
-        sf.setMissingValue(missingValue);
-        return sf;
-      
       case DATE: // fallthrough
       case LONG:
-        if( sortMissingLast ) {
+        return getSortField(field, SortField.Type.LONG, top, Long.MIN_VALUE, Long.MAX_VALUE);
-          missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
-        }
-        else if( sortMissingFirst ) {
-          missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
-        }
-        sf = new SortField( field.getName(), SortField.Type.LONG, top);
-        sf.setMissingValue(missingValue);
-        return sf;
-        
       case DOUBLE:
-        if( sortMissingLast ) {
+        return getSortField(field, SortField.Type.DOUBLE, top, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY);
-          missingValue = top ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
-        }
-        else if( sortMissingFirst ) {
-          missingValue = top ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
-        }
-        sf = new SortField( field.getName(), SortField.Type.DOUBLE, top);
-        sf.setMissingValue(missingValue);
-        return sf;
-        
       default:
         throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + field.name);
     }

solr/core/src/java/org/apache/solr/search/FunctionQParser.java

@@ -25,6 +25,7 @@ import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.facet.AggValueSource;
+import org.apache.solr.search.function.FieldNameValueSource;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -33,6 +34,7 @@ public class FunctionQParser extends QParser {
 
   public static final int FLAG_CONSUME_DELIMITER = 0x01;  // consume delimiter after parsing arg
   public static final int FLAG_IS_AGG = 0x02;
+  public static final int FLAG_USE_FIELDNAME_SOURCE = 0x04; // When a field name is encountered, use the placeholder FieldNameValueSource instead of resolving to a real ValueSource
   public static final int FLAG_DEFAULT = FLAG_CONSUME_DELIMITER;
 
   /** @lucene.internal */
@@ -373,11 +375,16 @@ public class FunctionQParser extends QParser {
           valueSource = new BoolConstValueSource(true);
         } else if ("false".equals(id)) {
           valueSource = new BoolConstValueSource(false);
+        } else {
+          if ((flags & FLAG_USE_FIELDNAME_SOURCE) != 0) {
+            // Don't try to create a ValueSource for the field, just use a placeholder.
+            valueSource = new FieldNameValueSource(id);
           } else {
             SchemaField f = req.getSchema().getField(id);
             valueSource = f.getType().getValueSource(f, this);
           }
         }
+      }
 
     }
     

solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java

@@ -16,13 +16,9 @@
  */
 package org.apache.solr.search;
 
-import org.apache.lucene.search.Query;
-import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.request.SolrQueryRequest;
 
-import java.util.List;
 /**
  * Parse Solr's variant on the Lucene QueryParser syntax.
  * <br>Other parameters:<ul>
@@ -42,53 +38,3 @@ public class LuceneQParserPlugin extends QParserPlugin {
   }
 }
 
-@Deprecated 
-class OldLuceneQParser extends LuceneQParser {
-  String sortStr;
-
-  public OldLuceneQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
-    super(qstr, localParams, params, req);
-  }
-
-  @Override
-  public Query parse() throws SyntaxError {
-    // handle legacy "query;sort" syntax
-    if (getLocalParams() == null) {
-      String qstr = getString();
-      if (qstr == null || qstr.length() == 0)
-        return null;
-      sortStr = getParams().get(CommonParams.SORT);
-      if (sortStr == null) {
-        // sort may be legacy form, included in the query string
-        List<String> commands = StrUtils.splitSmart(qstr,';');
-        if (commands.size() == 2) {
-          qstr = commands.get(0);
-          sortStr = commands.get(1);
-        } else if (commands.size() == 1) {
-          // This is need to support the case where someone sends: "q=query;"
-          qstr = commands.get(0);
-        }
-        else if (commands.size() > 2) {
-          throw new SyntaxError("If you want to use multiple ';' in the query, use the 'sort' param.");
-        }
-      }
-      setString(qstr);
-    }
-
-    return super.parse();
-  }
-
-  @Override
-  public SortSpec getSortSpec(boolean useGlobal) throws SyntaxError {
-    SortSpec sort = super.getSortSpec(useGlobal);
-    if (sortStr != null && sortStr.length()>0 && sort.getSort()==null) {
-      SortSpec oldSort = SortSpecParsing.parseSortSpec(sortStr, getReq());
-      if( oldSort.getSort() != null ) {
-        sort.setSortAndFields(oldSort.getSort(), oldSort.getSchemaFields());
-      }
-    }
-    return sort;
-  }
-
-}
-

solr/core/src/java/org/apache/solr/search/Sorting.java

@@ -16,15 +16,16 @@
  */
 package org.apache.solr.search;
 
+import org.apache.solr.schema.FieldType;
 import org.apache.lucene.search.*;
 
 /**
  * Extra lucene sorting utilities & convenience methods
  *
  *
- *
+ * @deprecated custom {@link FieldType}s should use the helper methods in the base class.  Other usage should leverage th underling lucene {@link SortField} classes directly.
  */
-
+@Deprecated
 public class Sorting {
 
 
@@ -37,14 +38,19 @@ public class Sorting {
    * @param nullLast    true if null should come last, regardless of sort order
    * @param nullFirst   true if null should come first, regardless of sort order
    * @return SortField
+   * @deprecated custom {@link FieldType}s should use {@link FieldType#getSortField}.  Other usage should leverage th underling lucene {@link SortField} classes directly.
    */
+  @Deprecated
   public static SortField getStringSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
     SortField sortField = new SortField(fieldName, SortField.Type.STRING, reverse);
     applyMissingFirstLast(sortField, reverse, nullLast, nullFirst);
     return sortField;
   }
 
-  /** Like {@link #getStringSortField}) except safe for tokenized fields */
+  /** Like {@link #getStringSortField}) except safe for tokenized fields
+   * @deprecated custom {@link FieldType}s should use {@link FieldType#getSortedSetSortField}.  Other usage should leverage th underling lucene {@link SortedSetSortField} classes directly.
+   */
+  @Deprecated
   public static SortField getTextSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
     SortField sortField = new SortedSetSortField(fieldName, reverse);
     applyMissingFirstLast(sortField, reverse, nullLast, nullFirst);

solr/core/src/java/org/apache/solr/search/ValueSourceParser.java

@@ -1017,14 +1017,14 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
     addParser("agg_min", new ValueSourceParser() {
       @Override
       public ValueSource parse(FunctionQParser fp) throws SyntaxError {
-        return new MinMaxAgg("min", fp.parseValueSource());
+        return new MinMaxAgg("min", fp.parseValueSource(FunctionQParser.FLAG_DEFAULT | FunctionQParser.FLAG_USE_FIELDNAME_SOURCE));
       }
     });
 
     addParser("agg_max", new ValueSourceParser() {
       @Override
       public ValueSource parse(FunctionQParser fp) throws SyntaxError {
-        return new MinMaxAgg("max", fp.parseValueSource());
+        return new MinMaxAgg("max", fp.parseValueSource(FunctionQParser.FLAG_DEFAULT | FunctionQParser.FLAG_USE_FIELDNAME_SOURCE));
       }
     });
 

solr/core/src/java/org/apache/solr/search/facet/MinMaxAgg.java

@@ -18,6 +18,7 @@ package org.apache.solr.search.facet;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Date;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.MultiDocValues;
@@ -25,9 +26,12 @@ import org.apache.lucene.index.OrdinalMap;
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.LongValues;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.schema.StrFieldSource;
+import org.apache.solr.search.function.FieldNameValueSource;
 
 public class MinMaxAgg extends SimpleAggValueSource {
   final int minmax; // a multiplier to reverse the normal order of compare if this is max instead of min (i.e. max will be -1)
@@ -41,28 +45,46 @@ public class MinMaxAgg extends SimpleAggValueSource {
   public SlotAcc createSlotAcc(FacetContext fcontext, int numDocs, int numSlots) throws IOException {
     ValueSource vs = getArg();
 
-    if (vs instanceof StrFieldSource) {
+    SchemaField sf = null;
-      String field = ((StrFieldSource) vs).getField();
+
-      SchemaField sf = fcontext.qcontext.searcher().getSchema().getField(field);
+    if (vs instanceof FieldNameValueSource) {
+      String field = ((FieldNameValueSource)vs).getFieldName();
+      sf = fcontext.qcontext.searcher().getSchema().getField(field);
+
       if (sf.multiValued() || sf.getType().multiValuedFieldCache()) {
-        if (sf.hasDocValues()) {
+        vs = null;
-          // dv
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "min/max aggregations can't be used on multi-valued field " + field);
       } else {
-          // uif
+        vs = sf.getType().getValueSource(sf, null);
       }
-      } else {
+    }
+
+    if (vs instanceof StrFieldSource) {
       return new SingleValuedOrdAcc(fcontext, sf, numSlots);
     }
+
+    // Since functions don't currently have types, we rely on the type of the field
+    if (sf != null && sf.getType().getNumberType() != null) {
+      switch (sf.getType().getNumberType()) {
+        case FLOAT:
+        case DOUBLE:
+          return new DFuncAcc(vs, fcontext, numSlots);
+        case INTEGER:
+        case LONG:
+          return new LFuncAcc(vs, fcontext, numSlots);
+        case DATE:
+          return new DateFuncAcc(vs, fcontext, numSlots);
+      }
     }
 
     // numeric functions
-    return new ValSlotAcc(vs, fcontext, numSlots);
+    return new DFuncAcc(vs, fcontext, numSlots);
   }
 
   @Override
   public FacetMerger createFacetMerger(Object prototype) {
-    if (prototype instanceof Number)
+    if (prototype instanceof Double)
-      return new NumericMerger();
+      return new NumericMerger(); // still use NumericMerger to handle NaN?
     else if (prototype instanceof Comparable) {
       return new ComparableMerger();
     } else {
@@ -114,8 +136,8 @@ public class MinMaxAgg extends SimpleAggValueSource {
     }
   }
 
-  class ValSlotAcc extends DoubleFuncSlotAcc {
+  class DFuncAcc extends DoubleFuncSlotAcc {
-    public ValSlotAcc(ValueSource values, FacetContext fcontext, int numSlots) {
+    public DFuncAcc(ValueSource values, FacetContext fcontext, int numSlots) {
       super(values, fcontext, numSlots, Double.NaN);
     }
 
@@ -129,6 +151,101 @@ public class MinMaxAgg extends SimpleAggValueSource {
         result[slotNum] = val;
       }
     }
+
+    @Override
+    public Object getValue(int slot) {
+      double val = result[slot];
+      if (Double.isNaN(val)) {
+        return null;
+      } else {
+        return val;
+      }
+    }
+  }
+
+  class LFuncAcc extends LongFuncSlotAcc {
+    FixedBitSet exists;
+    public LFuncAcc(ValueSource values, FacetContext fcontext, int numSlots) {
+      super(values, fcontext, numSlots, 0);
+      exists = new FixedBitSet(numSlots);
+    }
+
+    @Override
+    public void collect(int doc, int slotNum) throws IOException {
+      long val = values.longVal(doc);
+      if (val == 0 && !values.exists(doc)) return; // depend on fact that non existing values return 0 for func query
+
+      long currVal = result[slotNum];
+      if (currVal == 0 && !exists.get(slotNum)) {
+        exists.set(slotNum);
+        result[slotNum] = val;
+      } else if (Long.compare(val, currVal) * minmax < 0) {
+        result[slotNum] =  val;
+      }
+    }
+
+    @Override
+    public Object getValue(int slot) {
+      long val = result[slot];
+      if (val == 0 && !exists.get(slot)) {
+        return null;
+      } else {
+        return val;
+      }
+    }
+
+    @Override
+    public void resize(Resizer resizer) {
+      super.resize(resizer);
+      exists = resizer.resize(exists);
+    }
+
+    @Override
+    public int compare(int slotA, int slotB) {
+      long a = result[slotA];
+      long b = result[slotB];
+      boolean ea = a != 0 || exists.get(slotA);
+      boolean eb = b != 0 || exists.get(slotB);
+
+      if (ea != eb) {
+        if (ea) return 1;  // a exists and b doesn't TODO: we need context to be able to sort missing last!  SOLR-10618
+        if (eb) return -1; // b exists and a is missing
+      }
+
+      return Long.compare(a, b);
+    }
+
+    @Override
+    public void reset() {
+      super.reset();
+      exists.clear(0, exists.length());
+    }
+
+  }
+
+  class DateFuncAcc extends LongFuncSlotAcc {
+    private static final long MISSING = Long.MIN_VALUE;
+    public DateFuncAcc(ValueSource values, FacetContext fcontext, int numSlots) {
+      super(values, fcontext, numSlots, MISSING);
+    }
+
+    @Override
+    public void collect(int doc, int slotNum) throws IOException {
+      long val = values.longVal(doc);
+      if (val == 0 && !values.exists(doc)) return; // depend on fact that non existing values return 0 for func query
+
+      long currVal = result[slotNum];
+      if (Long.compare(val, currVal) * minmax < 0 || currVal == MISSING) {
+        result[slotNum] =  val;
+      }
+    }
+
+    // let compare be the default for now (since we can't yet correctly handle sortMissingLast
+
+    @Override
+    public Object getValue(int slot) {
+      return result[slot] == MISSING ? null : new Date(result[slot]);
+    }
   }
 
 

solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java

@@ -16,14 +16,6 @@
  */
 package org.apache.solr.search.facet;
 
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.search.DocIterator;
-import org.apache.solr.search.DocSet;
-import org.apache.solr.search.SolrIndexSearcher;
-
 import java.io.Closeable;
 import java.io.IOException;
 import java.lang.reflect.Array;
@@ -32,6 +24,16 @@ import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.DocSet;
+import org.apache.solr.search.SolrIndexSearcher;
+
 /**
  * Accumulates statistics separated by a slot number. 
  * There is a separate statistic per slot. The slot is usually an ordinal into a set of values, e.g. tracking a count
@@ -140,6 +142,38 @@ public abstract class SlotAcc implements Closeable {
       return values;
     }
 
+    public long[] resize(long[] old, long defaultValue) {
+      long[] values = new long[getNewSize()];
+      if (defaultValue != 0) {
+        Arrays.fill(values, 0, values.length, defaultValue);
+      }
+      for (int i = 0; i < old.length; i++) {
+        long val = old[i];
+        if (val != defaultValue) {
+          int newSlot = getNewSlot(i);
+          if (newSlot >= 0) {
+            values[newSlot] = val;
+          }
+        }
+      }
+      return values;
+    }
+
+    public FixedBitSet resize(FixedBitSet old) {
+      FixedBitSet values = new FixedBitSet(getNewSize());
+      int oldSize = old.length();
+
+      for(int oldSlot = 0;;) {
+        oldSlot = values.nextSetBit(oldSlot);
+        if (oldSlot == DocIdSetIterator.NO_MORE_DOCS) break;
+        int newSlot = getNewSlot(oldSlot);
+        values.set(newSlot);
+        if (++oldSlot >= oldSize) break;
+      }
+
+      return values;
+    }
+
     public <T> T[] resize(T[] old, T defaultValue) {
       T[] values = (T[]) Array.newInstance(old.getClass().getComponentType(), getNewSize());
       if (defaultValue != null) {
@@ -222,6 +256,40 @@ abstract class DoubleFuncSlotAcc extends FuncSlotAcc {
   }
 }
 
+abstract class LongFuncSlotAcc extends FuncSlotAcc {
+  long[] result;
+  long initialValue;
+
+  public LongFuncSlotAcc(ValueSource values, FacetContext fcontext, int numSlots, long initialValue) {
+    super(values, fcontext, numSlots);
+    this.initialValue = initialValue;
+    result = new long[numSlots];
+    if (initialValue != 0) {
+      reset();
+    }
+  }
+
+  @Override
+  public int compare(int slotA, int slotB) {
+    return Long.compare(result[slotA], result[slotB]);
+  }
+
+  @Override
+  public Object getValue(int slot) {
+    return result[slot];
+  }
+
+  @Override
+  public void reset() {
+    Arrays.fill(result, initialValue);
+  }
+
+  @Override
+  public void resize(Resizer resizer) {
+    result = resizer.resize(result, initialValue);
+  }
+}
+
 abstract class IntSlotAcc extends SlotAcc {
   int[] result; // use LongArray32
   int initialValue;

solr/core/src/java/org/apache/solr/search/function/FieldNameValueSource.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.function;
+
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+
+/** Placeholder value source.
+ * @lucene.internal */
+public class FieldNameValueSource extends ValueSource {
+  private String fieldName;
+
+  public FieldNameValueSource(String fieldName) {
+    this.fieldName = fieldName;
+  }
+
+  public String getFieldName() {
+    return fieldName;
+  }
+
+  @Override
+  public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+    throw new UnsupportedOperationException("FieldNameValueSource should not be directly used: " + this);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    return o instanceof FieldNameValueSource && fieldName.equals(((FieldNameValueSource)o).getFieldName());
+  }
+
+  @Override
+  public int hashCode() {
+    return fieldName.hashCode();
+  }
+
+  @Override
+  public String description() {
+    return "FIELDNAME(" + fieldName + ")";
+  }
+}

solr/core/src/java/org/apache/solr/update/processor/FieldLengthUpdateProcessorFactory.java

@@ -35,7 +35,7 @@ import static org.apache.solr.update.processor.FieldValueMutatingUpdateProcessor
  * <p>For example, with the configuration listed below any documents 
  * containing  String values (such as "<code>abcdef</code>" or 
  * "<code>xyz</code>") in a field declared in the schema using 
- * <code>TrieIntField</code> or <code>TrieLongField</code> 
+ * <code>IntPointField</code> or <code>LongPointField</code> 
  * would have those Strings replaced with the length of those fields as an 
  * Integer 
  * (ie: <code>6</code> and <code>3</code> respectively)
@@ -43,8 +43,8 @@ import static org.apache.solr.update.processor.FieldValueMutatingUpdateProcessor
  * <pre class="prettyprint">
  * &lt;processor class="solr.FieldLengthUpdateProcessorFactory"&gt;
  *   &lt;arr name="typeClass"&gt;
- *     &lt;str&gt;solr.TrieIntField&lt;/str&gt;
+ *     &lt;str&gt;solr.IntPointField&lt;/str&gt;
- *     &lt;str&gt;solr.TrieLongField&lt;/str&gt;
+ *     &lt;str&gt;solr.LongPointField&lt;/str&gt;
  *   &lt;/arr&gt;
  * &lt;/processor&gt;</pre>
  */

solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java

@@ -79,7 +79,7 @@ import static org.apache.solr.update.processor.FieldMutatingUpdateProcessor.SELE
  * In the ExampleFieldMutatingUpdateProcessorFactory configured below, 
  * fields will be mutated if the name starts with "foo" <i>or</i> "bar"; 
  * <b>unless</b> the field name contains the substring "SKIP" <i>or</i> 
- * the fieldType is (or subclasses) TrieDateField.  Meaning a field named
+ * the fieldType is (or subclasses) DatePointField.  Meaning a field named
  * "foo_SKIP" is guaranteed not to be selected, but a field named "bar_smith" 
  * that uses StrField will be selected.
  * </p>
@@ -92,7 +92,7 @@ import static org.apache.solr.update.processor.FieldMutatingUpdateProcessor.SELE
  *     &lt;str name="fieldRegex"&gt;.*SKIP.*&lt;/str&gt;
  *   &lt;/lst&gt;
  *   &lt;lst name="exclude"&gt;
- *     &lt;str name="typeClass"&gt;solr.TrieDateField&lt;/str&gt;
+ *     &lt;str name="typeClass"&gt;solr.DatePointField&lt;/str&gt;
  *   &lt;/lst&gt;
  * &lt;/processor&gt;</pre>
  * 

solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java

@@ -47,8 +47,8 @@ import org.slf4j.LoggerFactory;
  * </p>
  * <p>
  * The default selection behavior is to mutate both those fields that don't match
- * a schema field, as well as those fields that match a schema field with a field 
+ * a schema field, as well as those fields that match a schema field with a date
- * type that uses class solr.TrieDateField.
+ * field type.
  * </p>
  * <p>
  * If all values are parseable as dates (or are already Date), then the field will

solr/core/src/java/org/apache/solr/update/processor/ParseDoubleFieldUpdateProcessorFactory.java

@@ -38,8 +38,8 @@ import java.util.Locale;
  * </p>
  * <p>
  * The default selection behavior is to mutate both those fields that don't match
- * a schema field, as well as those fields that match a schema field with a field 
+ * a schema field, as well as those fields that match a schema field with a double
- * type that uses class solr.TrieDoubleField.
+ * field type.
  * </p>
  * <p>
  * If all values are parseable as double (or are already Double), then the field

solr/core/src/java/org/apache/solr/update/processor/ParseFloatFieldUpdateProcessorFactory.java

@@ -38,8 +38,8 @@ import java.util.Locale;
  * </p>
  * <p>
  * The default selection behavior is to mutate both those fields that don't match
- * a schema field, as well as those fields that match a schema field with a field 
+ * a schema field, as well as those fields that match a schema field with a float
- * type that uses class solr.TrieFloatField.
+ * field type.
  * </p>
  * <p>
  * If all values are parseable as float (or are already Float), then the field

solr/core/src/java/org/apache/solr/update/processor/ParseIntFieldUpdateProcessorFactory.java

@@ -35,8 +35,8 @@ import java.util.Locale;
  * </p>
  * <p>
  * The default selection behavior is to mutate both those fields that don't match
- * a schema field, as well as those fields that match a schema field with a field 
+ * a schema field, as well as those fields that match a schema field with an int
- * type that uses class solr.TrieIntField.
+ * field type.
  * </p>
  * <p>
  * If all values are parseable as int (or are already Integer), then the field

solr/core/src/java/org/apache/solr/update/processor/ParseLongFieldUpdateProcessorFactory.java

@@ -35,8 +35,8 @@ import java.util.Locale;
  * </p>
  * <p>
  * The default selection behavior is to mutate both those fields that don't match
- * a schema field, as well as those fields that match a schema field with a field 
+ * a schema field, as well as those fields that match a schema field with a long 
- * type that uses class solr.TrieLongField.
+ * field type.
  * </p>
  * <p>
  * If all values are parseable as long (or are already Long), then the field

solr/core/src/java/org/apache/solr/util/configuration/providers/package-info.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+/** 
+ * TODO
+ */
+package org.apache.solr.util.configuration.providers;
+
+

solr/core/src/test-files/solr/collection1/conf/schema-enums.xml

@@ -18,12 +18,22 @@
 <schema name="tiny" version="1.1">
   <field name="id" type="string" indexed="true" stored="true" required="true"/>
   <field name="_version_" type="long" indexed="true" stored="true" multiValued="false"/>
+  
   <!-- Test EnumField and EnumFieldType -->
   <field name="severity" type="severityType" indexed="${solr.tests.EnumFieldTest.indexed}" stored="true" multiValued="false" docValues="${solr.tests.numeric.dv}"/>
+  <!-- NOTE: because these test sortMissingLast/sortMissingFirst, we force indexed="true" so we don't get
+       random errors on schema init about inconsistent properties -->
+  <field name="severity_missingLast" type="severityType" indexed="true" stored="true" multiValued="false" docValues="${solr.tests.numeric.dv}" sortMissingLast="true"/>
+  <field name="severity_missingFirst" type="severityType" indexed="true" stored="true" multiValued="false" docValues="${solr.tests.numeric.dv}" sortMissingFirst="true"/>
+  
   <field name="severity_mv" type="severityType" indexed="${solr.tests.EnumFieldTest.indexed}" stored="true" multiValued="true" docValues="${solr.tests.numeric.dv}"/>
   <field name="text" type="text" indexed="true" stored="true" multiValued="true"/>
+  
   <uniqueKey>id</uniqueKey>
 
+  <copyField source="severity" dest="severity_missingLast" />
+  <copyField source="severity" dest="severity_missingFirst" />
+  
   <fieldType name="text" class="solr.TextField">
     <analyzer>
       <tokenizer class="solr.WhitespaceTokenizerFactory"/>

solr/core/src/test-files/solr/collection1/conf/solrconfig.xml

@@ -489,9 +489,6 @@
 
   <queryParser name="foo" class="FooQParserPlugin"/>
 
-  <!-- deprecated parser, delete once class is deleted in Solr 8.0 -->
-  <queryParser name="lucenePlusSort" class="solr.OldLuceneQParserPlugin"/>
-  
   <updateRequestProcessorChain name="dedupe">
     <processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
       <bool name="enabled">false</bool>

solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java

@@ -869,7 +869,7 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
 
     // testing everything from query level is hard because
     // time marches on ... and there is no easy way to reach into the
-    // bowels of TrieDateField and muck with the definition of "now"
+    // bowels of DatePointField and muck with the definition of "now"
     //    ...
     // BUT: we can test that crazy combinations of "NOW" all work correctly,
     // assuming the test doesn't take too long to run...

solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java

@@ -43,7 +43,6 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
     SolrQueryRequest req = null;
     Map<String,String> args = new HashMap<>();
     lrf.args.put(CommonParams.VERSION,"2.2");
-    lrf.args.put("defType","lucenePlusSort");
 
     // compact the index, keep things from getting out of hand
 
@@ -215,16 +214,16 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
             ,"*[count(//doc)=0]"
             );
     args = new HashMap<>();
-    args.put("defType","lucenePlusSort");
+    args.put("sort","val_s1 asc");
-    req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z];val_s1 asc",
+    req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z]",
                                     "/select", 0, 0 , args);
     assertQ(req
             ,"//*[@numFound='3'] "
             ,"*[count(//doc)=0]"
             );
     args = new HashMap<>();
-    args.put("defType","lucenePlusSort");
+    args.put("sort","val_s1 desc");
-    req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z];val_s1 desc",
+    req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z]",
                                     "/select", 0, 0 , args);
     assertQ(req
             ,"//*[@numFound='3'] "
@@ -518,11 +517,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
             ,"//@numFound[.='1'] "
             ,"//int[.='-2147483648']"
             );
-    assertQ(req("id:44;num_i1 asc;")
+    assertQ(req("q", "id:44", "sort","num_i1 asc")
             ,"//doc[1]/int[.='-2147483648'] "
             ,"//doc[last()]/int[.='2147483647']"
             );
-    assertQ(req("id:44;num_i1 desc;")
+    assertQ(req("q","id:44","sort","num_i1 desc")
             ,"//doc[1]/int[.='2147483647'] "
             ,"//doc[last()]/int[.='-2147483648']"
             );
@@ -561,11 +560,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
             ,"//@numFound[.='1'] "
             ,"//long[.='-9223372036854775808']"
             );
-    assertQ(req("id:44;num_l1 asc;")
+    assertQ(req("q","id:44","sort","num_l1 asc")
             ,"//doc[1]/long[.='-9223372036854775808'] "
             ,"//doc[last()]/long[.='9223372036854775807']"
             );
-    assertQ(req("id:44;num_l1 desc;")
+    assertQ(req("q","id:44", "sort", "num_l1 desc")
             ,"//doc[1]/long[.='9223372036854775807'] "
             ,"//doc[last()]/long[.='-9223372036854775808']"
             );
@@ -611,11 +610,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
     assertQ(req("num_f1:\"-1e20\"")
             ,"//@numFound[.='1']"
             );
-    assertQ(req("id:44;num_f1 asc;")
+    assertQ(req("q", "id:44", "sort", "num_f1 asc")
             ,"//doc[1]/float[.='-Infinity'] "
             ,"//doc[last()]/float[.='NaN']"
             );
-    assertQ(req("id:44;num_f1 desc;")
+    assertQ(req("q", "id:44", "sort","num_f1 desc")
             ,"//doc[1]/float[.='NaN'] "
             ,"//doc[last()]/float[.='-Infinity']"
             );
@@ -663,11 +662,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
     assertQ(req("num_d1:\"1e-100\"")
             ,"//@numFound[.='1']"
             );
-    assertQ(req("id:44;num_d1 asc;")
+    assertQ(req("q", "id:44", "sort", "num_d1 asc")
             ,"//doc[1]/double[.='-Infinity'] "
             ,"//doc[last()]/double[.='NaN']"
             );
-    assertQ(req("id:44;num_d1 desc;")
+    assertQ(req("q","id:44","sort","num_d1 desc")
             ,"//doc[1]/double[.='NaN'] "
             ,"//doc[last()]/double[.='-Infinity']"
             );
@@ -693,27 +692,27 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
             ,"*[count(//doc)=6]"
             );
 
-    assertQ(req("id:44; a_i1 asc,b_i1 desc")
+    assertQ(req("q","id:44", "sort", "a_i1 asc,b_i1 desc")
             ,"*[count(//doc)=6] "
             ,"//doc[3]/int[.='100'] "
             ,"//doc[4]/int[.='50']"
             );
-    assertQ(req("id:44;a_i1 asc  , b_i1 asc;")
+    assertQ(req("q","id:44", "sort", "a_i1 asc  , b_i1 asc")
             ,"*[count(//doc)=6] "
             ,"//doc[3]/int[.='50'] "
             ,"//doc[4]/int[.='100']"
             );
-    assertQ(req("id:44;a_i1 asc;")
+    assertQ(req("q", "id:44", "sort", "a_i1 asc")
             ,"*[count(//doc)=6] "
             ,"//doc[1]/int[.='-1'] "
             ,"//doc[last()]/int[.='15']"
             );
-    assertQ(req("id:44;a_i1 asc , score top;")
+    assertQ(req("q","id:44","sort","a_i1 asc , score top")
             ,"*[count(//doc)=6] "
             ,"//doc[1]/int[.='-1'] "
             ,"//doc[last()]/int[.='15']"
             );
-    assertQ(req("id:44; score top , a_i1 top, b_i1 bottom ;")
+    assertQ(req("q","id:44","sort","score top , a_i1 top, b_i1 bottom ")
             ,"*[count(//doc)=6] "
             ,"//doc[last()]/int[.='-1'] "
             ,"//doc[1]/int[.='15'] "
@@ -736,36 +735,36 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
     assertQ(req("id_i:[1000 TO 1010]")
             ,"*[count(//doc)=7]"
             );
-    assertQ(req("id_i:[1000 TO 1010]; b_i1 asc")
+    assertQ(req("q","id_i:[1000 TO 1010]","sort","b_i1 asc")
             ,"*[count(//doc)=7] "
             ,"//doc[1]/int[.='50'] "
             ,"//doc[2]/int[.='100']"
             );
-    assertQ(req("id_i:[1000 TO 1010]; b_i1 desc")
+    assertQ(req("q","id_i:[1000 TO 1010]","sort"," b_i1 desc")
             ,"*[count(//doc)=7] "
             ,"//doc[1]/int[.='100'] "
             ,"//doc[2]/int[.='50']"
             );
-    assertQ(req("id_i:[1000 TO 1010]; a_i1 asc,b_i1 desc")
+    assertQ(req("q","id_i:[1000 TO 1010]","sort"," a_i1 asc,b_i1 desc")
             ,"*[count(//doc)=7] "
             ,"//doc[3]/int[@name='b_i1' and .='100'] "
             ,"//doc[4]/int[@name='b_i1' and .='50']  "
             ,"//doc[5]/arr[@name='id_i' and .='1000']"
             );
-    assertQ(req("id_i:[1000 TO 1010]; a_i1 asc,b_i1 asc")
+    assertQ(req("q","id_i:[1000 TO 1010]","sort"," a_i1 asc,b_i1 asc")
             ,"*[count(//doc)=7] "
             ,"//doc[3]/int[@name='b_i1' and .='50'] "
             ,"//doc[4]/int[@name='b_i1' and .='100']  "
             ,"//doc[5]/arr[@name='id_i' and .='1000']"
             );
     // nullfirst tests
-    assertQ(req("id_i:[1000 TO 1002]; nullfirst asc")
+    assertQ(req("q","id_i:[1000 TO 1002]","sort"," nullfirst asc")
             ,"*[count(//doc)=3] "
             ,"//doc[1]/arr[@name='id_i' and .='1002']"
             ,"//doc[2]/arr[@name='id_i' and .='1001']  "
             ,"//doc[3]/arr[@name='id_i' and .='1000']"
             );
-    assertQ(req("id_i:[1000 TO 1002]; nullfirst desc")
+    assertQ(req("q","id_i:[1000 TO 1002]","sort"," nullfirst desc")
             ,"*[count(//doc)=3] "
             ,"//doc[1]/arr[@name='id_i' and .='1002']"
             ,"//doc[2]/arr[@name='id_i' and .='1000']  "
@@ -779,16 +778,16 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
 
     // Sort parsing exception tests.  (SOLR-6, SOLR-99)
     assertQEx( "can not sort unindexed fields",
-        req( "id_i:1000; shouldbeunindexed asc" ), 400 );
+        req( "q","id_i:1000", "sort", "shouldbeunindexed asc" ), 400 );
     
     assertQEx( "invalid query format",
-        req( "id_i:1000; nullfirst" ), 400 );
+        req( "q","id_i:1000", "sort", "nullfirst" ), 400 );
 
     assertQEx( "unknown sort field",
-        req( "id_i:1000; abcde12345 asc" ), 400 ); 
+        req( "q","id_i:1000", "sort", "abcde12345 asc" ), 400 ); 
 
     assertQEx( "unknown sort order",
-        req( "id_i:1000; nullfirst aaa" ), 400 );
+        req( "q","id_i:1000", "sort", "nullfirst aaa" ), 400 );
 
     resetExceptionIgnores();
 
@@ -1166,32 +1165,31 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
             );
     args = new HashMap<>();
     args.put("fl","score ");
-    args.put("defType","lucenePlusSort");
+    args.put("sort","id desc");
-    req = new LocalSolrQueryRequest(h.getCore(), "id:44;id desc;",
+    req = new LocalSolrQueryRequest(h.getCore(), "id:44",
                                     "/select", 0, 10, args);
     assertQ(req
             ,"//result[@maxScore>0]"
             );
     args = new HashMap<>();
     args.put("fl","score ");
-    args.put("defType","lucenePlusSort");
+    req = new LocalSolrQueryRequest(h.getCore(), "id:44",
-    req = new LocalSolrQueryRequest(h.getCore(), "id:44;",
                                     "/select", 0, 10, args);
     assertQ(req
             ,"//@maxScore = //doc/float[@name='score']"
             );
     args = new HashMap<>();
     args.put("fl","score ");
-    args.put("defType","lucenePlusSort");
+    args.put("sort","id desc");
-    req = new LocalSolrQueryRequest(h.getCore(), "id:44;id desc;",
+    req = new LocalSolrQueryRequest(h.getCore(), "id:44",
                                     "/select", 0, 10, args);
     assertQ(req
             ,"//@maxScore = //doc/float[@name='score']"
             );
     args = new HashMap<>();
     args.put("fl","*,score");
-    args.put("defType","lucenePlusSort");
+    args.put("sort","id desc");
-    req = new LocalSolrQueryRequest(h.getCore(), "id:44;id desc;",
+    req = new LocalSolrQueryRequest(h.getCore(), "id:44",
                                     "/select", 0, 0 , args);
     assertQ(req
             ,"//result[@maxScore>0]"

solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java

@@ -30,10 +30,10 @@ public class TestDistributedMissingSort extends BaseDistributedSearchTestCase {
     schemaString = "schema-distributed-missing-sort.xml";
   }
   
-  String sint1_ml = "one_i1_ml";    // TrieIntField, sortMissingLast=true, multiValued=false
+  String sint1_ml = "one_i1_ml";    // int field, sortMissingLast=true, multiValued=false
-  String sint1_mf = "two_i1_mf";    // TrieIntField, sortMissingFirst=true, multiValued=false
+  String sint1_mf = "two_i1_mf";    // int field, sortMissingFirst=true, multiValued=false
-  String long1_ml = "three_l1_ml";  // TrieLongField, sortMissingLast=true, multiValued=false
+  String long1_ml = "three_l1_ml";  // long field, sortMissingLast=true, multiValued=false
-  String long1_mf = "four_l1_mf";   // TrieLongField, sortMissingFirst=true, multiValued=false
+  String long1_mf = "four_l1_mf";   // long field, sortMissingFirst=true, multiValued=false
   String string1_ml = "five_s1_ml"; // StringField, sortMissingLast=true, multiValued=false
   String string1_mf = "six_s1_mf";  // StringField, sortMissingFirst=true, multiValued=false
 

solr/core/src/test/org/apache/solr/cloud/CdcrBootstrapTest.java

@@ -239,6 +239,7 @@ public class CdcrBootstrapTest extends SolrTestCaseJ4 {
     }
   }
 
+  @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-11278")
   public void testBootstrapWithContinousIndexingOnSourceCluster() throws Exception {
     // start the target first so that we know its zkhost
     MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr"));

solr/core/src/test/org/apache/solr/core/TestLazyCores.java

@@ -781,7 +781,6 @@ public class TestLazyCores extends SolrTestCaseJ4 {
     }
   }
 
-  @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-10101")
   // Insure that when a core is aged out of the transient cache, any uncommitted docs are preserved.
   // Note, this needs FS-based indexes to persist!
   // Cores 2, 3, 6, 7, 8, 9 are transient
@@ -814,7 +813,8 @@ public class TestLazyCores extends SolrTestCaseJ4 {
       openCores.clear();
       
       // We still should have 6, 7, 8, 9 loaded, their reference counts have NOT dropped to zero 
-      checkInCores(cc, "collection6", "collection7", "collection8", "collection9");
+      checkInCores(cc, "collection1", "collection5",
+          "collection6", "collection7", "collection8", "collection9");
 
       for (String coreName : coreList) {
         // The point of this test is to insure that when cores are aged out and re-opened

solr/core/src/test/org/apache/solr/handler/SearchHandlerTest.java

@@ -63,14 +63,14 @@ public class SearchHandlerTest extends AbstractSolrTestCase {
     );
     
     // Using legacy ';' param
-    assertQ(req("q", "title:test; val_s1 desc", "defType","lucenePlusSort")
+    assertQ(req("q", "title:test", "sort","val_s1 desc")
             ,"//*[@numFound='3']"
             ,"//result/doc[1]/str[@name='id'][.='12']"
             ,"//result/doc[2]/str[@name='id'][.='11']"
             ,"//result/doc[3]/str[@name='id'][.='10']"
             );
 
-    assertQ(req("q", "title:test; val_s1 asc", "defType","lucenePlusSort")
+    assertQ(req("q", "title:test", "sort", "val_s1 asc")
             ,"//*[@numFound='3']"
             ,"//result/doc[1]/str[@name='id'][.='10']"
             ,"//result/doc[2]/str[@name='id'][.='11']"

solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java

@@ -37,7 +37,6 @@ import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
-
 import org.apache.solr.common.util.NamedList;
 import org.junit.After;
 import org.junit.Before;

solr/core/src/test/org/apache/solr/schema/EnumFieldTest.java

@@ -16,6 +16,7 @@
  */
 package org.apache.solr.schema;
 
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.Set;
 import java.util.regex.Matcher;
@@ -385,6 +386,21 @@ public class EnumFieldTest extends SolrTestCaseJ4 {
             "//doc[4]/str[@name='" + FIELD_NAME + "']/text()='High'",
             "//doc[5]/str[@name='" + FIELD_NAME + "']/text()='Critical'"
     );
+
+    // missing first....
+    for (String dir : Arrays.asList("asc", "desc")) {
+      assertQ(req("fl", "id", "q", "*:*", "sort", FIELD_NAME + "_missingFirst " + dir + ", id desc")
+              , "//doc[1]/str[@name='id']/text()='9'"
+              , "//doc[2]/str[@name='id']/text()='8'"
+              );
+    }
+    // missing last...
+    for (String dir : Arrays.asList("asc", "desc")) {
+      assertQ(req("fl", "id", "q", "*:*", "sort", FIELD_NAME + "_missingLast " + dir + ", id desc")
+              , "//doc[6]/str[@name='id']/text()='9'"
+              , "//doc[7]/str[@name='id']/text()='8'"
+              );
+    }
   }
 
   @Test

solr/core/src/test/org/apache/solr/schema/TestPointFields.java

@@ -3330,7 +3330,14 @@ public class TestPointFields extends SolrTestCaseJ4 {
   private void doTestDoublePointFunctionQuery(String field) throws Exception {
     assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof PointField);
     int numVals = 10 * RANDOM_MULTIPLIER;
-    List<Double> values = getRandomDoubles(numVals, false);
+    // Restrict values to float range; otherwise conversion to float will cause truncation -> undefined results
+    List<Double> values = getRandomList(numVals, false, () -> {
+      Float f = Float.NaN;
+      while (f.isNaN()) {
+        f = Float.intBitsToFloat(random().nextInt());
+      }
+      return f.doubleValue();
+    });
     String assertNumFound = "//*[@numFound='" + numVals + "']";
     String[] idAscXpathChecks = new String[numVals + 1];
     String[] idAscNegXpathChecks = new String[numVals + 1];

solr/core/src/test/org/apache/solr/search/CursorMarkTest.java

@@ -230,15 +230,15 @@ public class CursorMarkTest extends SolrTestCaseJ4 {
           random().nextBytes(randBytes);
           val = new BytesRef(randBytes);
         } else if (fieldName.contains("int")) {
-          val = random().nextInt();                         // TrieIntField
+          val = random().nextInt();
         } else if (fieldName.contains("long")) {
-          val = random().nextLong();                        // TrieLongField
+          val = random().nextLong();
         } else if (fieldName.contains("float")) {
-          val = random().nextFloat() * random().nextInt();  // TrieFloatField
+          val = random().nextFloat() * random().nextInt();
         } else if (fieldName.contains("double")) {
-          val = random().nextDouble() * random().nextInt(); // TrieDoubleField
+          val = random().nextDouble() * random().nextInt();
         } else if (fieldName.contains("date")) {
-          val = random().nextLong();                        // TrieDateField
+          val = random().nextLong();
         } else if (fieldName.startsWith("currency")) {
           val = random().nextDouble();
         } else if (fieldName.startsWith("uuid")) {

solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java

@@ -94,14 +94,6 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
                       " +apache +solr");
   }
 
-  @Deprecated
-  public void testQueryLucenePlusSort() throws Exception {
-    assertQueryEquals("lucenePlusSort", 
-                      "apache solr", "apache  solr", "apache solr ; score desc");
-    assertQueryEquals("lucenePlusSort", 
-                      "+apache +solr", "apache AND solr", " +apache +solr; score desc");
-  }
-
   public void testQueryPrefix() throws Exception {
     SolrQueryRequest req = req("myField","foo_s");
     try {

solr/core/src/test/org/apache/solr/search/QueryParsingTest.java

@@ -43,7 +43,6 @@ public class QueryParsingTest extends SolrTestCaseJ4 {
     SolrQueryRequest req = req("df", "text");
 
     final String[] parsersTested = new String[] {
-      OldLuceneQParserPlugin.NAME,
       LuceneQParserPlugin.NAME,
       DisMaxQParserPlugin.NAME,
       ExtendedDismaxQParserPlugin.NAME

solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java

@@ -278,7 +278,7 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 {
       q = qParser.getQuery();
       assertEquals(26, ((TermInSetQuery)q).getTermData().size());
 
-      // large numeric filter query should use TermsQuery (for trie fields)
+      // large numeric filter query should use TermsQuery
       qParser = QParser.getParser("foo_ti:(1 2 3 4 5 6 7 8 9 10 20 19 18 17 16 15 14 13 12 11)", req);
       qParser.setIsFilter(true); // this may change in the future
       qParser.setParams(params);

solr/core/src/test/org/apache/solr/search/TestSort.java

@@ -271,9 +271,9 @@ public class TestSort extends SolrTestCaseJ4 {
 
         if (r.nextBoolean()) sfields.add( new SortField(null, SortField.Type.SCORE));
         // hit both use-cases of sort-missing-last
-        sfields.add( Sorting.getStringSortField("f", reverse, sortMissingLast, sortMissingFirst) );
+        sfields.add( getStringSortField("f", reverse, sortMissingLast, sortMissingFirst) );
         if (secondary) {
-          sfields.add( Sorting.getStringSortField("f2", reverse2, sortMissingLast2, sortMissingFirst2) );
+          sfields.add( getStringSortField("f2", reverse2, sortMissingLast2, sortMissingFirst2) );
         }
         if (r.nextBoolean()) sfields.add( new SortField(null, SortField.Type.SCORE));
 
@@ -355,5 +355,20 @@ public class TestSort extends SolrTestCaseJ4 {
     return new BitDocIdSet(obs);
   }  
   
+  private static SortField getStringSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
+    SortField sortField = new SortField(fieldName, SortField.Type.STRING, reverse);
     
+    // 4 cases:
+    // missingFirst / forward: default lucene behavior
+    // missingFirst / reverse: set sortMissingLast
+    // missingLast  / forward: set sortMissingLast
+    // missingLast  / reverse: default lucene behavior
+    
+    if (nullFirst && reverse) {
+      sortField.setMissingValue(SortField.STRING_LAST);
+    } else if (nullLast && !reverse) {
+      sortField.setMissingValue(SortField.STRING_LAST);
+    }
+    return sortField;
+  }
 }